#!/usr/bin/env python3
# encoding: utf-8
# api: cli
# type: filter
# title: dingonyms
# description: fetch synonyms from various web services
# version: 0.4
# license: PD
# category: dictionary
# keywords: glossary, synonyms, antonyms
# classifiers: search, dict
# architecture: all
# depends: deb:ding (>= 1.8), python (>= 3.7), python:requests (>= 2.4)
# url: https://fossil.include-once.org/pagetranslate/wiki/dingonyms

#
# CLI tool to extract synonyms/antonyms from online services, which formats
# them into dict structures (word|alt :: definition; etc.) suitable for `ding`.
#
# It's fairly basic, and not all result sets are structured alike.
# Furthermore the extraction schemes aren't likely to last for long; web
# scraping is typically a maintenance task.
# Only scans for singular words (most services wouldn't return results
# otherwise). And might spit out error messages for charset issues as well.
#
# SYNTAX
#
#    dingonyms --thesaurus "Define"
#    dingonyms --merriamwebster "find"
#    dingonyms --synonym "Wort"
#    dingonyms --reverso "Wort"
#    dingonyms --urban "bazinga"
#    dingonyms --openthesaurus "Wort"
#    dingonyms --woxikon "Wort"
#    dingonyms --synonyme.de "Wort"
#
# Flags can be abbreviated and also combined: --thes --merrweb would query two
# services at once, or --all even all. While --en or --de run through language-
# specific functions. (See the man page for more details.)
# Reason for supporting multiple sites is allowing to fall back on others if
# one extraction method breaks.
#
# CONFIG IN ~/.dingrc (take care to change `3` to available index)
#
#    set searchmeth(3,name) {Synonyms}
#    set searchmeth(3,type) {3}

# You might have to symlink ~/.local/bin/dingonyms into ~/bin after
# installation. pip-package binaries are often only picked up in
# terminal/interactive shells.
#



import sys, os, asyncio, re
import requests, json, html, textwrap
try:
    sys.stdout.reconfigure(encoding="utf-8")
except:
    pass# and pray

    """
        Online service backends and extraction.
        Not much of a real object, just a function collection.
        Docblock of each function starts with a --param regex.
    """
    def __init__(self):
        pass
    def run(self, callback, *a, **kw):
        """ stub for non-threaded calls, simply invokes callback right away """
        return callback(*a, **kw)
    def set_no_antonyms(self, *a):
        """ no | na | no-?an?to?\w* """
        out.no_antonyms = True
    def set_no_headers(self, *a):
        """ nh | no-?he?a?d?\w* """
        out.no_headers = True
    def set_async(self, *a):
        """ async | a?io | threads? | parallel """
        # Just redefines self.run() to utilize asyncio threads (not real async task→result schemes)
        threads = asyncio.get_event_loop()
        def run(callback, *a, **kw):
            threads.run_in_executor(None, lambda: callback(*a, **kw))
        self.run = run
        return threads # might have to implement a wait in __main__
       

    def thesaurus_raw(self, word, html=""):
        """ thesaurus-?(raw|htm) | raw | htm """
        if not html:
            html = http_get("https://www.thesaurus.com/browse/%s" % word)
        ls = []

                        )
        except:
            out.group("failed JSON extraction")
            self.thesaurus_raw(word, html)


    def openthesaurus(self, word):
        """ openthesaurus | open | ot | ope?nt\w* """
        out.site("OpenThesaurus.de")
        # there's a proper API here
        j = json.loads(http_get("https://www.openthesaurus.de/synonyme/search?q=%s&format=application/json&supersynsets=true" % word))
        for terms in j["synsets"]:
            supersyn = ""
            if terms["supersynsets"] and terms["supersynsets"][0]:
                supersyn = "; ".join([w["term"] for w in terms["supersynsets"][0]][0:3])
                supersyn = "("+supersyn+")"
            out.alternatives(
                "%s %s" % (word, supersyn),
                [w["term"] for w in terms["terms"]]
            )
            

    def woxikon(self, word):
        """ woxikon | w | wx | wxi?k\w* """
        out.site("Woxikon.de")
        html = http_get("https://synonyme.woxikon.de/synonyme/%s.php" % word)
        ls = []
        for add_word, grp in re.findall(' <a\s+href="[^"]+/synonyme/[\w.%-]+">(\w[^<]+)</a> | Bedeutung:\s+<b>(\w[^<]+)< | </html ', html, re.X):
            if add_word:
                ls.append(add_word)
            elif ls:
                out.alternatives("%s (%s)" % (word, grp), ls)
                ls = []


    def synonyme_de(self, word):
        """ synonyme[_\-.]?de | sd | de[_-]?syn\w* """
        out.site("Synonyme.de")
        html = http_get("https://www.synonyme.de/%s/" % word)
        ls = []
        rx = '''
            <span><b>(\w[^<]+)</b>\s+-\s+Bedeutung\s+für\s+(\w\S+)\s+\((\w+)\) |
            <p><span>\s*(Sonstige\s\d+) |
            <a\s+href="/\w[^/">]+/">\s*(\w\S+)\s*</a> |
            </html>
        '''
        for set_grp, set_word, verb, grp, add_word in re.findall(rx, html, re.X):
            if add_word:
                ls.append(add_word)
            elif ls:
                out.alternatives(word, ls)
                ls = []
            if set_grp or verb:
                word = "%s {%s} (%s)" % (set_word, verb[0].lower(), set_grp)
            elif grp:
                word = "%s (%s)" % (set_word, grp)


    def merriamwebster(self, word):
        """ merriam-?webster | mw | mer\w* | m\w*w\*b\w* | \w*web\w* """
        out.site("Merriam-Webster.com")
        html = http_get("https://www.merriam-webster.com/thesaurus/%s" % word)
        grp = "Synonyms"
        ls = []
        # word links here are decorated with types (noun/verb), and groups neatly include a reference to the search term (or possibly a different related term)
        rx = ' href="/thesaurus/([\w.-]+)\#(\w+)" | ="function-label">(?:Words\s)?(Related|Near\sAntonyms|Antonyms|Synonyms|\w+)\s\w+\s<em>([\w.-]+)</em> | (</html)'
        for add_word, verb, set_grp, set_word, endhtml in re.findall(rx, html, re.X):
            #print(row)
            if add_word:
                ls.append("%s {%s}" % (add_word, verb[0]))
            elif ls:
                out.alternatives(word + " {%s}" % grp, ls)
                ls = []
            if set_grp or set_word:
                grp, word = set_grp, set_word


    def synonym_com(self, word):
        """
            synonyms?(\.?com)?$ | s$ | sy$ | sy?n\w*\\b(?<!de) |
            
            Doing a fair bit of super-specific HTML transforms here, because
            there's a wealth of decoration. DOM traversal might have been simpler
            in this case.
        """
        out.site("Synonym.com")
        html = http_get("https://www.synonym.com/synonyms/%s" % word)
        html = re.sub('^.+?="result-group-container">', "", html, 0, re.S)
        html = re.sub('<div class="rightrail-container">.+$', "", html, 0, re.S)

            else:
                if ls:
                    out.alternatives(word, ls)
                    ls = []
                if antonyms:
                    word = " 🞬 {Antonyms}"
                    continue


                defs = re.sub('(<[^>]+>|\s+)+', " ", defs, 0, re.S).strip()
                defs = " |   ".join(textwrap.wrap(defs, 50))
                word = group + " {" + verb + "} [" + pron + "] |  (" + defs + ")"

                
    def urban(self, word):
        """ urban | u | u\w*[brn]\w* """

    def pt(self, word):
        """ pt | po[rtugueseal]* """
        self.reverso(word, "pt")
            

    def all(self, word):
        """ all | a | Run through all available services """
        for method in (self.thesaurus, self.merriamwebster, self.synonym_com, self.reverso, self.openthesaurus, self.woxikon, self.urban):
            self.run(method, word)
    def en(self, w):
        """ en | english """
        self.run(self.thesaurus, w)
        self.run(self.merriamwebster, w)
        self.run(self.synonym, w)
        self.run(self.reverso, w)
    def de(self, w):
        """ de | german """
        self.run(self.openthesaurus, w)
        self.run(self.woxikon, w)
        self.run(self.reverso, w, "de")
        self.run(self.synonyme_de, w)

# instantiate right away
lookup = lookup()


# __main__
def __main__():
    if len(sys.argv) == 1:
        return print("Syntax :: dingonyms --site word")
    word = "search"
    methods = []
    for arg in sys.argv[1:]:
        # separate --params from search word
        if not arg or arg == "--":
            continue
        elif not re.match("-", arg):
            word = arg
        else:
            for name, method in vars(lookup.__class__).items():
                if re.match("^[/+\–\-]+(%s)$" % method.__doc__, arg, re.X|re.I|re.U):
                    methods.append(name)
    if not methods:
        methods = ["thesaurus"]
    def run_methods(names, *args, is_async=False):
        for name in names:
            callback = getattr(lookup, name)
            if is_async and name not in ("all", "de", "en"):
                lookup.run(callback, *args)
            else:
                callback(*args)
            is_async |= name == "set_async"
    run_methods(methods, word.lower())
    pass
if __main__ == "__init__":
    __main__()

.\"t
.\" Automatically generated by Pandoc 2.5
.\"
.TH "dingonyms" "1" "" "for ding" "Version 0.4"
.hy
.SH NAME
.PP
dingonyms \[em] basic synonym/antonym lookup tool
.SH SYNOPSIS
.PP
\f[B]dingonyms\f[R] \[dq]word\[dq]
.PP
\f[B]dingonyms\f[R] [\f[I]\-\-thesaurus\f[R]]
[\f[I]\-\-merriamwebster\f[R]] [\f[I]\-\-synonym\f[R]] \[lq]find\[rq]
.PP
\f[B]dingonyms\f[R] [\f[I]\-\-openthesaurus\f[R] | \f[I]\-\-woxikon\f[R]
| \f[I]\-\-urban\f[R]] \[lq]Wort\[rq]
.PP
\f[B]dingonyms\f[R] [\f[I]\-\-reverso\f[R] | \f[I]\-\-fr\f[R] |
\f[I]\-\-es\f[R] | \f[I]\-\-it\f[R] | \f[I]\-\-jp\f[R]]
\[lq]bonjour\[rq]
.PP
\f[B]dingonyms\f[R] [\f[I]\-\-no\-antonyms\f[R]] [\f[I]\-\-async\f[R]]
\f[I]\-\-all\f[R] \[lq]word\[rq]
.SH DESCRIPTION
.PP
To be used as lookup tool for \f[B]ding\f[R](1).
Scans the specified online service for synonyms.
.PD 0
.P
.PD

T{
\-\-woxikon
T}@T{
\-wx \-\-woxi
T}@T{
DE
T}
T{
\-\-synonyme_de
T}@T{
\-sd \[en]desyn
T}@T{
DE
T}
T{
\-\-no\-antonyms
T}@T{
\-na
T}@T{
FLAG
T}
T{
\-\-no\-headers
T}@T{
\-nh
T}@T{
FLAG
T}
T{
\-\-async
T}@T{
\[en]parallel \[en]io
T}@T{
FLAG
T}
.TE
.PP
They can also be combined (\f[B]dingonyms\f[R] \f[I]\-t \-mw \-u\f[R]
\[lq]find\[rq]), so two or more services get queried at once.
.IP \[bu] 2
And obviously there\[cq]s also \f[I]\-\-all\f[R] to query all supported
sites.
.IP \[bu] 2
Reverso offers some alternative languages (\-fr, \-it, \-ru),
.IP \[bu] 2
while \f[I]\-\-en\f[R] and \f[I]\-\-de\f[R] \f[B]combine\f[R] the other
main services.
.PP
Execution flags must be noted before any translation service:
.IP \[bu] 2
The \f[I]\-\-no\-antonyms\f[R] flag isn\[cq]t consistently supported by
all backends.
(It\[cq]s actually more effort to skip around unwanted data, rather just
outputting it as it comes in.)

.IP \[bu] 2

\f[I]\-\-no\-headers\f[R] only skips the \[lq]\[u270E] {Sitename}\[rq]
titles before any output.
.IP \[bu] 2
And \f[I]\-\-asnyc\f[R] speeds up the processing of multiple sites, at
the cost of potentially intermingling their output.
It\[cq]s a rather crude threading scheme, and no care has been taken to
rewrite dingonyms to fully support that.
It\[cq]s primarily meant for \-\-all and \-\-de, \-\-en.
There\[cq]s preliminary support for asyncing selected sites as well
(e.g.\ \f[B]dingonyms \[en]async ++thes ++merr ++urb
\[lq]word\[rq]\f[R]), where it would be less likely mash up antonyms.
(The execution speed of the regex+print loops determines if there\[cq]s
intermingling, if the HTTP requests complete at the same time.)
.SH CONFIG
.PP
The search services can be configured through the GUI (somewhat fiddly
with Change+New), or by editing \f[I]\[ti]/.dingrc\f[R] directly.
To have all services present for example:
.IP
.nf

set searchmeth(6,maxresults) {200}
set searchmeth(6,minlength) {2}
set searchmeth(6,shapedresult) {1}
set searchmeth(6,foldedresult) {0}
\f[R]
.fi
.PP
\f[B]Beware not to overwrite existing/custom entries.\f[R]
.PD 0
.P
.PD
If the entries don\[cq]t show up, ding might not see dingonyms in the
PATH:
.SH SETUP
.PP
ding started from the Desktop environment might not find dingonyms, when
installed per pip.
You\[cq]ll have to symlink it:
.RS
.PP
\f[I]ln \-s \[ti]/.local/bin/dingonyms \[ti]/bin/\f[R]
.RE
.PP
Or reconfigure \f[I]\[ti]/.profile\f[R] to generally include
\[ti]/.local/bin; which doesn\[cq]t seem to be standard in most distros.
.SH SEE ALSO
.PP
\f[B]ding\f[R](1), \f[B]python\f[R](1)

% dingonyms(1) for ding | Version 0.4


NAME
====

dingonyms — basic synonym/antonym lookup tool


SYNOPSIS
========

  **dingonyms** \"word\"

  **dingonyms** \[*\--thesaurus*] \[*\--merriamwebster*] \[*\--synonym*] "find"

  **dingonyms** \[*\--openthesaurus* | *\--woxikon* | *\--urban*] "Wort"

  **dingonyms** \[*\--reverso* | *\--fr* | *\--es* | *\--it* | *\--jp*] "bonjour"

  **dingonyms** \[*\--no-antonyms*] \[*\--async*] *\--all* "word"



DESCRIPTION
===========

To be used as lookup tool for **ding**(1). Scans the specified online service for synonyms.  
It's not very useful as CLI tool by itself, because of the particular dict formatting.



PARAMS
======

Parameters can be specified with one \- or two \-\- dashes (or \+ signs even).  
They can also be abbreviated.

+-------------------+---------------------------------------------+-------+
| \--urban          | -u --urb \--ubn                             |LEXICON|
+-------------------+---------------------------------------------+-------+
| \--openthesaurus  | -ot \--othes --open                         | DE    |
+-------------------+---------------------------------------------+-------+
| \--woxikon        | -wx \--woxi                                 | DE    |
+-------------------+---------------------------------------------+-------+
| \--synonyme_de    | -sd --desyn                                 | DE    |
+-------------------+---------------------------------------------+-------+
| \--no-antonyms    | -na                                         | FLAG  |
+-------------------+---------------------------------------------+-------+
| \--no-headers     | -nh                                         | FLAG  |
+-------------------+---------------------------------------------+-------+
| \--async          | --parallel --io                             | FLAG  |
+-------------------+---------------------------------------------+-------+

They can also be combined (**dingonyms** *-t -mw -u* "find"), so two
or more services get queried at once.

 * And obviously there's also *\--all* to query all supported sites.
 * Reverso offers some alternative languages (-fr, -it, -ru),
 * while *\--en* and *\--de* **combine** the other main services.

Execution flags must be noted before any translation service:

 * The *\--no-antonyms* flag isn't consistently supported by all backends.
   (It's actually more effort to skip around unwanted data, rather just
   outputting it as it comes in.)
 * *\--no-headers* only skips the "✎ {Sitename}" titles before any output.
 * And *\--asnyc* speeds up the processing of multiple sites, at the cost
   of potentially intermingling their output. It's a rather crude
   threading scheme, and no care has been taken to rewrite dingonyms to
   fully support that. It's primarily meant for \--all and \--de, \--en.
   There's preliminary support for asyncing selected sites as well
   (e.g. **dingonyms --async ++thes ++merr ++urb "word"**), where it would
   be less likely mash up antonyms. (The execution speed of the regex+print
   loops determines if there's intermingling, if the HTTP requests complete
   at the same time.)


CONFIG
======

The search services can be configured through the GUI (somewhat
fiddly with Change+New), or by editing *~/.dingrc* directly.

    set searchmeth(6,grepopts) {--urban}
    set searchmeth(6,maxlength) {30}
    set searchmeth(6,maxresults) {200}
    set searchmeth(6,minlength) {2}
    set searchmeth(6,shapedresult) {1}
    set searchmeth(6,foldedresult) {0}

**Beware not to overwrite existing/custom entries.**  
If the entries don't show up, ding might not see dingonyms in the PATH:


SETUP
=====

ding started from the Desktop environment might not find dingonyms, when
installed per pip. You'll have to symlink it:

> *ln -s ~/.local/bin/dingonyms ~/bin/*

Or reconfigure *~/.profile* to generally include ~/.local/bin; which doesn't
seem to be standard in most distros.


SEE ALSO
========

**ding**(1), **python**(1)