# license: PD
# category: dictionary
# keywords: glossary, synonyms, antonyms
# classifiers: search, dict
# architecture: all
# depends: deb:ding (>= 1.8), python (>= 3.6), python:requests (>= 2.4)
# url: https://fossil.include-once.org/pagetranslate/wiki/dingonyms
# doc-format: text/markdown
#
# CLI tool to extract synonyms/antonyms from online services, which formats
# them into dict structures (`word|alt :: definition; etc.`) suitable for
# [`ding`](https://www-user.tu-chemnitz.de/~fri/ding/).
#
# ![img](https://fossil.include-once.org/pagetranslate/raw/ac0a03111ddc72?m=image/png)
#
# It's fairly basic, and not all result sets are structured alike.
# Furthermore the extraction schemes aren't likely to last for long; web
# scraping is typically a maintenance task.  
# Only scans for singular words (most services wouldn't return results
# otherwise). And might spit out error messages for charset issues as well.
#
# ### SYNTAX
#
# >     dingonyms --thesaurus "Define"
# >     dingonyms --merriamwebster "find"
# >     dingonyms --synonym "Wort"
# >     dingonyms --reverso "Wort"
# >     dingonyms --urban "bazinga"
# >     dingonyms --openthesaurus "Wort"
# >     dingonyms --woxikon "Wort"
# >     dingonyms --synonyme.de "Wort"
#
# Flags can be abbreviated and also combined: `--thes --merrweb` would query two
# services at once, or `--all` even all. While `--en` or `--de` run through language-
# specific functions. (See the man page for more details. There is a man page.)
# Reason for supporting multiple sites is allowing to fall back on others if
# one extraction method breaks.
#
# ### CONFIG IN ~/.dingrc (take care to change `3` to available index)
#
# >     set searchmeth(3,name) {Synonyms}
# >     set searchmeth(3,type) {3}
# >     set searchmeth(3,dictfile) {}
# >     set searchmeth(3,separator) { :: }
# >     set searchmeth(3,language1) {Group}
# >     set searchmeth(3,language2) {Synonyms}
# >     set searchmeth(3,grepcmd) {dingonyms}
# >     set searchmeth(3,grepopts) {--async --thesaurus --merriamwebster --synonyms}
# >     set searchmeth(3,maxlength) {30}
# >     set searchmeth(3,maxresults) {200}
# >     set searchmeth(3,minlength) {2}
# >     set searchmeth(3,shapedresult) {1}
# >     set searchmeth(3,foldedresult) {0}
#
# You might want to add one entry for each search backend even.
# (Unique index, title/name, and grepopts --parameter each.)
#
# ### SETUP (pip3 install -U dingonyms)
#
# You might have to symlink `~/.local/bin/dingonyms` into `~/bin` after
# installation. pip-package binaries are often only picked up in
# terminal/interactive shells.
#



import sys, os, asyncio, re

        threads = asyncio.get_event_loop()
        def run(callback, *a, **kw):
            threads.run_in_executor(None, lambda: callback(*a, **kw))
        self.run = run
        return threads # might have to implement a wait in __main__
       

    def thesaurus_raw(self, word, lang=None, html=""):
        """ thesaurus-?(raw|htm) | raw | htm """
        if not html:
            html = http_get(f"https://www.thesaurus.com/browse/{word}")
        ls = []
        grp = "synonym"
        # look for word links, or grouping divs (not many reliable html structures or legible class names etc.)
        rx = ''' "/browse/([\w.-]+)" | <div\s+id="(meanings|synonyms|antonyms|[a-z]+)" | (</html) '''

                    if grp in d and len(d[grp]):
                        out.alternatives(
                            "%s {%s} (%s)" % (d["entry"], d["pos"], d["definition"]),
                            [word["term"] for word in d[grp]]
                        )
        except:
            out.group("failed JSON extraction")
            self.thesaurus_raw(html=html)


    def openthesaurus(self, word):
        """ openthesaurus | open | ot | ope?nt\w* """
        # there's a proper API here
        j = json.loads(
            http_get(f"https://www.openthesaurus.de/synonyme/search?q={word}&format=application/json&supersynsets=true")

            if not text:
                continue
            # at this point, it's becoming custom output to wrap flow text into Tk/ding window
            text = re.sub("^[\s|]+", "", text)
            text = textwrap.wrap(text, 45)
            print(f"{word} {' | '*(len(text)-1)} :: {'|'.join(text)}")

#@todo?
#https://en.wiktionary.org/w/api.php?action=query&format=json&titles=bluebird&prop=extracts&exintro=True&explaintext=True
#http://www.freedictionary.org/?Query=bluebird

    def reverso(self, word, lang="en"):
        """
            reverso | re?v\w* |
            
            Now this one is interesting, because it provides for additional languages.
        """
        if not re.match("^(nl|it|jp|fr|es|pt)$", lang):
            lang = "en"
        html = http_get(f"https://synonyms.reverso.net/synonym/{lang}/{word}")
        out.site("Reverso.net")
        rx = """
           ="words-options.*?<p>(\w+)</p> |
           <a\shref="/synonym/\w+/([\w.-]+)" |
           <p>(Antonyms):</p> |
           (</html>)
        """
        grp = word
        ls = []
        for set_verb, add_word, antonyms, endhtml in re.findall(rx, html, re.X|re.S|re.U):
            if add_word:
                ls.append(add_word)
            elif ls:
                out.alternatives(grp, ls)
                ls = []
            if antonyms:
                grp = "🞬 " + grp + " ❙ {Antonyms}"
            if set_verb:
                grp = word + " {%s}" % set_verb
        
    

    def dictcc(self, word, lang="www"):
        """  dictcc | cc | (en|de)[-_/:>]+(\w\w) """
        lang = re.sub('\W', '', lang)
        if not re.match("^(en|de)(en|de|sv|is|ru|ro|fr|it|sk|pt|nl|hu|fi|la|es|bg|hr|no|cs|da|tr|pl|eo|sr|el|sk|fr|hu|nl|pl|is|es|sq|ru|sv|no|fi|it|cs|pt|da|hr|bg|ro)", lang):
            lang = "www"
        html = http_get(f"https://{lang}.dict.cc/?s={word}")
        out.site("dict.cc")
        rx = """

            <td[^>]*> (<(?:a|dfn).+?) </td>\s*
            <td[^>]*> (<(?:a|dfn|div).+?) </td></tr>
             | ^var\dc\dArr = new Array\((.+)\)    # json list just contains raw words however
             | (</script><table)
        """

        for left,right,json,endhtml in re.findall(rx, html, re.X|re.M):
            if endhtml:

                break


            out.alternatives(
                "| ".join(textwrap.wrap(out.unhtml(left), 50)),
                textwrap.wrap(out.unhtml(right), 50)
            )
            

    def all(self, word):
        """ all | a | Run through all available services """
        for method in (self.thesaurus, self.merriamwebster, self.synonym_com, self.reverso, self.openthesaurus, self.woxikon, self.urban):
            self.run(method, word)
    def en(self, w):

        self.run(self.reverso, w, "de")
        self.run(self.synonyme_de, w)

# instantiate right away
lookup = lookup()


# entry_points for console_scripts
def __main__():
    if len(sys.argv) == 1:
        return print("Syntax :: dingonyms --site word")
    word = "search"
    methods = []
    for arg in sys.argv[1:]:
        # separate --params from search word
        if not arg or arg == "--":
            continue
        elif not re.match("[/+\–\-]+", arg):
            word = arg
        else:
            for name, method in vars(lookup.__class__).items():
                # match according to method name or regex in docstring
                rx = method.__doc__ or method.__name__
                m = re.match(f"^ [/+\–\-]+ ({rx}) $", arg, re.X|re.I|re.U)
                if m:
                    methods.append((name, m.group(1).lower()))  # list of method names and --param
    if not methods:
        methods = [("thesaurus","-t")]
    # invoke method names, potentially after --async got enabled (this is actually a workaround to prevent --all from doubly running in the thread pool)
    def run_methods(name_and_param, word, is_async=False):
        for name, param in name_and_param:
            callback = getattr(lookup, name)
            args = [word]
            if callback.__code__.co_argcount == 3: # pass --lang param where supported
                args.append(param)
            if is_async and name not in ("all", "de", "en"):
                args.prepend(callback)
                callback = lookup.run
            if callback:
                callback(*args)
            is_async |= name == "set_async"
    run_methods(methods, word.lower())
    pass
def dictcc():
    bin, lang, *word = sys.argv # syntax: dictcc en-fr -- "word"
    if word:
        word = [w for w in word if not w.startswith("-")][0]
    else:
        word, lang = lang, "www"
    lookup.set_no_headers()
    lookup.dictcc(word, lang)
if __main__ == "__init__":
    __main__()

.PP
\f[B]dingonyms\f[R] [\f[I]\-\-reverso\f[R] | \f[I]\-\-fr\f[R] |
\f[I]\-\-es\f[R] | \f[I]\-\-it\f[R] | \f[I]\-\-jp\f[R]]
\[lq]bonjour\[rq]
.PP
\f[B]dingonyms\f[R] [\f[I]\-\-no\-antonyms\f[R]] [\f[I]\-\-async\f[R]]
\f[I]\-\-all\f[R] \[lq]word\[rq]
.PP
\f[B]dictcc\f[R] [\f[I]en\-es\f[R] | \f[I]en\-ru\f[R] | \f[I]de\-nl\f[R]
| \f[I]de\-it\f[R]] \[lq]translate\[rq]
.SH DESCRIPTION
.PP
To be used as lookup tool for \f[B]ding\f[R](1).
Scans the specified online service for synonyms.
.PD 0
.P
.PD

T{
\-\-urban
T}@T{
\-u \[en]urb \-\-ubn
T}@T{
LEXICON
T}
T{
\-\-dictcc
T}@T{
\[en]en\-es \[en]en\-it \[en]de\-fr \[en]en\-pt
T}@T{
DICT
T}
T{
\-\-openthesaurus
T}@T{
\-ot \-\-othes \[en]open
T}@T{
DE
T}

T{
\-\-synonyme_de
T}@T{
\-sd \[en]desyn
T}@T{
DE
T}
T{
\-\-all
T}@T{
(\-t \-mw \-syn \-rev \-ot \-wx \-urban)
T}@T{
MIXED
T}
T{
\-\-en
T}@T{
(\-t \-mw \-syn \-rev)
T}@T{
MIXED
T}
T{
\-\-de
T}@T{
(\-ot \-wx \-sd)
T}@T{
MIXED
T}
T{
\-\-no\-antonyms
T}@T{
\-na
T}@T{
FLAG
T}

FLAG
T}
.TE
.PP
They can also be combined (\f[B]dingonyms\f[R] \f[I]\-t \-mw \-u\f[R]
\[lq]find\[rq]), so two or more services get queried at once.
.IP \[bu] 2
And obviously there\[cq]s also \f[I]\-\-all\f[R] to query \[lq]all\[rq]
supported sites.
.IP \[bu] 2
While \f[I]\-\-en\f[R] and \f[I]\-\-de\f[R] combine the other main
services.
.IP \[bu] 2
Reverso offers some alternative languages (\-fr, \-it, \-ru).
.IP \[bu] 2
And \-\-dictcc that allows for pair\-language specifiers (\-en\-tr,
\-de\-fr)

.PP
Execution flags should/must be noted before any translation service:
.IP \[bu] 2
The \f[I]\-\-no\-antonyms\f[R] flag isn\[cq]t consistently supported by
all backends.
(It\[cq]s actually more effort to skip around unwanted data, rather just
outputting it as it comes in.)

rewrite dingonyms to fully support that.
It\[cq]s primarily meant for \-\-all and \-\-de, \-\-en.
There\[cq]s preliminary support for asyncing selected sites as well
(e.g.\ \f[B]dingonyms \-\-async ++thes ++merr ++urb
\[lq]word\[rq]\f[R]), where it would be less likely mash up antonyms.
(The execution speed of the regex+print loops determines if there\[cq]s
any mashup whenever HTTP requests complete at the same time.)
.SH DICTCC
.PP
The dictcc mode is special, in that it provides an actual translation
dictionary between different languages.
The default mode is en\[->]de, but alternative \f[I]\-\-from\-to\f[R]
options can be specified.
Additionally there\[cq]s a commandline shortcut just for this backend:
.PP
\f[B]dictcc\f[R] \f[I]en\-it\f[R] \-\- \[lq]translate\[rq]
.PP
The \-\- is optional as is the \-\-lng\-lng dash prefix.
Otherwise it\[cq]s the same as invoking dingonyms.
.SH CONFIG
.PP
The search services can be configured through the GUI (somewhat fiddly
with Change+New), or by editing \f[I]\[ti]/.dingrc\f[R] directly.
To have all services present for example:
.IP
.nf

  **dingonyms** \[*\--openthesaurus* | *\--woxikon* | *\--urban*] "Wort"

  **dingonyms** \[*\--reverso* | *\--fr* | *\--es* | *\--it* | *\--jp*] "bonjour"

  **dingonyms** \[*\--no-antonyms*] \[*\--async*] *\--all* "word"

  **dictcc** \[*en-es* | *en-ru* | *de-nl* | *de-it*] "translate"


DESCRIPTION
===========

To be used as lookup tool for **ding**(1). Scans the specified online service for synonyms.  
It's not very useful as CLI tool by itself, because of the particular dict formatting.

| Parameter         | Aliases                                     | Class |
|-------------------|---------------------------------------------|-------|
| \--thesaurus      | -t --thes                                   | EN    |
| \--merriamwebster | -mw --merr \--webster \--merweb             | EN    |
| \--synonym        | -s --syn -\-synonym.com                     | EN    |
| \--reverso        | \--rev  //  -fr -es -it -pt -nl -ru -jp     | EN/\**|
| \--urban          | -u --urb \--ubn                             |LEXICON|
| \--dictcc         | --en-es --en-it --de-fr --en-pt             | DICT  |
| \--openthesaurus  | -ot \--othes --open                         | DE    |
| \--woxikon        | -wx \--woxi                                 | DE    |
| \--synonyme_de    | -sd --desyn                                 | DE    |
| \--all            | (-t -mw -syn -rev -ot -wx -urban)           | MIXED |
| \--en             | (-t -mw -syn -rev)                          | MIXED |
| \--de             | (-ot -wx -sd)                               | MIXED |
| \--no-antonyms    | -na                                         | FLAG  |
| \--no-headers     | -nh                                         | FLAG  |
| \--async          | \--parallel \--io                           | FLAG  |

They can also be combined (**dingonyms** *-t -mw -u* "find"), so two
or more services get queried at once.

 * And obviously there's also *\--all* to query "all" supported sites.
 * While *\--en* and *\--de* combine the other main services.
 * Reverso offers some alternative languages (-fr, -it, -ru).

 * And \--dictcc that allows for pair-language specifiers (-en-tr, -de-fr)

Execution flags should/must be noted before any translation service:

 * The *\--no-antonyms* flag isn't consistently supported by all backends.
   (It's actually more effort to skip around unwanted data, rather just
   outputting it as it comes in.)
 * *\--no-headers* to omit the "✎ {Sitename}" titles before any output.
 * And *\--asnyc* speeds up the processing of multiple sites, at the cost
   of potentially intermingling their output. It's a rather crude
   threading scheme, and no care has been taken to rewrite dingonyms to
   fully support that. It's primarily meant for \--all and \--de, \--en.
   There's preliminary support for asyncing selected sites as well
   (e.g. **dingonyms \--async ++thes ++merr ++urb "word"**), where it would
   be less likely mash up antonyms. (The execution speed of the regex+print
   loops determines if there's any mashup whenever HTTP requests complete
   at the same time.)


DICTCC
======

The dictcc mode is special, in that it provides an actual translation
dictionary between different languages. The default mode is en→de, but
alternative *\--from-to* options can be specified. Additionally there's
a commandline shortcut just for this backend:

  **dictcc** *en-it* \-- "translate"

The \-- is optional as is the \--lng-lng dash prefix. Otherwise it's the
same as invoking dingonyms.


CONFIG
======

The search services can be configured through the GUI (somewhat
fiddly with Change+New), or by editing *~/.dingrc* directly.
To have all services present for example:

    #long_description="README.md",
    packages=[""],
    package_dir={"": "."},
    py_modules=['dingonyms'],
    entry_points={
        "console_scripts": [
            "dingonyms=dingonyms:__main__",
            "dictcc=dingonyms:dictcc",
        ]
    }
)