Overview
Comment:prepare packae uncovery, fixate utf-8 decode with duck typing and exception
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: b2ed9dcfa1758eed2b732e634401932a3869431ca4cc0402f23f7b8d4246a468
User & Date: mario on 2022-10-31 06:10:48
Other Links: manifest | tags
Context
2022-10-31
06:11
add get_data probe in pyz test check-in: 77cb85f450 user: mario tags: trunk
06:10
prepare packae uncovery, fixate utf-8 decode with duck typing and exception check-in: b2ed9dcfa1 user: mario tags: trunk
2022-10-30
15:54
update template (RTD-like) check-in: feb199bddf user: mario tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified pluginconf/__init__.py from [9584d270b3] to [66c11c0c8c].

︙︙
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# api-docs: https://fossil.include-once.org/pluginspec/doc/trunk/html/index.html
# docs: https://fossil.include-once.org/pluginspec/
# url: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data
# config: -
# format: off
# permissive: 0.75
# pylint: disable=invalid-name
# console-scripts: flit-pluginconf=pluginconf.flit:main, pluginconf.flit=pluginconf.flit:main
#
# Provides plugin lookup and meta data extraction utility functions.
# It's used to abstract module+option management in applications.
# For consolidating internal use and external/tool accessibility.
# Generally these functions are highly permissive / error tolerant,
# to preempt initialization failures for applications.
#








|








14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# api-docs: https://fossil.include-once.org/pluginspec/doc/trunk/html/index.html
# docs: https://fossil.include-once.org/pluginspec/
# url: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data
# config: -
# format: off
# permissive: 0.75
# pylint: disable=invalid-name
# console-scripts: flit-pluginconf=pluginconf.flit:main
#
# Provides plugin lookup and meta data extraction utility functions.
# It's used to abstract module+option management in applications.
# For consolidating internal use and external/tool accessibility.
# Generally these functions are highly permissive / error tolerant,
# to preempt initialization failures for applications.
#
︙︙
107
108
109
110
111
112
113

114
115
116
117
118
119
120



import sys
import os
import os.path
import re
import functools

import pkgutil
import inspect
try:
    from gzip import decompress as gzip_decode  # Py3 only
except ImportError:
    try:
        from compat2and3 import gzip_decode   # st2 stub








>








107
108
109
110
111
112
113
114
115
116
117
118
119
120
121



import sys
import os
import os.path
import re
import functools
import itertools
import pkgutil
import inspect
try:
    from gzip import decompress as gzip_decode  # Py3 only
except ImportError:
    try:
        from compat2and3 import gzip_decode   # st2 stub
︙︙
273
274
275
276
277
278
279
280
281

282
283
284
285
286
287
288
289

290
291
292
293
294
295
296

297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

321


322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345

    | src         | str     | From already uncovered script code.             |
    | module      | str     | Lookup per pkgutil, relative to plugin_base     |
    | frame       | int     | Extract comment header of caller (default).     |
    | extra_base  | list    | Additional search directories.                  |
    | max_length  | list    | Maximum size to read from files (6K default).   |
    | **Returns** | dict    | Extracted comment fields, with config: preparsed|

    The result dictionary (`PluginMeta`) has fields accessible as e.g. `pmd["title"]`
    or `pmd.version`. The documentation block after all fields: is called

    `["doc"]`. And `pmd.config` already parsed as a list (`OptionList`) of dictionaries.
    """

    # Try via pkgutil first,
    # find any plugins.* modules, or main packages
    if module:
        filename = module
        for base in plugin_base + kwargs.get("extra_base", []):

            try:
                #log.debug(f"mod={base} fn={filename}.py")
                src = get_data(filename=filename+".py", decode=True, file_root=base)
                if src:
                    break
            except (IOError, OSError, FileNotFoundError):
                continue  # plugin_meta_extract() will print a notice later


    # Real filename/path
    elif filename and os.path.exists(filename):
        src = open(filename).read(kwargs.get("max_length", 6144))

    # Else get source directly from caller
    elif not src and not filename:
        module = inspect.getmodule(sys._getframe(frame+1)) # decorator+1
        filename = inspect.getsourcefile(module)
        src = inspect.getcomments(module)

    # Assume it's a filename matches …/base.zip/…/int.py
    elif filename:
        int_fn = ""
        while len(filename) and not os.path.exists(filename): # pylint: disable=len-as-condition
            filename, add = os.path.split(filename)
            int_fn = add + "/" + int_fn
        if len(filename) >= 3 and int_fn and zipfile.is_zipfile(filename):
            src = zipfile.ZipFile(filename, "r").read(int_fn.strip("/"))

    # Extract source comment into meta dict
    if not src:
        src = ""
    if isinstance(src, bytes):

        src = src.decode("utf-8", errors='replace')


    return plugin_meta_extract(src, filename)


# Comment and field extraction logic
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
@renamed_arguments({"fn": "filename"})
def plugin_meta_extract(src="", filename=None, literal=False):
    """
    Finds the first comment block. Splits key:value header
    fields from comment. Turns everything into an dict, with
    some stub fields if absent. Dashes substituted for underscores.

    | Parameters  | | |
    |-------------|---------|---------------------------------|
    | src         | str     | from existing source code       |
    | filename    | str     | set filename attribute          |
    | literls     | bool    | just split comment from doc     |
    | **Returns** | dict    | fields                          |
    """

    # Defaults
    meta = {
        "id": os.path.splitext(os.path.basename(filename or ""))[0],
        "fn": filename,








|
|
>
|





<
|
>


|




>











|











|
>
|
>
>
















|








274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289

290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351

    | src         | str     | From already uncovered script code.             |
    | module      | str     | Lookup per pkgutil, relative to plugin_base     |
    | frame       | int     | Extract comment header of caller (default).     |
    | extra_base  | list    | Additional search directories.                  |
    | max_length  | list    | Maximum size to read from files (6K default).   |
    | **Returns** | dict    | Extracted comment fields, with config: preparsed|

    The result dictionary (`PluginMeta`) has fields accessible as e.g. `meta["title"]`
    or `meta.version`. The documentation block after all fields: is called
    `meta["doc"]`.
    And `meta.config` already parsed as a list (`OptionList`) of dictionaries.
    """

    # Try via pkgutil first,
    # find any plugins.* modules, or main packages
    if module:

        search = plugin_base + kwargs.get("extra_base", [])
        for base, sfx in itertools.product(search, [".py"]):
            try:
                #log.debug(f"mod={base} fn={filename}.py")
                src = get_data(filename=module+sfx, decode=True, file_root=base)
                if src:
                    break
            except (IOError, OSError, FileNotFoundError):
                continue  # plugin_meta_extract() will print a notice later
        filename = module

    # Real filename/path
    elif filename and os.path.exists(filename):
        src = open(filename).read(kwargs.get("max_length", 6144))

    # Else get source directly from caller
    elif not src and not filename:
        module = inspect.getmodule(sys._getframe(frame+1)) # decorator+1
        filename = inspect.getsourcefile(module)
        src = inspect.getcomments(module)

    # Assume it's a filename matching …/base.zip/…/int.py
    elif filename:
        int_fn = ""
        while len(filename) and not os.path.exists(filename): # pylint: disable=len-as-condition
            filename, add = os.path.split(filename)
            int_fn = add + "/" + int_fn
        if len(filename) >= 3 and int_fn and zipfile.is_zipfile(filename):
            src = zipfile.ZipFile(filename, "r").read(int_fn.strip("/"))

    # Extract source comment into meta dict
    if not src:
        src = ""
    if hasattr(src, "decode"):
        try:
            src = src.decode("utf-8", errors='replace')
        except UnicodeDecodeError:
            pass
    return plugin_meta_extract(src, filename)


# Comment and field extraction logic
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
@renamed_arguments({"fn": "filename"})
def plugin_meta_extract(src="", filename=None, literal=False):
    """
    Finds the first comment block. Splits key:value header
    fields from comment. Turns everything into an dict, with
    some stub fields if absent. Dashes substituted for underscores.

    | Parameters  | | |
    |-------------|---------|---------------------------------|
    | src         | str     | from existing source code       |
    | filename    | str     | set filename attribute          |
    | literal     | bool    | just split comment from doc     |
    | **Returns** | dict    | fields                          |
    """

    # Defaults
    meta = {
        "id": os.path.splitext(os.path.basename(filename or ""))[0],
        "fn": filename,
︙︙
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572

    """
    Utility function which collect defaults from plugin meta data to
    a config store. Which in the case of streamtuner2 is really just a
    dictionary `conf{}` and a plugin list in `conf.plugins{}`.

    | Parameters  | | |
    |-------------|---------|--------------------------------------|
    |conf_options | dict 🔁 | storage for amassed options          |
    |conf_plugins | dict 🔁 | enable status based on plugin state/priority: |
    |meta         | dict    | input plugin meta data (invoke once per plugin)|
    |module       | str     | basename of meta: blocks plugin file |
    | **Returns** | None    | -                                    |
    """

    # Option defaults, if not yet defined
    for opt in meta.get("config", []):
        if "name" not in opt or "value" not in opt:
            continue








|
|
|
|








561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578

    """
    Utility function which collect defaults from plugin meta data to
    a config store. Which in the case of streamtuner2 is really just a
    dictionary `conf{}` and a plugin list in `conf.plugins{}`.

    | Parameters  | | |
    |-------------|---------|--------------------------------------|
    | conf_options| dict 🔁 | storage for amassed options          |
    | conf_plugins| dict 🔁 | enable status based on plugin state/priority: |
    | meta        | dict    | input plugin meta data (invoke once per plugin)|
    | module      | str     | basename of meta: blocks plugin file |
    | **Returns** | None    | -                                    |
    """

    # Option defaults, if not yet defined
    for opt in meta.get("config", []):
        if "name" not in opt or "value" not in opt:
            continue
︙︙