Internet radio browser GUI for music/video streams from various directory services.

⌈⌋ ⎇ branch:  streamtuner2


Check-in [492cb2aacf]

Overview
Comment:A bit more PEP8, without undoing all readability.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 492cb2aacf7d42f25559180a64762cdf46a448c4
User & Date: mario on 2015-05-18 22:21:55
Other Links: manifest | tags
Context
2015-05-19
22:37
Fetch all 5 available pages from Xiph directory. Rewrite to use more directed regex extraction mode. Minor help page updates. check-in: 2590478319 user: mario tags: trunk
2015-05-18
22:21
A bit more PEP8, without undoing all readability. check-in: 492cb2aacf user: mario tags: trunk
22:21
Allow Exif-wrapped JPEGs as well (not just JFIF regexp check). Comment on merging row["favourite"] merge via prepare_filters list. check-in: 6d03cb84d3 user: mario tags: trunk
Changes

Modified action.py from [ec6873c4fb] to [005725eacf].

whitespace changes only

Modified pluginconf.py from [d5e7efa673] to [2b0d85b26d].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# encoding: UTF-8
# api: python
# type: handler
# category: io
# title: Plugin configuration
# description: Read meta data, pyz/package contents, module locating
# version: 0.6
# priority: core
# doc: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data
# config: -
#
# Provides plugin lookup and meta data extraction utility functions.
# It's used to abstract module+option management in applications.
# For consolidating internal use and external/tool accessibility.
#
# The key:value format is language-agnostic. It's basically YAML in








|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# encoding: UTF-8
# api: python
# type: handler
# category: io
# title: Plugin configuration
# description: Read meta data, pyz/package contents, module locating
# version: 0.6
# priority: core
# docs: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data
# config: -
#
# Provides plugin lookup and meta data extraction utility functions.
# It's used to abstract module+option management in applications.
# For consolidating internal use and external/tool accessibility.
#
# The key:value format is language-agnostic. It's basically YAML in
72
73
74
75
76
77
78
79
80
81
82
83
84

85

86
87
88
89

90

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
# for example.
#
# Plugin loading thus becomes as simple as __import__("ext.local").
# The attachaed plugin_state config dictionary in most cases can just
# list module basenames, if there's only one set to manage.



import sys
import os
import re
import pkgutil
import inspect

try: from compat2and3 import gzip_decode

except: from gzip import decompress as gzip_decode # Py3 only
import zipfile
import argparse


__all__ = ["get_data", "module_list", "plugin_meta", "dependency", "add_plugin_defaults"]




# Injectables
# ‾‾‾‾‾‾‾‾‾‾‾
log_ERR = lambda *x:None

# File lookup relation for get_data(), should name a top-level package.
module_base = "config"

# Package/module names for module_list() and plugin_meta() lookups.
# All associated paths will be scanned for module/plugin basenames.
plugin_base = ["channels"]



# Resource retrieval
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Fetches file content from install path or from within PYZ
# archive. This is just an alias and convenience wrapper for
# pkgutil.get_data().
# Utilizes the module_base / file_base as top-level reference.
#
def get_data(fn, decode=False, gz=False, file_base=None):
    try:
        bin = pkgutil.get_data(file_base or module_base, fn)
        if gz:
            bin = gzip_decode(bin)
        if decode:
            return bin.decode("utf-8", errors='ignore')
        else:
            return str(bin)
    except:
        pass#log_ERR("get_data() didn't find:", fn, "in", file_base)



# Plugin name lookup
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Search through ./plugins/ (and other configured plugin_base
# names → paths) and get module basenames.
#







<





>
|
>
|



>
|
>
|












<



















|
|







72
73
74
75
76
77
78

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# for example.
#
# Plugin loading thus becomes as simple as __import__("ext.local").
# The attachaed plugin_state config dictionary in most cases can just
# list module basenames, if there's only one set to manage.



import sys
import os
import re
import pkgutil
import inspect
try:
    from compat2and3 import gzip_decode
except:
    from gzip import decompress as gzip_decode  # Py3 only
import zipfile
import argparse

__all__ = [
    "get_data", "module_list", "plugin_meta",
    "dependency", "add_plugin_defaults"
]


# Injectables
# ‾‾‾‾‾‾‾‾‾‾‾
log_ERR = lambda *x:None

# File lookup relation for get_data(), should name a top-level package.
module_base = "config"

# Package/module names for module_list() and plugin_meta() lookups.
# All associated paths will be scanned for module/plugin basenames.
plugin_base = ["channels"]



# Resource retrieval
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Fetches file content from install path or from within PYZ
# archive. This is just an alias and convenience wrapper for
# pkgutil.get_data().
# Utilizes the module_base / file_base as top-level reference.
#
def get_data(fn, decode=False, gz=False, file_base=None):
    try:
        bin = pkgutil.get_data(file_base or module_base, fn)
        if gz:
            bin = gzip_decode(bin)
        if decode:
            return bin.decode("utf-8", errors='ignore')
        else:
            return str(bin)
    except:
        # log_ERR("get_data() didn't find:", fn, "in", file_base)
        pass


# Plugin name lookup
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Search through ./plugins/ (and other configured plugin_base
# names → paths) and get module basenames.
#
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
            paths.append(mp)

    # Should list plugins within zips as well as local paths
    ls = pkgutil.iter_modules(paths + extra_paths)
    return [name for loader,name,ispkg in ls]



# Plugin => meta dict
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# This is a trivial wrapper to assemble a complete dictionary
# of available/installed plugins. It associates each plugin name
# with a its meta{} fields.
#
def all_plugin_meta():
    return {
        name: plugin_meta(module=name) for name in module_list()
    }



# Plugin meta data extraction
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Can fetch infos from different sources:
#
#   fn=      read literal files, or .pyz contents







<










<







143
144
145
146
147
148
149

150
151
152
153
154
155
156
157
158
159

160
161
162
163
164
165
166
            paths.append(mp)

    # Should list plugins within zips as well as local paths
    ls = pkgutil.iter_modules(paths + extra_paths)
    return [name for loader,name,ispkg in ls]



# Plugin => meta dict
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# This is a trivial wrapper to assemble a complete dictionary
# of available/installed plugins. It associates each plugin name
# with a its meta{} fields.
#
def all_plugin_meta():
    return {
        name: plugin_meta(module=name) for name in module_list()
    }



# Plugin meta data extraction
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Can fetch infos from different sources:
#
#   fn=      read literal files, or .pyz contents
178
179
180
181
182
183
184
185

186
187
188
189
190
191
192
    # Try via pkgutil first,
    # find any plugins.* modules, or main packages
    if module:
       fn = module
       for base in plugin_base + extra_base:
           try:
               src = get_data(fn=fn+".py", decode=True, file_base=base)
               if src: break

           except:
               continue  # plugin_meta_extract() will print a notice later

    # Real filename/path
    elif fn and os.path.exists(fn):
        src = open(fn).read(4096)








|
>







178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    # Try via pkgutil first,
    # find any plugins.* modules, or main packages
    if module:
       fn = module
       for base in plugin_base + extra_base:
           try:
               src = get_data(fn=fn+".py", decode=True, file_base=base)
                if src:
                    break
           except:
               continue  # plugin_meta_extract() will print a notice later

    # Real filename/path
    elif fn and os.path.exists(fn):
        src = open(fn).read(4096)

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

    # Extract source comment into meta dict
    if not src:
        src = ""
    if not isinstance(src, str):
        src = src.decode("utf-8", errors='replace')
    return plugin_meta_extract(src, fn)



# Comment and field extraction logic
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Finds the first comment block. Splits key:value header
# fields from comment. Turns everything into an dict, with
# some stub fields if absent.







<







208
209
210
211
212
213
214

215
216
217
218
219
220
221

    # Extract source comment into meta dict
    if not src:
        src = ""
    if not isinstance(src, str):
        src = src.decode("utf-8", errors='replace')
    return plugin_meta_extract(src, fn)



# Comment and field extraction logic
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Finds the first comment block. Splits key:value header
# fields from comment. Turns everything into an dict, with
# some stub fields if absent.
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
    for field in rx.keyval.findall(src):
        meta[field[0]] = field[1].strip()
    meta["config"] = plugin_meta_config(meta.get("config") or "")

    return meta



# Unpack config: structures
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Further breaks up the meta['config'] descriptor.
# Creates an array from JSON/YAML option lists.
#
# config:
#   { name: 'var1', type: text, value: "default, ..." }







<







254
255
256
257
258
259
260

261
262
263
264
265
266
267
    for field in rx.keyval.findall(src):
        meta[field[0]] = field[1].strip()
    meta["config"] = plugin_meta_config(meta.get("config") or "")

    return meta



# Unpack config: structures
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Further breaks up the meta['config'] descriptor.
# Creates an array from JSON/YAML option lists.
#
# config:
#   { name: 'var1', type: text, value: "default, ..." }
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
        }
        for field in rx.options.findall(entry):
            opt[field[0]] = (field[1] or field[2] or field[3] or "").strip()
        config.append(opt)
    return config



# Comment extraction regexps
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Pretty crude comment splitting approach. But works
# well enough already. Technically a YAML parser would
# do better; but is likely overkill.
#
class rx:







<







280
281
282
283
284
285
286

287
288
289
290
291
292
293
        }
        for field in rx.options.findall(entry):
            opt[field[0]] = (field[1] or field[2] or field[3] or "").strip()
        config.append(opt)
    return config



# Comment extraction regexps
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Pretty crude comment splitting approach. But works
# well enough already. Technically a YAML parser would
# do better; but is likely overkill.
#
class rx:
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
        ["':$]?   (\w*)  ["']?                 # key or ":key" or '$key'
        \s* [:=] \s*                           # "=" or ":"
     (?:  "  ([^"]*)  " 
       |  '  ([^']*)  '                        #  "quoted" or 'singl' values
       |     ([^,]*)                           #  or unquoted literals
     )
    """, re.X)



# ArgumentParser options conversion
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# As variation of in-application config: options, this method converts
# cmdline argument specifiers.
#







<







303
304
305
306
307
308
309

310
311
312
313
314
315
316
        ["':$]?   (\w*)  ["']?                 # key or ":key" or '$key'
        \s* [:=] \s*                           # "=" or ":"
     (?:  "  ([^"]*)  " 
       |  '  ([^']*)  '                        #  "quoted" or 'singl' values
       |     ([^,]*)                           #  or unquoted literals
     )
    """, re.X)



# ArgumentParser options conversion
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# As variation of in-application config: options, this method converts
# cmdline argument specifiers.
#
352
353
354
355
356
357
358
359

360
361
362
363
364
365


366
367
368

369
370
371

372
373
374
375
376
377
378
379
380
381
    typing = re.findall("bool|str|\[\]|const|false|true", opt["type"])
    naming = re.findall("\[\]", opt["name"])
    name   = re.findall("(?<!-)\\b\\w+", opt["name"])
    nargs  = re.findall("\\b\d+\\b|[\?\*\+]", opt["type"]) or [None]
    is_arr = "[]" in (naming + typing) and nargs == [None]
    is_bool= "bool" in typing
    false_b = "false" in typing or opt["value"] in ("0", "false")
    #print("\nname=", name, "is_arr=", is_arr, "is_bool=", is_bool, "bool_d=", false_b, "naming=", naming, "typing=", typing)


    # Populate partially - ArgumentParser has aversions to many parameter combinations
    kwargs = dict(
        args     = args,
        dest     = name[0] if not name[0] in args else None,
        action   = is_arr and "append"  or  is_bool and false_b and "store_false"  or  is_bool and "store_true"  or  "store",


        nargs    = nargs[0],
        default  = opt.get("default") or opt["value"],
        type     = None if is_bool  else  ("int" in typing and int  or  "bool" in typing and bool  or  str),

        choices  = opt["select"].split("|") if "select" in opt else None,
        required = "required" in opt or None,
        help     = opt["description"] if not "hidden" in opt else argparse.SUPPRESS

    )
    return {k:w for k,w in kwargs.items() if w is not None}



# Minimal depends: probing
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Now this definitely requires customization. Each plugin can carry
# a list of (soft-) dependency names.
#







|
>

|



|
>
>


|
>


|
>


<







349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375

376
377
378
379
380
381
382
    typing = re.findall("bool|str|\[\]|const|false|true", opt["type"])
    naming = re.findall("\[\]", opt["name"])
    name   = re.findall("(?<!-)\\b\\w+", opt["name"])
    nargs  = re.findall("\\b\d+\\b|[\?\*\+]", opt["type"]) or [None]
    is_arr = "[]" in (naming + typing) and nargs == [None]
    is_bool= "bool" in typing
    false_b = "false" in typing or opt["value"] in ("0", "false")
    # print("\nname=", name, "is_arr=", is_arr, "is_bool=", is_bool,
    # "bool_d=", false_b, "naming=", naming, "typing=", typing)

    # Populate combination as far as ArgumentParser permits
    kwargs = dict(
        args     = args,
        dest     = name[0] if not name[0] in args else None,
        action   = is_arr and "append"
                   or  is_bool and false_b and "store_false"
                   or  is_bool and "store_true"  or  "store",
        nargs    = nargs[0],
        default  = opt.get("default") or opt["value"],
        type     = None if is_bool  else  ("int" in typing and int
                   or  "bool" in typing and bool  or  str),
        choices  = opt["select"].split("|") if "select" in opt else None,
        required = "required" in opt or None,
        help     = opt["description"] if not "hidden" in opt
                   else argparse.SUPPRESS
    )
    return {k:w for k,w in kwargs.items() if w is not None}



# Minimal depends: probing
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Now this definitely requires customization. Each plugin can carry
# a list of (soft-) dependency names.
#
408
409
410
411
412
413
414
415

416
417
418

419

420

421



422
423
424
425
426
427
428
429
430

431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
    def depends(self, plugin):
        if plugin.get("depends"):
            d = self.deps(plugin["depends"])
            if not self.cmp(d, self.have):
                return False
        return True

    # basic list pre-filtering (skip __init__, filter by api:, exclude installed & same-version plugins)

    def valid(self, newpl):
        id = newpl.get("$name")
        have_ver = self.have.get(id, {}).get("version", "0")

        return id.find("__") < 0                        \

           and newpl.get("api") == "streamtuner2"       \

           and have_ver < newpl.get("version", "0.0")




    # Split trivial "pkg, mod >= 1, uikit < 4.0" list
    def deps(self, dep_str):
        d = []
        for dep in re.split(r"\s*[,;]+\s*", dep_str):
            # skip deb:pkg-name, rpm:name, bin:name etc.
            if not len(dep) or dep.find(":") >= 0:
                continue
            # find comparison and version num

            m = re.search(r"([\w.-]+)\s*([>=<!~]+)\s*([\d.]+([-~.]\w+)*)", dep + " >= 0")
            if m and m.group(2):
                d.append([m.group(i) for i in (1,2,3)])
        return d
    
    # Do actual comparison
    def cmp(self, d, have):
        r = True
        for name, op, ver in d:
            # skip unknown plugins, might be python module references ("depends: re, json")
            if not have.get(name, {}).get("version"):
                continue
            curr = have[name]["version"]
            tbl = {
               ">=": curr >= ver,
               "<=": curr <= ver,
               "==": curr == ver,
               ">":  curr > ver,
               "<":  curr < ver,
               "!=": curr != ver,
            }
            r &= tbl.get(op, True)
        return r



# Add plugin defaults to conf.* store
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Utility function which applies plugin meta data to a config
# store. Which in the case of streamtuner2 is really just a
# dictionary `conf{}` and a plugin list in `conf.plugins{}`.







|
>

<

>
|
>
|
>
|
>
>
>









>
|








|













<







409
410
411
412
413
414
415
416
417
418

419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461

462
463
464
465
466
467
468
    def depends(self, plugin):
        if plugin.get("depends"):
            d = self.deps(plugin["depends"])
            if not self.cmp(d, self.have):
                return False
        return True

    # basic list pre-filtering (skip __init__, filter by api:,
    # exclude installed & same-version plugins)
    def valid(self, newpl):

        have_ver = self.have.get(id, {}).get("version", "0")
        id = newpl.get("$name", "__invalid")
        if id.find("__") == 0:
            pass
        elif newpl.get("api") != "streamtuner2":
            pass
        elif have_ver >= newpl.get("version", "0.0"):
            pass
        else:
            return True

    # Split trivial "pkg, mod >= 1, uikit < 4.0" list
    def deps(self, dep_str):
        d = []
        for dep in re.split(r"\s*[,;]+\s*", dep_str):
            # skip deb:pkg-name, rpm:name, bin:name etc.
            if not len(dep) or dep.find(":") >= 0:
                continue
            # find comparison and version num
            dep += " >= 0"
            m = re.search(r"([\w.-]+)\s*([>=<!~]+)\s*([\d.]+([-~.]\w+)*)", dep)
            if m and m.group(2):
                d.append([m.group(i) for i in (1,2,3)])
        return d
    
    # Do actual comparison
    def cmp(self, d, have):
        r = True
        for name, op, ver in d:
            # skip unknown plugins, might be python module references
            if not have.get(name, {}).get("version"):
                continue
            curr = have[name]["version"]
            tbl = {
               ">=": curr >= ver,
               "<=": curr <= ver,
               "==": curr == ver,
               ">":  curr > ver,
               "<":  curr < ver,
               "!=": curr != ver,
            }
            r &= tbl.get(op, True)
        return r



# Add plugin defaults to conf.* store
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Utility function which applies plugin meta data to a config
# store. Which in the case of streamtuner2 is really just a
# dictionary `conf{}` and a plugin list in `conf.plugins{}`.
477
478
479
480
481
482
483
484

485
486

                     val = int(opt["value"])
                 else:
                     val = str(opt["value"])
                 conf_options[opt["name"]] = val

    # Initial plugin activation status
    if module and module not in conf_plugins:
         conf_plugins[module] = meta.get("priority") in ("core", "builtin", "always", "default", "standard")











|
>
|

>
484
485
486
487
488
489
490
491
492
493
494
495
                     val = int(opt["value"])
                 else:
                     val = str(opt["value"])
                 conf_options[opt["name"]] = val

    # Initial plugin activation status
    if module and module not in conf_plugins:
        conf_plugins[module] = meta.get("priority") in (
            "core", "builtin", "always", "default", "standard"
        )