Index: channels/pluginmanager2.py ================================================================== --- channels/pluginmanager2.py +++ channels/pluginmanager2.py @@ -131,11 +131,13 @@ # Clean up placeholders in vbox _ = [self.vbox.remove(c) for c in self.vbox.get_children()[3:]] # Attach available downloads after checking dependencies # e.g. newpl["depends"] = "streamtuner2 < 2.2.0, config >= 2.5" - dep = pluginconf.dependency() + import pluginconf.depends + dep = pluginconf.depends.DependencyValidation() + dep.api = ["python", "streamtuner2"] for newpl in meta: if dep.valid(newpl, log.DEBUG_VALIDITY) and dep.depends(newpl, log.DEBUG_DEPENDS): self.add_plugin(newpl) else: log.DEBUG("plugin fails dependencies:", newpl) Index: config.py ================================================================== --- config.py +++ config.py @@ -10,11 +10,11 @@ # { arg: --nt, type: boolean, name: nothreads, description: Disable threading/gtk_idle UI. } # { arg: -D, type: boolean, name: debug, description: Enable debug messages on console } # { arg: action, type: str *, name: action[], description: CLI interface commands. } # { arg: -x, type: boolean, name: exit, hidden: 1 } # { arg: -V, type: boolean, name: version, description: Print version. } -# { arg: -w, type: boolean, name: pydoc, hiden: 1 } +# { arg: -w, type: boolean, name: pydoc, hidden: 1 } # version: 2.8 # priority: core # depends: pluginconf >= 0.1, os, json, re, zlib, pkgutil # # Ties together the global conf.* object. It's typically used DELETED pluginconf.py Index: pluginconf.py ================================================================== --- pluginconf.py +++ pluginconf.py @@ -1,593 +0,0 @@ -# encoding: utf-8 -# api: python -# type: extract -# category: config -# title: Plugin configuration -# description: Read meta data, pyz/package contents, module locating -# version: 0.7.5 -# state: stable -# classifiers: documentation -# license: PD -# priority: core -# docs: https://fossil.include-once.org/pluginspec/ -# url: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data -# config: - -# -# Provides plugin lookup and meta data extraction utility functions. -# It's used to abstract module+option management in applications. -# For consolidating internal use and external/tool accessibility. -# -# The key:value format is language-agnostic. It's basically YAML in -# a topmost script comment. For Python only # hash comments though. -# Uses common field names, a documentation block, and an obvious -# `config: { .. }` spec for options and defaults. -# -# It neither imposes a specific module/plugin API, nor config storage, -# and doesn't fixate module loading. It's really just meant to look -# up meta infos. -# This approach avoids in-code values/inspection, externalized meta -# descriptors, and any hodgepodge or premature module loading just to -# uncover module description fields. -# -# plugin_meta() -# ‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Is the primary function to extract a meta dictionary from files. -# It either reads from a given module= name, a literal fn=, or just -# src= code, and as fallback inspects the last stack frame= else. -# -# module_list() -# ‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Returns basenames of available/installed plugins. It uses the -# plugin_base=[] list for module relation. Which needs to be set up -# beforehand, or injected. -# -# add_plugin_defaults() -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Populates a config_options{} and plugin_states{} list. Used for -# initial setup, or when adding new plugins, etc. Both dicts might -# also be ConfigParser stores, or implement magic __set__ handling -# to act on state changes. -# -# get_data() -# ‾‾‾‾‾‾‾‾‾‾ -# Is mostly an alias for pkgutil.get_data(). It abstracts the main -# base path, allows PYZ usage, and adds some convenience flags.‾ -# It's somewhat off-scope for plugin management, but used internally. -# -# argparse_map() -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Converts a list of config: options with arg: attribute for use as -# argparser parameters. -# -# dependency().depends()/.valid() -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Probes a new plugins` depends: list against installed base modules. -# Utilizes each version: fields and allows for virtual modules, or -# alternatives and honors alias: names. -# -# -# Generally this scheme concerns itself more with plugin basenames. -# That is: module scripts in a package like `ext.plg1` and `ext.plg2`. -# It can be initialized by injecting the plugin-package basename into -# plugin_base = []. The associated paths will be used for module -# lookup via pkgutil.iter_modules(). -# -# And a central module can be extended with new lookup locations best -# by attaching new locations itself via module.__path__ + ["./local"] -# for example. -# -# Plugin loading thus becomes as simple as __import__("ext.local"). -# The attached plugin_state config dictionary in most cases can just -# list module basenames, if there's only one set to manage. - - -import sys -import os -import re -import pkgutil -import inspect -try: - from distutils.spawn import find_executable -except: - try: - from compat2and3 import find_executable - except: - def find_executable(str): - pass -try: - from gzip import decompress as gzip_decode # Py3 only -except: - try: - from compat2and3 import gzip_decode # st2 stub - except: - def gzip_decode(bytes): - import zlib - return zlib.decompress(bytes, 16 + zlib.MAX_WBITS) # not fully compatible -import zipfile -import argparse - -__all__ = [ - "get_data", "module_list", "plugin_meta", - "dependency", "add_plugin_defaults" -] - - -# Injectables -# ‾‾‾‾‾‾‾‾‾‾‾ -log_ERR = lambda *x: None - -# File lookup relation for get_data(), should name a top-level package. -module_base = "config" # equivalent PluginBase(package=…) - -# Package/module names for module_list() and plugin_meta() lookups. -# All associated paths will be scanned for module/plugin basenames. -plugin_base = ["channels"] # equivalent to `searchpath` in PluginBase - - -# Resource retrieval -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Fetches file content from install path or from within PYZ -# archive. This is just an alias and convenience wrapper for -# pkgutil.get_data(). -# Utilizes the module_base / file_base as top-level reference. -# -def get_data(fn, decode=False, gz=False, file_base=None): - try: - bin = pkgutil.get_data(file_base or module_base, fn) - if gz: - bin = gzip_decode(bin) - if decode: - return bin.decode("utf-8", errors='ignore') - else: - return str(bin) - except: - # log_ERR("get_data() didn't find:", fn, "in", file_base) - pass - - -# Plugin name lookup -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Search through ./plugins/ (and other configured plugin_base -# names → paths) and get module basenames. -# -def module_list(extra_paths=[]): - - # Convert plugin_base package names into paths for iter_modules - paths = [] - for mp in plugin_base: - if sys.modules.get(mp): - paths += sys.modules[mp].__path__ - elif os.path.exists(mp): - paths.append(mp) - - # Should list plugins within zips as well as local paths - ls = pkgutil.iter_modules(paths + extra_paths) - return [name for loader, name, ispkg in ls] - - -# Plugin => meta dict -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# This is a trivial wrapper to assemble a complete dictionary -# of available/installed plugins. It associates each plugin name -# with a its meta{} fields. -# -def all_plugin_meta(): - return { - name: plugin_meta(module=name) for name in module_list() - } - - -# Plugin meta data extraction -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Can fetch infos from different sources: -# -# fn= read literal files, or .pyz contents -# -# src= from already uncovered script code -# -# module= lookup per pkgutil, from plugin bases -# or top-level modules -# -# frame= extract comment header of caller -# (default) -# -def plugin_meta(fn=None, src=None, module=None, frame=1, extra_base=[]): - - # Try via pkgutil first, - # find any plugins.* modules, or main packages - if module: - fn = module - for base in plugin_base + extra_base: - try: - src = get_data(fn=fn+".py", decode=True, file_base=base) - if src: - break - except: - continue # plugin_meta_extract() will print a notice later - - # Real filename/path - elif fn and os.path.exists(fn): - src = open(fn).read(4096) - - # Else get source directly from caller - elif not src and not fn: - module = inspect.getmodule(sys._getframe(frame)) - fn = inspect.getsourcefile(module) - src = inspect.getcomments(module) - - # Assume it's a filename within a zip - elif fn: - intfn = "" - while fn and len(fn) and not os.path.exists(fn): - fn, add = os.path.split(fn) - intfn = add + "/" + intfn - if len(fn) >= 3 and intfn and zipfile.is_zipfile(fn): - src = zipfile.ZipFile(fn, "r").read(intfn.strip("/")) - - # Extract source comment into meta dict - if not src: - src = "" - if not isinstance(src, str): - src = src.decode("utf-8", errors='replace') - return plugin_meta_extract(src, fn) - - -# Comment and field extraction logic -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Finds the first comment block. Splits key:value header -# fields from comment. Turns everything into an dict, with -# some stub fields if absent. -# -def plugin_meta_extract(src="", fn=None, literal=False): - - # Defaults - meta = { - "id": os.path.splitext(os.path.basename(fn or ""))[0], - "fn": fn, - "api": "python", - "type": "module", - "category": None, - "priority": None, - "version": "0", - "title": fn, - "description": "no description", - "config": [], - "doc": "" - } - - # Extract coherent comment block - src = src.replace("\r", "") - if not literal: - src = rx.comment.search(src) - if not src: - log_ERR("Couldn't read source meta information:", fn) - return meta - src = src.group(0) - src = rx.hash.sub("", src).strip() - - # Split comment block - if src.find("\n\n") > 0: - src, meta["doc"] = src.split("\n\n", 1) - - # Turn key:value lines into dictionary - for field in rx.keyval.findall(src): - meta[field[0].replace("-", "_")] = field[1].strip() - meta["config"] = plugin_meta_config(meta.get("config") or "") - - return meta - - -# Unpack config: structures -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Further breaks up the meta['config'] descriptor. -# Creates an array from JSON/YAML option lists. -# -# config: -# { name: 'var1', type: text, value: "default, ..." } -# { name=option2, type=boolean, $value=1, etc. } -# -# Stubs out name, value, type, description if absent. -# -def plugin_meta_config(str): - config = [] - for entry in rx.config.findall(str): - entry = entry[0] or entry[1] - opt = { - "type": None, - "name": None, - "description": "", - "value": None - } - for field in rx.options.findall(entry): - opt[field[0]] = (field[1] or field[2] or field[3] or "").strip() - # normalize type - opt["type"] = config_opt_type_map.get(opt["type"], opt["type"] or "str") - # preparse select: - if opt.get("select"): - opt["select"] = config_opt_parse_select(opt.get("select", "")) - config.append(opt) - return config - -# split up `select: 1=on|2=more|3=title` or `select: foo|bar|lists` -def config_opt_parse_select(s): - if re.search("([=:])", s): - return dict(rx.select_dict.findall(s)) - else: - return dict([(v, v) for v in rx.select_list.findall(s)]) - -# normalize type:names to `str`, `text`, `bool`, `int`, `select`, `dict` -config_opt_type_map = dict( - longstr="text", string="str", boolean="bool", checkbox="bool", integer="int", number="int", - choice="select", options="select", table="dict", array="dict" -) - - -# Comment extraction regexps -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Pretty crude comment splitting approach. But works -# well enough already. Technically a YAML parser would -# do better; but is likely overkill. -# -class rx: - comment = re.compile(r"""(^ {0,4}#.*\n)+""", re.M) - hash = re.compile(r"""(^ {0,4}# {0,3}\r*)""", re.M) - keyval = re.compile(r""" - ^([\w-]+):(.*$(?:\n(?![\w-]+:).+$)*) # plain key:value lines - """, re.M | re.X) - config = re.compile(r""" - \{ (.+?) \} | \< (.+?) \> # JSOL/YAML scheme {...} dicts - """, re.X) - options = re.compile(r""" - ["':$]? (\w*) ["']? # key or ":key" or '$key' - \s* [:=] \s* # "=" or ":" - (?: " ([^"]*) " - | ' ([^']*) ' # "quoted" or 'singl' values - | ([^,]*) # or unquoted literals - ) - """, re.X) - select_dict = re.compile(r"(\w+)\s*[=:>]+\s*([^=,|:]+)") - select_list = re.compile(r"\s*([^,|;]+)\s*") - - - -# ArgumentParser options conversion -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# As variation of in-application config: options, this method converts -# cmdline argument specifiers. -# -# config: -# { arg: -i, name: input[], type: str, description: input files } -# -# Which allows to collect argumentparser options from different plugins. -# The only difference to normal config entries is the `arg:` attribute. -# -# · It adds array arguments with a [] name suffix, or a `*` type suffix. -# Else even a `?` or `+` and numeric counts after the type flag. -# -# · Understands the types `str`, `int` and `bool`. -# -# · Entries may carry a `hidden: 1` or `required: 1` attribute. -# -# · And `help:` is an alias to `description:` -# And `default:` an alias for `value:` -# -# · While `type: select` utilizes the `select: a|b|c` format as usual. -# -# ArgParsers const=, metavar= flag, or type=file are not aliased here. -# -# Basically returns a dictionary that can be fed per **kwargs directly -# to an ArgumentParsers add_argument(). Iterate over each plugins -# meta['config'][] options to convert them. -# -def argparse_map(opt): - if not ("arg" in opt and opt["name"] and opt["type"]): - return {} - - # Extract --flag names - args = opt["arg"].split() + re.findall(r"-+\w+", opt["name"]) - - # Prepare mapping options - typing = re.findall(r"bool|str|\[\]|const|false|true", opt["type"]) - naming = re.findall(r"\[\]", opt["name"]) - name = re.findall(r"(?= 2.0, bin:wkhtmltoimage, python < 3.5 -# -# Here only in-application modules are honored, system references -# ignored. Unknown plugin names are also skipped. A real install -# helper might want to auto-tick them on, etc. This example is just -# meant for probing downloadable plugins. -# -# The .valid() helper only asserts the api: string, or skips existing -# modules, and if they're more recent. -# While .depends() compares minimum versions against existing modules. -# -# In practice there's little need for full-blown dependency resolving -# for application-level modules. -# -class dependency(object): - - # prepare list of known plugins and versions - def __init__(self, add={}, core=["st2", "uikit", "config", "action"]): - self.have = { - "python": { "version": sys.version } - } - # inject virtual modules - for name, meta in add.items(): - if isinstance(meta, bool): meta = 1 if meta else -1 - if isinstance(meta, tuple): meta = ".".join(str(n) for n in meta) - if isinstance(meta, (int, float, str)): meta = {"version": str(meta)} - self.have[name] = meta - # read plugins/* - self.have.update(all_plugin_meta()) - # add core modules - for name in core: - self.have[name] = plugin_meta(module=name, extra_base=["config"]) - # aliases - for name, meta in self.have.copy().items(): - if meta.get("alias"): - for alias in re.split(r"\s*[,;]\s*", meta["alias"]): - self.have[alias] = self.have[name] - - # basic plugin pre-screening (skip __init__, filter by api:, - # exclude installed & same-version plugins) - def valid(self, newpl, _log=lambda *x:0): - id = newpl.get("$name", "__invalid") - have_ver = self.have.get(id, {}).get("version", "0") - if id.find("__") == 0: - _log("wrong id") - pass - elif newpl.get("api") not in ("python", "streamtuner2"): - _log("wrong api") - pass - elif set((newpl.get("status"), newpl.get("priority"))).intersection(set(("obsolete", "broken"))): - _log("wrong status") - pass - elif have_ver >= newpl.get("version", "0.0"): - _log("newer version already installed") - pass - else: - return True - - # Verify depends: and breaks: against existing plugins/modules - def depends(self, plugin, _log=lambda *x:0): - r = True - if plugin.get("depends"): - r &= self.and_or(self.split(plugin["depends"]), self.have) - if plugin.get("breaks"): - r &= self.neither(self.split(plugin["breaks"]), self.have) - _log(r) - return r - - # Split trivial "pkg | alt, mod >= 1, uikit < 4.0" string into nested list [[dep],[alt,alt],[dep]] - def split(self, dep_str): - dep_cmp = [] - for alt_str in re.split(r"\s*[,;]+\s*", dep_str): - alt_cmp = [] - # split alternatives | - for part in re.split(r"\s*\|+\s*", alt_str): - # skip deb:pkg-name, rpm:name, bin:name etc. - if not len(part): - continue - if part.find(":") >= 0: - self.have[part] = { "version": self.module_test(*part.split(":")) } - # find comparison and version num - part += " >= 0" - m = re.search(r"([\w.:-]+)\s*\(?\s*([>==": curr >= ver, - "<=": curr <= ver, - "==": curr == ver, - ">": curr > ver, - "<": curr < ver, - "!=": curr != ver, - } - r = tbl.get(op, True) - #print "log.VERSION_COMPARE: ", name, " → (", curr, op, ver, ") == ", r - return r - - # Compare nested structure of [[dep],[alt,alt]] - def and_or(self, deps, have, r = True): - #print deps - return not False in [True in [self.cmp(d, have) for d in alternatives] for alternatives in deps] - - # Breaks/Conflicts: check [[or],[or]] - def neither(self, deps, have): - return not True in [self.cmp(d, have, absent=None) for cnd in deps for d in cnd] - - # Resolves/injects complex "bin:name" or "python:name" dependency URNs - def module_test(self, type, name): - return "1" # disabled for now - if "_" + type in dir(self): - return "1" if bool(getattr(self, "_" + type)(name)) else "-1" - # `bin:name` lookup - def _bin(self, name): - return find_executable(name) - # `python:module` test - def _python(self, name): - return __import__("imp").find_module(name) is not None - - -# Add plugin defaults to conf.* store -# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ -# Utility function which applies plugin meta data to a config -# store. Which in the case of streamtuner2 is really just a -# dictionary `conf{}` and a plugin list in `conf.plugins{}`. -# -# Adds each default option value: to conf_options{}. And sets -# first plugin state (enabled/disabled) in conf_plugins{} list, -# depending on priority: classifier. -# -def add_plugin_defaults(conf_options, conf_plugins, meta={}, module=""): - - # Option defaults, if not yet defined - for opt in meta.get("config", []): - if "name" in opt and "value" in opt: - _value = opt.get("value", "") - _name = opt.get("name") - _type = opt.get("type") - if _name not in conf_options: - # typemap - if _type == "bool": - val = _value.lower() in ("1", "true", "yes", "on") - elif _type == "int": - val = int(_value) - elif _type in ("table", "list"): - val = [ re.split(r"\s*[,;]\s*", s.strip()) for s in re.split(r"\s*[|]\s*", _value) ] - elif _type == "dict": - val = dict([ re.split(r"\s*(?:=>+|==*|-+>|:=+)\s*", s.strip(), 1) for s in re.split(r"\s*[|;,]\s*", _value) ]) - else: - val = str(_value) - conf_options[_name] = val - - # Initial plugin activation status - if module and module not in conf_plugins: - conf_plugins[module] = meta.get("priority") in ( - "core", "builtin", "always", "default", "standard" - ) - - ADDED pluginconf/depends.py Index: pluginconf/depends.py ================================================================== --- pluginconf/depends.py +++ pluginconf/depends.py @@ -0,0 +1,190 @@ +# encoding: utf-8 +# api: pluginconf +##type: class +# category: config +# title: Dependency verification +# description: Check depends: lines +# depends: pluginconf >= 0.7 +# version: 0.5 +# state: beta +# license: PD +# priority: optional +# +# This is a rather basic depends: checker, mostly for local and +# installable modules. It's largely built around streamtuner2 +# requirements, and should be customized. +# +# DependencyValidation().depends()/.valid() +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +# Probes a new plugins` depends: list against installed base modules. +# Utilizes each version: fields and allows for virtual modules, or +# alternatives and honors alias: names. +# + + +import pluginconf +import re +try: + from distutils.spawn import find_executable +except ImportError: + try: + from compat2and3 import find_executable + except ImportError: + def find_executable(name): + pass +import zipfile +import logging + + +# Minimal depends: probing +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +class DependencyValidation(object): + """ + Now this definitely requires customization. Each plugin can carry + a list of (soft-) dependency names. + + # depends: config, appcore >= 2.0, bin:wkhtmltoimage, python < 3.5 + + Here only in-application modules are honored, system references + ignored. Unknown plugin names are also skipped. A real install + helper might want to auto-tick them on, etc. This example is just + meant for probing downloadable plugins. + + The .valid() helper only asserts the api: string, or skips existing + modules, and if they're more recent. + While .depends() compares minimum versions against existing modules. + + In practice there's little need for full-blown dependency resolving + for application-level modules. + """ + + """ supported APIs """ + api = ["python", "streamtuner2"] + + """ debugging """ + log = logging.getLogger("pluginconf.dependency") + + # prepare list of known plugins and versions + def __init__(self, add={}, core=["st2", "uikit", "config", "action"]): + self.have = { + "python": {"version": sys.version} + } + + # inject virtual modules + for name, meta in add.items(): + if isinstance(meta, bool): + meta = 1 if meta else -1 + if isinstance(meta, tuple): + meta = ".".join(str(n) for n in meta) + if isinstance(meta, (int, float, str)): + meta = {"version": str(meta)} + self.have[name] = meta + + # read plugins/* + self.have.update(all_plugin_meta()) + + # add core modules + for name in core: + self.have[name] = plugin_meta(module=name, extra_base=["config"]) + + # aliases + for name, meta in self.have.copy().items(): + if meta.get("alias"): + for alias in re.split(r"\s*[,;]\s*", meta["alias"]): + self.have[alias] = self.have[name] + + # basic plugin pre-screening (skip __init__, filter by api:, + # exclude installed & same-version plugins) + def valid(self, new_plugin): + id = new_plugin.get("$name", "__invalid") + have_ver = self.have.get(id, {}).get("version", "0") + if id.find("__") == 0: + self.log.debug("wrong/no id") + elif new_plugin.get("api") not in self.api: + self.log.debug("not in allowed APIs") + elif {new_plugin.get("status"), new_plugin.get("priority")} & {"obsolete", "broken"}: + self.log.debug("wrong status (obsolete/broken)") + elif have_ver >= new_plugin.get("version", "0.0"): + self.log.debug("newer version already installed") + else: + return True + + # Verify depends: and breaks: against existing plugins/modules + def depends(self, plugin): + result = True + if plugin.get("depends"): + result &= self.and_or(self.split(plugin["depends"]), self.have) + if plugin.get("breaks"): + result &= self.neither(self.split(plugin["breaks"]), self.have) + self.log.debug("plugin '%s' matching requirements: %i", plugin["id"], result) + return result + + # Split trivial "pkg | alt, mod>=1, uikit<4.0" string into nested list [[dep],[alt,alt],[dep]] + def split(self, dep_str): + dep_cmp = [] + for alt_str in re.split(r"\s*[,;]+\s*", dep_str): + alt_cmp = [] + # split alternatives | + for part in re.split(r"\s*\|+\s*", alt_str): + # skip deb:pkg-name, rpm:name, bin:name etc. + if not len(part): + continue + if part.find(":") >= 0: + self.have[part] = {"version": self.module_test(*part.split(":"))} + # find comparison and version num + part += " >= 0" + m = re.search(r"([\w.:-]+)\s*\(?\s*([>==": curr >= ver, + "<=": curr <= ver, + "==": curr == ver, + ">": curr > ver, + "<": curr < ver, + "!=": curr != ver, + } + r = tbl.get(op, True) + #print "log.VERSION_COMPARE: ", name, " → (", curr, op, ver, ") == ", r + return r + + # Compare nested structure of [[dep],[alt,alt]] + def and_or(self, deps, have, r=True): + #print deps + return not False in [ + True in [self.cmp(d, have) for d in alternatives] for alternatives in deps + ] + + # Breaks/Conflicts: check [[or],[or]] + def neither(self, deps, have): + return not True in [ + self.cmp(d, have, absent=None) for cnd in deps for d in cnd + ] + + # Resolves/injects complex "bin:name" or "python:name" dependency URNs + def module_test(self, type, name): + return "1" # disabled for now + if "_" + type in dir(self): + return "1" if bool(getattr(self, "_" + type)(name)) else "-1" + + # `bin:name` lookup + def _bin(self, name): + return find_executable(name) + + # `python:module` test + def _python(self, name): + return __import__("imp").find_module(name) is not None + ADDED pluginconf/pluginconf.py Index: pluginconf/pluginconf.py ================================================================== --- pluginconf/pluginconf.py +++ pluginconf/pluginconf.py @@ -0,0 +1,491 @@ +# encoding: utf-8 +# api: python +##type: extract +# category: config +# title: Plugin configuration +# description: Read meta data, pyz/package contents, module locating +# version: 0.7.7 +# state: stable +# classifiers: documentation +# license: PD +# priority: core +# docs: https://fossil.include-once.org/pluginspec/ +# url: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data +# config: - +# +# Provides plugin lookup and meta data extraction utility functions. +# It's used to abstract module+option management in applications. +# For consolidating internal use and external/tool accessibility. +# +# The key:value format is language-agnostic. It's basically YAML in +# a topmost script comment. For Python only # hash comments though. +# Uses common field names, a documentation block, and an obvious +# `config: { .. }` spec for options and defaults. +# +# It neither imposes a specific module/plugin API, nor config storage, +# and doesn't fixate module loading. It's really just meant to look +# up meta infos. +# This approach avoids in-code values/inspection, externalized meta +# descriptors, and any hodgepodge or premature module loading just to +# uncover module description fields. +# +# plugin_meta() +# ‾‾‾‾‾‾‾‾‾‾‾‾‾ +# Is the primary function to extract a meta dictionary from files. +# It either reads from a given module= name, a literal fn=, or just +# src= code, and as fallback inspects the last stack frame= else. +# +# module_list() +# ‾‾‾‾‾‾‾‾‾‾‾‾‾ +# Returns basenames of available/installed plugins. It uses the +# plugin_base=[] list for module relation. Which needs to be set up +# beforehand, or injected. +# +# add_plugin_defaults() +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +# Populates a config_options{} and plugin_states{} list. Used for +# initial setup, or when adding new plugins, etc. Both dicts might +# also be ConfigParser stores, or implement magic __set__ handling +# to act on state changes. +# +# get_data() +# ‾‾‾‾‾‾‾‾‾‾ +# Is mostly an alias for pkgutil.get_data(). It abstracts the main +# base path, allows PYZ usage, and adds some convenience flags.‾ +# It's somewhat off-scope for plugin management, but used internally. +# +# argparse_map() +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +# Converts a list of config: options with arg: attribute for use as +# argparser parameters. +# +# +# Generally this scheme concerns itself more with plugin basenames. +# That is: module scripts in a package like `ext.plg1` and `ext.plg2`. +# It can be initialized by injecting the plugin-package basename into +# plugin_base = []. The associated paths will be used for module +# lookup via pkgutil.iter_modules(). +# +# And a central module can be extended with new lookup locations best +# by attaching new locations itself via module.__path__ + ["./local"] +# for example. +# +# Plugin loading thus becomes as simple as __import__("ext.local"). +# The attached plugin_state config dictionary in most cases can just +# list module basenames, if there's only one set to manage. + + +import sys +import os +import re +import functools +import pkgutil +import inspect +try: + from gzip import decompress as gzip_decode # Py3 only +except ImportError: + try: + from compat2and3 import gzip_decode # st2 stub + except ImportError: + import zlib + def gzip_decode(bytestr): + """ haphazard workaround """ + return zlib.decompress(bytestr, 16 + zlib.MAX_WBITS) +import zipfile +import argparse + +__all__ = [ + "get_data", "module_list", "plugin_meta", "add_plugin_defaults" +] + + +# Injectables +# ‾‾‾‾‾‾‾‾‾‾‾ +""" injectable callback function for logging """ +log_ERR = lambda *x: None + +""" +File lookup relation for get_data(), should name a top-level package. +(Equivalent PluginBase(package=…)) +""" +module_base = "config" + +""" +Package/module names for module_list() and plugin_meta() lookups. +All associated paths will be scanned for module/plugin basenames. +(Equivalent to `searchpath` in PluginBase) +""" +plugin_base = ["channels"] + + +# Compatiblity +# ‾‾‾‾‾‾‾‾‾‾‾‾ +def renamed_arguments(renamed): + """ map old argument names """ + def wrapped(func): + def execute(*args, **kwargs): + return func(*args, **{ + renamed.get(key, key): value + for key, value in kwargs.items() + }) + functools.update_wrapper(execute, func) + return execute + return wrapped + + +# Resource retrieval +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +@renamed_arguments({"fn": "filename", "gz": "gzip"}) +def get_data(filename, decode=False, gzip=False, file_base=None): + """ + Fetches file content from install path or from within PYZ + archive. This is just an alias and convenience wrapper for + pkgutil.get_data(). + Utilizes the module_base / plugin_base as top-level reference. + + :arg str fn: filename in pyz or bundle + :arg bool decode: text file decoding utf-8 + :arg bool gz: automatic gzdecode + :arg str file_base: alternative base module reference + """ + try: + data = pkgutil.get_data(file_base or module_base, filename) + if gzip: + data = gzip_decode(data) + if decode: + return data.decode("utf-8", errors='ignore') + return str(data) + except: + # log_ERR("get_data() didn't find:", fn, "in", file_base) + pass + + +# Plugin name lookup +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +def module_list(extra_paths=None): + """ + Search through ./plugins/ (and other configured plugin_base + names → paths) and get module basenames. + + :arg list extra_paths: in addition to plugin_base list + """ + + # Convert plugin_base package names into paths for iter_modules + paths = [] + for mp in plugin_base: + if sys.modules.get(mp): + paths += sys.modules[mp].__path__ + elif os.path.exists(mp): + paths.append(mp) + + # Should list plugins within zips as well as local paths + ls = pkgutil.iter_modules(paths + (extra_paths or [])) + return [name for loader, name, ispkg in ls] + + +# Plugin => meta dict +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +def all_plugin_meta(): + """ + This is a trivial wrapper to assemble a complete dictionary + of available/installed plugins. It associates each plugin name + with a its meta{} fields. + """ + return { + name: plugin_meta(module=name) for name in module_list() + } + + +# Plugin meta data extraction +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +@renamed_arguments({"filename": "fn"}) +def plugin_meta(fn=None, src=None, module=None, frame=1, **kwargs): + """ + Extract plugin meta data block from different sources: + + :arg str fn: read literal files, or .pyz contents + :arg str src: from already uncovered script code + :arg str module: lookup per pkgutil, from plugin_base or top-level modules + :arg int frame: extract comment header of caller (default) + :arg list extra_base: additional search directories + :arg ist max_length: maximum size to read from files + """ + + # Try via pkgutil first, + # find any plugins.* modules, or main packages + if module: + fn = module + for base in plugin_base + kwargs.get("extra_base", []): + try: + src = get_data(fn=fn+".py", decode=True, file_base=base) + if src: + break + except: + continue # plugin_meta_extract() will print a notice later + + # Real filename/path + elif fn and os.path.exists(fn): + src = open(fn).read(kwargs.get("max_length", 6144)) + + # Else get source directly from caller + elif not src and not fn: + module = inspect.getmodule(sys._getframe(frame+1)) # decorator+1 + fn = inspect.getsourcefile(module) + src = inspect.getcomments(module) + + # Assume it's a filename within a zip + elif fn: + intfn = "" + while fn and len(fn) and not os.path.exists(fn): + fn, add = os.path.split(fn) + intfn = add + "/" + intfn + if len(fn) >= 3 and intfn and zipfile.is_zipfile(fn): + src = zipfile.ZipFile(fn, "r").read(intfn.strip("/")) + + # Extract source comment into meta dict + if not src: + src = "" + if not isinstance(src, str): + src = src.decode("utf-8", errors='replace') + return plugin_meta_extract(src, fn) + + +# Comment and field extraction logic +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +def plugin_meta_extract(src="", fn=None, literal=False): + """ + Finds the first comment block. Splits key:value header + fields from comment. Turns everything into an dict, with + some stub fields if absent. + + :arg str src: from existing source code + :arg int fn: set filename attribute + :arg bool literla: just split comment from doc + """ + + # Defaults + meta = { + "id": os.path.splitext(os.path.basename(fn or ""))[0], + "fn": fn, + "api": "python", + "type": "module", + "category": None, + "priority": None, + "version": "0", + "title": fn, + "description": "no description", + "config": [], + "doc": "" + } + + # Extract coherent comment block + src = src.replace("\r", "") + if not literal: + src = rx.comment.search(src) + if not src: + log_ERR("Couldn't read source meta information:", fn) + return meta + src = src.group(0) + src = rx.hash.sub("", src).strip() + + # Split comment block + if src.find("\n\n") > 0: + src, meta["doc"] = src.split("\n\n", 1) + + # Turn key:value lines into dictionary + for field in rx.keyval.findall(src): + meta[field[0].replace("-", "_")] = field[1].strip() + meta["config"] = plugin_meta_config(meta.get("config") or "") + + return meta + + +# Unpack config: structures +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +def plugin_meta_config(str): + """ + Further breaks up the meta['config'] descriptor. + Creates an array from JSON/YAML option lists. + + :arg str str: unprocessed config: field + + Stubs out name, value, type, description if absent. + # config: + { name: 'var1', type: text, value: "default, ..." } + { name=option2, type=boolean, $value=1, etc. } + """ + + config = [] + for entry in rx.config.findall(str): + entry = entry[0] or entry[1] + opt = { + "type": None, + "name": None, + "description": "", + "value": None + } + for field in rx.options.findall(entry): + opt[field[0]] = (field[1] or field[2] or field[3] or "").strip() + # normalize type + opt["type"] = config_opt_type_map.get(opt["type"], opt["type"] or "str") + # preparse select: + if opt.get("select"): + opt["select"] = config_opt_parse_select(opt.get("select", "")) + config.append(opt) + return config + +# split up `select: 1=on|2=more|3=title` or `select: foo|bar|lists` +def config_opt_parse_select(s): + if re.search("([=:])", s): + return dict(rx.select_dict.findall(s)) + else: + return dict([(v, v) for v in rx.select_list.findall(s)]) + +# normalize type:names to `str`, `text`, `bool`, `int`, `select`, `dict` +config_opt_type_map = dict( + longstr="text", string="str", boolean="bool", checkbox="bool", integer="int", number="int", + choice="select", options="select", table="dict", array="dict" +) + + +# Comment extraction regexps +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +class rx: + """ + Pretty crude comment splitting approach. But works + well enough already. Technically a YAML parser would + do better; but is likely overkill. + """ + + comment = re.compile(r"""(^ {0,4}#.*\n)+""", re.M) + hash = re.compile(r"""(^ {0,4}#{1,2} {0,3}\r*)""", re.M) + keyval = re.compile(r""" + ^([\w-]+):(.*$(?:\n(?![\w-]+:).+$)*) # plain key:value lines + """, re.M | re.X) + config = re.compile(r""" + \{ ((?: [^\{\}]+ | \{[^\}]*\} )+) \} # JSOL/YAML scheme {...} dicts + | \< (.+?) \> # old HTML style + """, re.X) + options = re.compile(r""" + ["':$]? (\w*) ["']? # key or ":key" or '$key' + \s* [:=] \s* # "=" or ":" + (?: " ([^"]*) " + | ' ([^']*) ' # "quoted" or 'singl' values + | ([^,]*) # or unquoted literals + ) + """, re.X) + select_dict = re.compile(r"(\w+)\s*[=:>]+\s*([^=,|:]+)") + select_list = re.compile(r"\s*([^,|;]+)\s*") + + + +# ArgumentParser options conversion +# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ +def argparse_map(opt): + """ + As variation of in-application config: options, this method converts + cmdline argument specifiers. + + # config: + { arg: -i, name: input[], type: str, description: input files } + + Which allows to collect argumentparser options from different plugins. + The only difference to normal config entries is the `arg:` attribute. + + · It adds array arguments with a [] name suffix, or a `*` type suffix. + Else even a `?` or `+` and numeric counts after the type flag. + · Understands the types `str`, `int` and `bool`. + · Entries may carry a `hidden: 1` or `required: 1` attribute. + · And `help:` is an alias to `description:` + And `default:` an alias for `value:` + · While `type: select` utilizes the `select: a|b|c` format as usual. + + ArgParsers const=, metavar= flag, or type=file are not aliased here. + Basically returns a dictionary that can be fed per **kwargs directly + to an ArgumentParsers add_argument(). Iterate over each plugins + meta['config'][] options to convert them. + """ + if not ("arg" in opt and opt["name"] and opt["type"]): + return {} + + # Extract --flag names + args = opt["arg"].split() + re.findall(r"-+\w+", opt["name"]) + + # Prepare mapping options + typing = re.findall(r"bool|str|\[\]|const|false|true", opt["type"]) + naming = re.findall(r"\[\]", opt["name"]) + name = re.findall(r"(?+|==*|-+>|:=+)\s*", s.strip(), 1) + for s in re.split(r"\s*[|;,]\s*", _value) + ]) + else: + val = str(_value) + conf_options[_name] = val + + # Initial plugin activation status + if module and module not in conf_plugins: + conf_plugins[module] = meta.get("priority") in ( + "core", "builtin", "always", "default", "standard" + ) Index: st2.py ================================================================== --- st2.py +++ st2.py @@ -15,13 +15,14 @@ # category: sound # depends: python (>= 2.7) | python3 (>= 3.2), python:pygtk | python:gobject, # python:requests (>= 1.1), python:pyquery, python:pillow, python:xdg # alias: streamtuner2, main # id: st2 -# pack: *.py, gtk3.xml.gz, bin, channels/__init__.py, bundle/*.py, CREDITS, help/index.page, -# streamtuner2.desktop, README, help/streamtuner2.1=/usr/share/man/man1/, -# NEWS=/usr/share/doc/streamtuner2/, icon.png=/usr/share/pixmaps/streamtuner2.png +# pack: *.py, pluginconf/*.py, gtk3.xml.gz, bin, channels/__init__.py, bundle/*.py, +# streamtuner2.desktop, icon.png=/usr/share/pixmaps/streamtuner2.png, +# help/index.page, help/streamtuner2.1=/usr/share/man/man1/, +# NEWS=/usr/share/doc/streamtuner2/, README, CREDITS # architecture: all # # Streamtuner2 is a GUI for browsing internet radio directories, # music collections, and video services - grouped by genres or # categories. It runs your preferred audio player or streamripper