1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
-
+
+
+
+
+
+
|
# encoding: utf-8
# api: python
##type: extract
# category: config
# title: Plugin configuration
# description: Read meta data, pyz/package contents, module locating
# version: 0.7.9
# version: 0.8.0
# state: stable
# classifiers: documentation
# depends: python >= 2.7
# suggests: python:flit, python:PySimpleGUI
# license: PD
# priority: core
# api-docs: https://fossil.include-once.org/pluginspec/doc/trunk/html/index.html
# docs: https://fossil.include-once.org/pluginspec/
# url: http://fossil.include-once.org/streamtuner2/wiki/plugin+meta+data
# config: -
# format: off
# permissive: 0.75
# pylint: disable=invalid-name
# console-scripts: flit-pluginconf=pluginconf.flit:main, pluginconf.flit=pluginconf.flit:main
#
# Provides plugin lookup and meta data extraction utility functions.
# It's used to abstract module+option management in applications.
# For consolidating internal use and external/tool accessibility.
# Generally these functions are highly permissive / error tolerant,
# to preempt initialization failures for applications.
#
# The key:value format is language-agnostic. It's basically YAML in
# a topmost script comment. For Python only # hash comments though.
# Uses common field names, a documentation block, and an obvious
# `config: { .. }` spec for options and defaults.
#
# It neither imposes a specific module/plugin API, nor config storage,
# and doesn't fixate module loading. It's really just meant to look
# up meta infos.
# This approach avoids in-code values/inspection, externalized meta
# descriptors, and any hodgepodge or premature module loading just to
# uncover module description fields.
#
# plugin_meta()
# ‾‾‾‾‾‾‾‾‾‾‾‾‾
# Is the primary function to extract a meta dictionary from files.
# It either reads from a given module= name, a literal fn=, or just
# src= code, and as fallback inspects the last stack frame= else.
#
# The resulting dict allows [key] and .key access. The .config
# list further access by option .name.
#
# module_list()
# ‾‾‾‾‾‾‾‾‾‾‾‾‾
# Returns basenames of available/installed plugins. It uses the
# plugin_base=[] list for module relation. Which needs to be set up
# beforehand, or injected.
#
|
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
+
+
-
+
-
-
-
+
+
+
-
-
-
+
+
+
+
-
+
-
+
-
+
-
-
-
-
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
-
+
+
-
+
+
-
+
-
+
-
-
-
+
+
+
-
-
+
-
+
-
+
-
-
+
+
-
+
-
-
+
+
|
#
# argparse_map()
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Converts a list of config: options with arg: attribute for use as
# argparser parameters.
#
#
# Simple __import__() scheme
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
# Generally this scheme concerns itself more with plugin basenames.
# That is: module scripts in a package like `ext.plg1` and `ext.plg2`.
# That is: module scripts in a package like `plugins.plg1`. To do so,
# It can be initialized by injecting the plugin-package basename into
# plugin_base = []. The associated paths will be used for module
# lookup via pkgutil.iter_modules().
# Ideally a designated reference base_module should be extended with
# new lookup locations via `module.__path__ += ["./local"]` for exmple.
# Plugin loading thus becomes as simple as __import__("ext.local").
# Using a plugin_state config dictionary in most cases can just list
# Thus a plugin_state config dictionary in most cases can just list
# module basenames, if there's only one namespace to manage. (Plugin
# names unique across project.)
# names unique across application.)
"""
Plugin meta extraction and module lookup
Plugin meta extraction and module lookup.
* Main function `plugin_meta(filename=…)` unpacks
[meta fields](https://fossil.include-once.org/pluginspec/)
into dictionaries.
* Other utility code is about module location, but requires
some initialization.
"PluginMeta", "all_plugin_meta",
"PluginMeta", "OptionList", "all_plugin_meta",
"data_root", "plugin_base", "config_opt_type_map",
]
# Injectables
# ‾‾‾‾‾‾‾‾‾‾‾
""" injectable callback function for logging """
(Equivalent PluginBase(package=…))
(Equivalent to `PluginBase(package=…)`)
"""
module_base = "config"
Package/module names for module_list() and plugin_meta() lookups.
All associated paths will be scanned for module/plugin basenames.
(Equivalent to `searchpath` in PluginBase)
Package/module names (or directories) for module_list() and plugin_meta()
lookups. Associated paths (`__path__`) will be scanned for module/plugin
basenames. (Similar to `PluginBase(searchpath=…)`)
"""
plugin_base = ["channels"]
""" normalize config type: names to `str`, `text`, `bool`, `int`, `select`, `dict` """
def get_data(filename, decode=False, gzip=False, file_base=None):
def get_data(filename, decode=False, gzip=False, file_root=None):
"""
Fetches file content from install path or from within PYZ
archive. This is just an alias and convenience wrapper for
pkgutil.get_data().
Utilizes the module_base / plugin_base as top-level reference.
Utilizes the data_root as top-level reference.
Parameters
----------
filename : str
filename in pyz or bundle
decode : bool
text file decoding utf-8
gzip : bool
automatic gzdecode
file_base : list
alternative base module reference
file_root : list
alternative base module (application or pyz root)
Returns
-------
str : file contents
"""
try:
data = pkgutil.get_data(file_base or module_base, filename)
data = pkgutil.get_data(file_root or data_root, filename)
if gzip:
data = gzip_decode(data)
if decode:
return data.decode("utf-8", errors='ignore')
return str(data)
except:
log.error("get_data() didn't find:", fn, "in", file_base)
except: #(FileNotFoundError, IOError, OSError, ImportError, gzip.BadGzipFile):
log.error("get_data() didn't find '%s' in '%s'", filename, file_root)
pass
# Plugin name lookup
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
def module_list(extra_paths=None):
"""
|
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
|
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
|
-
+
-
+
+
+
+
+
+
-
+
-
+
|
# Plugin meta data extraction
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
@renamed_arguments({"fn": "filename"})
def plugin_meta(filename=None, src=None, module=None, frame=1, **kwargs):
"""
Extract plugin meta data block from different sources:
Extract plugin meta data block from specified source.
Parameters
----------
filename : str
Read literal files, or .pyz contents.
src : str
From already uncovered script code.
module : str
Lookup per pkgutil, from plugin_base or top-level modules.
frame : int
Extract comment header of caller (default).
extra_base : list
Additional search directories.
max_length : list
Maximum size to read from files.
Maximum size to read from files (6K default).
Returns
-------
dict : Extracted comment fields, with config: preparsed
The result dictionary has fields accessible as e.g. `pmd["title"]`
or `pmd.version`. The documentation block after all fields: is called
["doc"]`. And `pmd.config` already parsed as a list of dictionaries.
"""
# Try via pkgutil first,
# find any plugins.* modules, or main packages
if module:
filename = module
for base in plugin_base + kwargs.get("extra_base", []):
try:
#log.debug(f"mod={base} fn={filename}.py")
src = get_data(filename=filename+".py", decode=True, file_base=base)
src = get_data(filename=filename+".py", decode=True, file_root=base)
if src:
break
except:
except (IOError, OSError, FileNotFoundError):
continue # plugin_meta_extract() will print a notice later
# Real filename/path
elif filename and os.path.exists(filename):
src = open(filename).read(kwargs.get("max_length", 6144))
# Else get source directly from caller
|
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
|
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
|
-
+
-
-
-
+
+
+
-
+
|
}
# Extract coherent comment block
src = src.replace("\r", "")
if not literal:
src = rx.comment.search(src)
if not src:
log.error("Couldn't read source meta information:", filename)
log.warn("Couldn't read source meta information: %s", filename)
return meta
src = src.group(0)
src = rx.hash.sub("", src).strip()
# Split comment block
if src.find("\n\n") > 0:
src, meta["doc"] = src.split("\n\n", 1)
# Turn key:value lines into dictionary
for field in rx.keyval.findall(src):
meta[field[0].replace("-", "_").lower()] = field[1].strip()
meta["config"] = plugin_meta_config(meta.get("config") or "")
return PluginMeta(meta)
# Dict/list wrappers
# ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
class PluginMeta(dict):
"""
Plugin meta data, as dictionary with alternative .property access.
Returned for each `plugin_meta()` result, and config: options.
Non-existent .fieldnames just resolve to `""`.
Plugin meta data as dictionary`{}`, or alternatively `.property` access.
Returned for each `plugin_meta()` result, and individual `config:` options.
Absent `.field` access resolves to `""`.
"""
def __getattr__(self, key, default=""):
""" Return [key] for .property access, else `""`. """
if key == "config":
default = OptionList()
return self.get(key, default)
def __setattr__(self, key, val):
""" Shouldn't really have this, but for parity. """
self[key] = val
class OptionList(list):
"""
List of config: options, with additional .name access (name= from option entry).
List of `config:` options, with alernative `.name` access (lookup by name= from option entry).
"""
def __getattr__(self, key):
""" Returns list entry with name= equaling .name access """
for opt in self:
if opt.name == key:
return opt
|