Internet radio browser GUI for music/video streams from various directory services.

⌈⌋ ⎇ branch:  streamtuner2


Check-in [e136a78c73]

Overview
Comment:Finalize allowed filename extensions for exporting. Normalize Python3 string decoding (errors='ignore' per default). Update XSPF and SMIL export. Use row={} template now, instead of just carrying over title= to rewritten playlists.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | action-mapfmts
Files: files | file ages | folders
SHA1: e136a78c735bba10b060e52afa4fc67dab63ed74
User & Date: mario on 2015-04-10 16:40:44
Other Links: branch diff | manifest | tags
Context
2015-04-10
17:34
Fix str→bytes saving for Py3. Tmplement tmp_files[] cleanup. Leaf check-in: 36c234a70b user: mario tags: action-mapfmts
16:40
Finalize allowed filename extensions for exporting. Normalize Python3 string decoding (errors='ignore' per default). Update XSPF and SMIL export. Use row={} template now, instead of just carrying over title= to rewritten playlists. check-in: e136a78c73 user: mario tags: action-mapfmts
13:50
Implement filename update in SaveAs dialog on changing FileFilter (.m3u, .pls, .xspf) extension. check-in: 0a9cb60b3a user: mario tags: action-mapfmts
Changes

Modified action.py from [a1f8b1c311] to [42bec82228].

15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

38
39
40
41
42
43
44
# Each channel plugin has a .listtype which defines the linked
# audio playlist format. It's "pls", seldomly "m3u", or "xspf".
# Some channels list raw "srv" addresses, while Youtube "href"
# entries to Flash videos.
#
# As fallback the playlist URL is retrieved and its MIME type
# checked, and its content regexped to guess the link format.
# Lastly a playlist type suitable for audio players recreated.
# Which is somewhat of a security feature, playlists get cleaned
# up this way. The conversion is not strictly necessary for all
# players, as basic PLS is supported by most.
#
# And finally this module is also used by exporting and (perhaps
# in the future) playlist importing features.


import re
import os
from ahttp import fix_url as http_fix_url, session
from config import conf, __print__ as debug, dbg
import platform
import copy
import json



# Coupling to main window
#
main = None









|
|














>







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# Each channel plugin has a .listtype which defines the linked
# audio playlist format. It's "pls", seldomly "m3u", or "xspf".
# Some channels list raw "srv" addresses, while Youtube "href"
# entries to Flash videos.
#
# As fallback the playlist URL is retrieved and its MIME type
# checked, and its content regexped to guess the link format.
# Lastly a playlist format suitable for audio players recreated.
# Which is somewhat of a security feature; playlists get cleaned
# up this way. The conversion is not strictly necessary for all
# players, as basic PLS is supported by most.
#
# And finally this module is also used by exporting and (perhaps
# in the future) playlist importing features.


import re
import os
from ahttp import fix_url as http_fix_url, session
from config import conf, __print__ as debug, dbg
import platform
import copy
import json
from datetime import datetime


# Coupling to main window
#
main = None


126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
#
def help(*args):
    run("yelp /usr/share/doc/streamtuner2/help/")


# Calls player for stream url and format
#
def play(url, audioformat="audio/mpeg", source="pls", row={}):
    cmd = mime_app(audioformat, conf.play)
    cmd = interpol(cmd, url, source, row)
    run(cmd)


# Call streamripper
#
def record(url, audioformat="audio/mpeg", source="href", row={}):
    cmd = mime_app(audioformat, conf.record)
    cmd = interpol(cmd, url, source, row)
    run(cmd)


# OS shell command escaping
#
def quote(ins):
    if type(ins) is str:







|

|





|

|







127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#
def help(*args):
    run("yelp /usr/share/doc/streamtuner2/help/")


# Calls player for stream url and format
#
def play(row={}, audioformat="audio/mpeg", source="pls", url=None):
    cmd = mime_app(audioformat, conf.play)
    cmd = interpol(cmd, url or row["url"], source, row)
    run(cmd)


# Call streamripper
#
def record(row={}, audioformat="audio/mpeg", source="href", url=None):
    cmd = mime_app(audioformat, conf.record)
    cmd = interpol(cmd, url or row["url"], source, row)
    run(cmd)


# OS shell command escaping
#
def quote(ins):
    if type(ins) is str:
173
174
175
176
177
178
179

180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#  · Also understands short aliases %l, %f, %d.
#  · And can embed %title or %genre placeholders.
#  · Replace .pls URL with local .m3u file depending on map.
#
def interpol(cmd, url, source="pls", row={}):

    # inject other meta fields

    if row:
        for field in row:
            cmd = cmd.replace("%"+field, "%r" % row.get(field))

    # add default if cmd has no %url placeholder
    if cmd.find("%") < 0:
        cmd = cmd + " %pls"
        # "pls" as default requires no conversion for most channels, and seems broadly supported by players

    # standard placeholders
    for dest, rx in placeholder_map.items():
        if re.search(rx, cmd, re.X):
            # from .pls to .m3u
            fn_or_urls = convert_playlist(url, listfmt(source), listfmt(dest), local_file=True, title=row.get("title", ""))
            # insert quoted URL/filepath
            return re.sub(rx, quote(fn_or_urls), cmd, 2, re.X)

    return "false"


# Substitute .pls URL with local .m3u, or direct srv addresses, or leaves URL asis.
#  · Takes a single input `url`.
#  · But returns a list of [urls] after playlist extraction.
#  · If repackaging as .m3u/.pls/.xspf, returns the local [fn].
#
# TODO: This still needs some rewrite to reuse the incoming row={},
# and keep station titles for converted playlists.
#
def convert_playlist(url, source, dest, local_file=True, title=""):
    urls = []
    debug(dbg.PROC, "convert_playlist(", url, source, dest, ")")

    # Leave alone if format matches, or if "srv" URL class, or if not http (local path, mms:/rtsp:)
    if source == dest or source in ("srv", "href") or not re.match("(https?|spdy)://", url):
        return [url]
    
    # Retrieve from URL
    (mime, cnt) = http_probe_get(url)
    
    # Leave streaming server as is







>













|







|



<
<
<
|



|







174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206



207
208
209
210
211
212
213
214
215
216
217
218
#  · Also understands short aliases %l, %f, %d.
#  · And can embed %title or %genre placeholders.
#  · Replace .pls URL with local .m3u file depending on map.
#
def interpol(cmd, url, source="pls", row={}):

    # inject other meta fields
    row = copy.copy(row)
    if row:
        for field in row:
            cmd = cmd.replace("%"+field, "%r" % row.get(field))

    # add default if cmd has no %url placeholder
    if cmd.find("%") < 0:
        cmd = cmd + " %pls"
        # "pls" as default requires no conversion for most channels, and seems broadly supported by players

    # standard placeholders
    for dest, rx in placeholder_map.items():
        if re.search(rx, cmd, re.X):
            # from .pls to .m3u
            fn_or_urls = convert_playlist(url, listfmt(source), listfmt(dest), local_file=True, row=row)
            # insert quoted URL/filepath
            return re.sub(rx, quote(fn_or_urls), cmd, 2, re.X)

    return "false"


# Substitute .pls URL with local .m3u, or direct srv addresses, or leaves URL asis.
#  · Takes a single input `url` (and original row{} as template).
#  · But returns a list of [urls] after playlist extraction.
#  · If repackaging as .m3u/.pls/.xspf, returns the local [fn].
#



def convert_playlist(url, source, dest, local_file=True, row={}):
    urls = []
    debug(dbg.PROC, "convert_playlist(", url, source, dest, ")")

    # Leave alone if format matches, or if already "srv" URL, or if not http (local path, mms:/rtsp:)
    if source == dest or source in ("srv", "href") or not re.match("(https?|spdy)://", url):
        return [url]
    
    # Retrieve from URL
    (mime, cnt) = http_probe_get(url)
    
    # Leave streaming server as is
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
            debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls)
            
    # Return original, or asis for srv targets
    if not urls:
        return [url]
    elif dest in ("srv", "href"):
        return urls
    debug( urls )

    # Otherwise convert to local file
    if local_file:
        fn, is_unique = tmp_fn(cnt, dest)
        with open(fn, "wb") as f:
            debug(dbg.DATA, "exporting with format:", dest, " into filename:", fn)
            f.write( save_playlist(source="srv", multiply=True).export(urls=urls, dest=dest, title=title) )
        return [fn]
    else:
        return urls



# Tries to fetch a resource, aborts on ICY responses.







<






|







242
243
244
245
246
247
248

249
250
251
252
253
254
255
256
257
258
259
260
261
262
            debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls)
            
    # Return original, or asis for srv targets
    if not urls:
        return [url]
    elif dest in ("srv", "href"):
        return urls


    # Otherwise convert to local file
    if local_file:
        fn, is_unique = tmp_fn(cnt, dest)
        with open(fn, "wb") as f:
            debug(dbg.DATA, "exporting with format:", dest, " into filename:", fn)
            f.write( save_playlist(source="srv", multiply=True).export(urls, row, dest) )
        return [fn]
    else:
        return urls



# Tries to fetch a resource, aborts on ICY responses.
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
        mime = listfmt_t.get(mime)
    # Raw content (mp3, flv)
    elif mediafmt_t.get(mime):
        debug(dbg.ERR, "Got media MIME type for expected playlist", mime, " on url=", url)
        mime = mediafmt_t.get(mime)
        return (mime, url)
    # Rejoin body
    content = "\n".join(r.iter_lines())
    return (mime, content)



# Extract URLs from playlist formats:
#
class extract_playlist(object):







|







279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
        mime = listfmt_t.get(mime)
    # Raw content (mp3, flv)
    elif mediafmt_t.get(mime):
        debug(dbg.ERR, "Got media MIME type for expected playlist", mime, " on url=", url)
        mime = mediafmt_t.get(mime)
        return (mime, url)
    # Rejoin body
    content = "\n".join(str.decode(errors='replace') for str in r.iter_lines())
    return (mime, content)



# Extract URLs from playlist formats:
#
class extract_playlist(object):
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344


345


346
347
348
349
350
351
352
       "asf":  r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ",
       "raw":  r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ",
    }


# Save rows in one of the export formats.
#
# The export() version uses urls[]+title= as input, converts it into a
# list of rows{} beforehand.
#
# While store() requires rows{} to begin with, to perform a full
# conversion. Can save directly to a file name.
#
class save_playlist(object):

    # if converting
    source = "pls"
    # expand multiple server URLs into duplicate entries in target playlist
    multiply = True
    # constructor
    def __init__(self, source, multiply):
        self.source = source
        self.multiply = multiply
    

    # Used by playlist_convert(), to transform a list of extracted URLs
    # into a local .pls/.m3u collection again. Therefore injects the
    # `title` back into each of the URL rows.
    def export(self, urls=None, title=None, dest="pls"):


        rows = [ { "url": url, "title": title } for url in urls ]


        return self.store(rows, dest)

    # Export a playlist from rows{}
    def store(self, rows=None, dest="pls"):
    
        # can be just a single entry
        rows = copy.deepcopy(rows)







|
|


















|
|
>
>
|
>
>







314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
       "asf":  r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ",
       "raw":  r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ",
    }


# Save rows in one of the export formats.
#
# The export() version uses urls[]+row/title= as input, converts it into
# a list of rows{} beforehand.
#
# While store() requires rows{} to begin with, to perform a full
# conversion. Can save directly to a file name.
#
class save_playlist(object):

    # if converting
    source = "pls"
    # expand multiple server URLs into duplicate entries in target playlist
    multiply = True
    # constructor
    def __init__(self, source, multiply):
        self.source = source
        self.multiply = multiply
    

    # Used by playlist_convert(), to transform a list of extracted URLs
    # into a local .pls/.m3u collection again. Therefore injects the
    # `title` back into each of the URL rows / or uses row{} template.
    def export(self, urls=[], row={}, dest="pls", title=None):
        row["title"] = row.get("title", title or "unnamed stream")
        rows = []
        for url in urls:
            row.update(url=url)
            rows.append(row)
        return self.store(rows, dest)

    # Export a playlist from rows{}
    def store(self, rows=None, dest="pls"):
    
        # can be just a single entry
        rows = copy.deepcopy(rows)
398
399
400
401
402
403
404
405
406
407

408
409
410
411
412
413
414
    # JSON (native lists of streamtuner2)
    def json(self, rows):
        return json.dumps(rows, indent=4)


    # XSPF
    def xspf(self, rows):
        return """<?xml version="1.0" encoding="UTF-8"?>\n"""					\
            + """<?http header="Content-Type: application/xspf+xml" ?>\n"""			\
            + """<playlist version="1" xmlns="http://xspf.org/ns/0/">\n\t<trackList>\n"""	\

            + "".join("""\t\t<track>\n%s\t\t</track>\n""" % self.xspf_row(row, self.xspf_map) for row in rows)	\
            + """\t</trackList>\n</playlist>\n"""
    # individual tracks
    def xspf_row(self, row, map):
        return "".join("""\t\t\t<%s>%s</%s>\n""" % (tag, xmlentities(row[attr]), tag) for attr,tag in map.items() if row.get(attr))
    # dict to xml tags
    xspf_map = dict(title="title", url="location", homepage="info", playing="annotation", description="info")







|
|
|
>







400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
    # JSON (native lists of streamtuner2)
    def json(self, rows):
        return json.dumps(rows, indent=4)


    # XSPF
    def xspf(self, rows):
        return """<?xml version="1.0" encoding="UTF-8"?>\n"""				\
            + """<?http header="Content-Type: application/xspf+xml; x-ns='http://xspf.org/ns/0/'; x-gen=streamtuner2" ?>\n"""	\
            + """<playlist version="1" xmlns="http://xspf.org/ns/0/">\n"""		\
            + """\t<date>%s</date>\n\t<trackList>\n""" % datetime.now().isoformat()	\
            + "".join("""\t\t<track>\n%s\t\t</track>\n""" % self.xspf_row(row, self.xspf_map) for row in rows)	\
            + """\t</trackList>\n</playlist>\n"""
    # individual tracks
    def xspf_row(self, row, map):
        return "".join("""\t\t\t<%s>%s</%s>\n""" % (tag, xmlentities(row[attr]), tag) for attr,tag in map.items() if row.get(attr))
    # dict to xml tags
    xspf_map = dict(title="title", url="location", homepage="info", playing="annotation", description="info")
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
            txt += """\t<entry>\n\t\t<title>%s</title>\n\t\t<ref href="%s"/>\n\t</entry>\n""" % (xmlentities(row["title"]), xmlentities(row["url"]))
        txt += """</asx>\n"""
        return txt


    # SMIL
    def smil(self, rows):
        txt = """<smil>\n<head>\n\t<meta name="title" content="%s""/>\n</head>\n<body>\n\t<seq>\n""" % (rows[0]["title"])
        for row in rows:
            if row.get("url"):
                txt += """\t\t<audio src="%s"/>\n""" % row["url"]
        txt += """\t</seq>\n</body>\n</smil>\n"""
        return txt









|







432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
            txt += """\t<entry>\n\t\t<title>%s</title>\n\t\t<ref href="%s"/>\n\t</entry>\n""" % (xmlentities(row["title"]), xmlentities(row["url"]))
        txt += """</asx>\n"""
        return txt


    # SMIL
    def smil(self, rows):
        txt = """<smil>\n<head>\n\t<meta name="title" content="%s"/>\n</head>\n<body>\n\t<seq>\n""" % (rows[0]["title"])
        for row in rows:
            if row.get("url"):
                txt += """\t\t<audio src="%s"/>\n""" % row["url"]
        txt += """\t</seq>\n</body>\n</smil>\n"""
        return txt


Modified channels/__init__.py from [b6c3442f9b] to [9cf8287e24].

483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513

    #--------------------------- actions ---------------------------------

    # Invoke action.play() for current station.
    # Can be overridden to provide channel-specific "play" alternative
    def play(self):
        row = self.row()
        if row:
            # playlist and audio type
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            # invoke audio player
            action.play(row["url"], audioformat, listformat, row)
        else:
            self.status("No station selected for playing.")
        return row

    # Start streamripper/youtube-dl/etc
    def record(self):
        row = self.row()
        if row:
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            action.record(row.get("url"), audioformat, listformat, row=row)
        return row



    #--------------------------- utility functions -----------------------

    







|




|







|


|







483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513

    #--------------------------- actions ---------------------------------

    # Invoke action.play() for current station.
    # Can be overridden to provide channel-specific "play" alternative
    def play(self):
        row = self.row()
        if row and "url" in row:
            # playlist and audio type
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            # invoke audio player
            action.play(row, audioformat, listformat)
        else:
            self.status("No station selected for playing.")
        return row

    # Start streamripper/youtube-dl/etc
    def record(self):
        row = self.row()
        if row and "url" in row:
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            action.record(row, audioformat, listformat)
        return row



    #--------------------------- utility functions -----------------------

    

Modified channels/exportcat.py from [1a24ff64fc] to [02ad2daff2].

45
46
47
48
49
50
51
52







53
54
    def savewindow(self, *w):
        cn = self.parent.channel()
        source = cn.listformat
        streams = cn.streams[cn.current]
        fn = uikit.save_file("Export category", None, "%s.%s.%s" % (cn.module, cn.current, conf.export_format))
        __print__(dbg.PROC, "Exporting category to", fn)
        if fn:
            dest = re.findall("\.(m3u|pls|xspf|jspf|json|smil|wpl)8?$", fn)[0]







            action.save_playlist(source="asis", multiply=False).file(rows=streams, fn=fn, dest=dest)
        pass            







|
>
>
>
>
>
>
>


45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    def savewindow(self, *w):
        cn = self.parent.channel()
        source = cn.listformat
        streams = cn.streams[cn.current]
        fn = uikit.save_file("Export category", None, "%s.%s.%s" % (cn.module, cn.current, conf.export_format))
        __print__(dbg.PROC, "Exporting category to", fn)
        if fn:
            dest = re.findall("\.(m3u8?|pls|xspf|jspf|json|smil|asx)$", fn.lower())
            if dest:
                dest = dest[0]
            else:
                self.parent.status("Unsupported export playlist type (file extension).")
                return
            if dest == "m3u8":
                dest = "m3u"
            action.save_playlist(source="asis", multiply=False).file(rows=streams, fn=fn, dest=dest)
        pass            

Modified config.py from [39c7681328] to [cbe5c8fdc6].

338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
#
def get_data(fn, decode=False, gz=False, file_base="config"):
    try:
        bin = pkgutil.get_data(file_base, fn)
        if gz:
            bin = gzip_decode(bin)
        if decode:
            return bin.decode("utf-8")
        else:
            return str(bin)
    except:
        pass


# Search through ./channels/ and get module basenames.







|







338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
#
def get_data(fn, decode=False, gz=False, file_base="config"):
    try:
        bin = pkgutil.get_data(file_base, fn)
        if gz:
            bin = gzip_decode(bin)
        if decode:
            return bin.decode("utf-8", errors='ignore')
        else:
            return str(bin)
    except:
        pass


# Search through ./channels/ and get module basenames.
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
            intfn = add + "/" + intfn
        if len(fn) >= 3 and intfn and zipfile.is_zipfile(fn):
            src = zipfile.ZipFile(fn, "r").read(intfn.strip("/"))
            
    if not src:
        src = ""
    if type(src) is not str:
        src = src.decode("utf-8")

    return plugin_meta_extract(src, fn)


# Actual comment extraction logic
def plugin_meta_extract(src="", fn=None, literal=False):








|







394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
            intfn = add + "/" + intfn
        if len(fn) >= 3 and intfn and zipfile.is_zipfile(fn):
            src = zipfile.ZipFile(fn, "r").read(intfn.strip("/"))
            
    if not src:
        src = ""
    if type(src) is not str:
        src = src.decode("utf-8", errors='replace')

    return plugin_meta_extract(src, fn)


# Actual comment extraction logic
def plugin_meta_extract(src="", fn=None, literal=False):

Modified gtk3.xml.gz from [636af55d84] to [7df44525d1].

cannot compute difference between binary files

Modified uikit.py from [6ca85da059] to [913007a2bb].

395
396
397
398
399
400
401
402
403
404
405
406
407
408
409

    # Callback for changed FileFilter, updates current filename extension
    @staticmethod
    def save_file_filterchange(c):
        fn, ext = c.get_filename(), c.get_filter().get_name()
        if fn and ext:
            fn = os.path.basename(fn)
            c.set_current_name(re.sub(r"\.(m3u|pls|xspf|jspf|asx|json|smil|wpl)$", ext, fn))
        
    
    
    
    # pass updates from another thread, ensures that it is called just once
    @staticmethod
    def do(lambda_func):







|







395
396
397
398
399
400
401
402
403
404
405
406
407
408
409

    # Callback for changed FileFilter, updates current filename extension
    @staticmethod
    def save_file_filterchange(c):
        fn, ext = c.get_filename(), c.get_filter().get_name()
        if fn and ext:
            fn = os.path.basename(fn)
            c.set_current_name(re.sub(r"\.(m3u|pls|xspf|jspf|asx|json|smil|wpl)$", ext.strip("*"), fn))
        
    
    
    
    # pass updates from another thread, ensures that it is called just once
    @staticmethod
    def do(lambda_func):