Internet radio browser GUI for music/video streams from various directory services.

⌈⌋ ⎇ branch:  streamtuner2


Check-in [58d2981ca9]

Overview
Comment:Fix homepage url issue / quote() checks for list first now. Restructure playlist extraction into ordered dict; use in conver_playlist as probe formats. Make xml/json url decoding explicit, prepare for custom extractors (e.g. real json or xml traversal, full row/title extraction).
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 58d2981ca982ec258e00b9bd955567ede90c3551
User & Date: mario on 2015-04-18 20:39:08
Other Links: manifest | tags
Context
2015-04-18
20:41
Add [feeds] support for Jamendo. (Pretty much only albums, so track lookup works.) Better support for configured audio format/mime. Ogg Vorbis now works in albums and playlists. check-in: 90b1672010 user: mario tags: trunk
20:39
Fix homepage url issue / quote() checks for list first now. Restructure playlist extraction into ordered dict; use in conver_playlist as probe formats. Make xml/json url decoding explicit, prepare for custom extractors (e.g. real json or xml traversal, full row/title extraction). check-in: 58d2981ca9 user: mario tags: trunk
20:37
Guard appstate_init channels.current setting for absent plugins. check-in: 24fb9b895e user: mario tags: trunk
Changes

Modified action.py from [fc5074bffb] to [820487cd33].

53
54
55
56
57
58
59

60
61
62
63
64
65
66
    "video/x-ms-asf":       "asx",
    "application/xspf+xml": "xspf",
    "*/*":                  "href",  # "href" for unknown responses
    "url/direct":           "srv",
    "url/youtube":          "href",
    "url/http":             "href",
    "audio/x-pn-realaudio": "ram",

    "application/smil":     "smil",
    "application/vnd.ms-wpl":"smil",
    "audio/x-ms-wax":       "asx",
    "video/x-ms-asf":       "asx",
    "x-urn/st2-script":     "script", # unused
    "application/x-shockwave-flash": "href",  # fallback
}







>







53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    "video/x-ms-asf":       "asx",
    "application/xspf+xml": "xspf",
    "*/*":                  "href",  # "href" for unknown responses
    "url/direct":           "srv",
    "url/youtube":          "href",
    "url/http":             "href",
    "audio/x-pn-realaudio": "ram",
    "application/json":     "json",
    "application/smil":     "smil",
    "application/vnd.ms-wpl":"smil",
    "audio/x-ms-wax":       "asx",
    "video/x-ms-asf":       "asx",
    "x-urn/st2-script":     "script", # unused
    "application/x-shockwave-flash": "href",  # fallback
}
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    srv = "(%srv | %d | %s) \\b",
)

# Playlist format content probing (assert type)
playlist_content_map = [
   ("pls",  r""" (?i)\[playlist\].*NumberOfEntries """),
   ("xspf", r""" <\?xml .* <playlist .* ((?i)http://xspf\.org)/ns/0/ """),
   ("m3u",  r""" ^ \s* #(EXT)?M3U """),
   ("asx" , r""" <asx\b """),
   ("smil", r""" <smil[^>]*> .* <seq> """),
   ("html", r""" (?i)<(audio|video)\b[^>]+\bsrc\s*=\s*["']?https?:// """),
   ("wpl",  r""" <\?wpl \s+ version="1\.0" \s* \?> """),
   ("b4s",  r""" <WinampXML> """),   # http://gonze.com/playlists/playlist-format-survey.html
   ("jspf", r""" ^ \s* \{ \s* "playlist": \s* \{ """),
   ("asf",  r""" ^ \[Reference\] .*? ^Ref\d+= """),







|







90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    srv = "(%srv | %d | %s) \\b",
)

# Playlist format content probing (assert type)
playlist_content_map = [
   ("pls",  r""" (?i)\[playlist\].*NumberOfEntries """),
   ("xspf", r""" <\?xml .* <playlist .* ((?i)http://xspf\.org)/ns/0/ """),
   ("m3u",  r""" ^ \s* \#(EXT)?M3U """),
   ("asx" , r""" <asx\b """),
   ("smil", r""" <smil[^>]*> .* <seq> """),
   ("html", r""" (?i)<(audio|video)\b[^>]+\bsrc\s*=\s*["']?https?:// """),
   ("wpl",  r""" <\?wpl \s+ version="1\.0" \s* \?> """),
   ("b4s",  r""" <WinampXML> """),   # http://gonze.com/playlists/playlist-format-survey.html
   ("jspf", r""" ^ \s* \{ \s* "playlist": \s* \{ """),
   ("asf",  r""" ^ \[Reference\] .*? ^Ref\d+= """),
117
118
119
120
121
122
123

124
125
126
127
128
129
130
    except: debug(dbg.ERR, "Command not found:", cmd)


# Start web browser
#
def browser(url):
    bin = conf.play.get("url/http", "sensible-browser")

    run(bin + " " + quote(url))


# Open help browser, streamtuner2 pages
#
def help(*args):
    run("yelp /usr/share/doc/streamtuner2/help/")







>







118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    except: debug(dbg.ERR, "Command not found:", cmd)


# Start web browser
#
def browser(url):
    bin = conf.play.get("url/http", "sensible-browser")
    print url
    run(bin + " " + quote(url))


# Open help browser, streamtuner2 pages
#
def help(*args):
    run("yelp /usr/share/doc/streamtuner2/help/")
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    cmd = interpol(cmd, url or row["url"], source, row)
    run(cmd)


# OS shell command escaping
#
def quote(ins):
    if type(ins) is str:
        return "%r" % str(ins)
    else:
        return " ".join(["%r" % str(s) for s in ins])


# Convert e.g. "text/x-scpls" MIME types to just "pls" monikers
#
def listfmt(t = "pls"):
    return listfmt_t.get(t, t) # e.g. "pls" or still "text/x-unknown"








|
|

|







147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
    cmd = interpol(cmd, url or row["url"], source, row)
    run(cmd)


# OS shell command escaping
#
def quote(ins):
    if type(ins) is list:
        return " ".join(["%r" % str(s) for s in ins])
    else:
        return "%r" % str(ins)


# Convert e.g. "text/x-scpls" MIME types to just "pls" monikers
#
def listfmt(t = "pls"):
    return listfmt_t.get(t, t) # e.g. "pls" or still "text/x-unknown"

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
    for probe,rx in playlist_content_map:
        if re.search(rx, cnt, re.X|re.S):
            probe = listfmt(probe)
            break # with `probe` set

    # Check ambiguity (except pseudo extension)
    if len(set([source, mime, probe])) > 1:
        debug(dbg.ERR, "Possible playlist format mismatch:", (source, mime, probe, ext))

    # Extract URLs from content
    for fmt in ["pls", "xspf", "asx", "smil", "jspf", "m3u", "json", "asf", "jamj", "raw"]:
        if not urls and fmt in (source, mime, probe, ext, "raw"):
            urls = extract_playlist(cnt).format(fmt)
            debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls)
            
    # Return original, or asis for srv targets
    if not urls:
        return [url]







|


|







232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    for probe,rx in playlist_content_map:
        if re.search(rx, cnt, re.X|re.S):
            probe = listfmt(probe)
            break # with `probe` set

    # Check ambiguity (except pseudo extension)
    if len(set([source, mime, probe])) > 1:
        debug(dbg.ERR, "Possible playlist format mismatch:", "listformat={}, http_mime={}, rx_probe={}, ext={}".format(source, mime, probe, ext))

    # Extract URLs from content
    for fmt in [id[0] for id in extract_playlist.extr_urls]:
        if not urls and fmt in (source, mime, probe, ext, "raw"):
            urls = extract_playlist(cnt).format(fmt)
            debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls)
            
    # Return original, or asis for srv targets
    if not urls:
        return [url]
287
288
289
290
291
292
293




294
295
296
297
298
299
300
301
302
303





304

305
306

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326

327
328
329
330
331
332
333
    content = "\n".join(str.decode(errors='replace') for str in r.iter_lines())
    return (mime, content)



# Extract URLs from playlist formats:
#




class extract_playlist(object):

    # Content of playlist file
    src = ""
    def __init__(self, text):
        self.src = text
        
    # Extract only URLs from given source type
    def format(self, fmt):
        debug(dbg.DATA, "input regex:", fmt, len(self.src))





        # regex

        urls = re.findall(self.extr_urls[fmt], self.src, re.X)
        # xml entities

        urls = [xmlunescape(url) for url in urls]
        # json escaping
        urls = [url.replace("\\/", "/") for url in urls]
        # uniques
        urls = list(set(urls))
        return urls

    # Only look out for URLs, not local file paths
    extr_urls = {
       "pls":  r"(?im) ^ \s*File\d* \s*=\s* (\w+://[^\s]+) ",
       "m3u":  r" (?m) ^( \w+:// [^#\n]+ )",
       "xspf": r" (?x) <location> (\w+://[^<>\s]+) </location> ",
       "asx":  r" (?x) <ref \b[^>]+\b href \s*=\s* [\'\"] (\w+://[^\s\"\']+) [\'\"] ",
       "smil": r" (?x) <(?:audio|video|media)\b [^>]+ \b src \s*=\s* [^\"\']? \s* (\w+://[^\"\'\s]+) ",
       "jspf": r" (?x) \"location\" \s*:\s* \"(\w+://[^\"\s]+)\" ",
       "jamj": r" (?x) \"audio\" \s*:\s* \"(\w+:\\?/\\?/[^\"\s]+)\" ",
       "json": r" (?x) \"url\" \s*:\s* \"(\w+://[^\"\s]+)\" ",
       "asf":  r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ",
       "raw":  r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ",
    }



# Save rows in one of the export formats.
#
# The export() version uses urls[]+row/title= as input, converts it into
# a list of rows{} beforehand.
#







>
>
>
>









|
>
>
>
>
>
|
>
|
|
>
|
|
|
|
|
<

|
|
|
|
|
|
|
|
|
|
|
|
<
>







289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324

325
326
327
328
329
330
331
332
333
334
335
336
337

338
339
340
341
342
343
344
345
    content = "\n".join(str.decode(errors='replace') for str in r.iter_lines())
    return (mime, content)



# Extract URLs from playlist formats:
#
# It's entirely regex-based at the moment, because that's more
# resilient against mailformed XSPF or JSON.
# Needs proper extractors later for real playlist *imports*.
#
class extract_playlist(object):

    # Content of playlist file
    src = ""
    def __init__(self, text):
        self.src = text
        
    # Extract only URLs from given source type
    def format(self, fmt):
        debug(dbg.DATA, "input extractor/regex:", fmt, len(self.src))

        # find extractor
        if fmt in dir(self):
            return self.__dict__[fmt]()

        # regex scheme
        rx, decode = dict(self.extr_urls)[fmt]
        urls = re.findall(rx, self.src, re.X)
        # decode urls
        if decode in ("xml", "*"):
            urls = [xmlunescape(url) for url in urls]
        if decode in ("json", "*"):
            urls = [url.replace("\\/", "/") for url in urls]
        # only uniques
        return list(set(urls))


    # Only look out for URLs, not local file paths, nor titles
    extr_urls = (
       ("pls",  (r"(?im) ^ \s*File\d* \s*=\s* (\w+://[^\s]+) ", None)),
       ("m3u",  (r" (?m) ^( \w+:// [^#\n]+ )", None)),
       ("xspf", (r" (?x) <location> (\w+://[^<>\s]+) </location> ", "xml")),
       ("asx",  (r" (?x) <ref \b[^>]+\b href \s*=\s* [\'\"] (\w+://[^\s\"\']+) [\'\"] ", "xml")),
       ("smil", (r" (?x) <(?:audio|video|media)\b [^>]+ \b src \s*=\s* [^\"\']? \s* (\w+://[^\"\'\s]+) ", "xml")),
       ("jspf", (r" (?x) \"location\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "json")),
       ("jamj", (r" (?x) \"audio\" \s*:\s* \"(\w+:\\?/\\?/[^\"\s]+)\" ", "json")),
       ("json", (r" (?x) \"url\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "json")),
       ("asf",  (r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ", "xml")),
       ("raw",  (r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ", "*")),

    )


# Save rows in one of the export formats.
#
# The export() version uses urls[]+row/title= as input, converts it into
# a list of rows{} beforehand.
#