Check-in [58d2981ca9]
Overview
Comment: | Fix homepage url issue / quote() checks for list first now. Restructure playlist extraction into ordered dict; use in conver_playlist as probe formats. Make xml/json url decoding explicit, prepare for custom extractors (e.g. real json or xml traversal, full row/title extraction). |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
58d2981ca982ec258e00b9bd955567ed |
User & Date: | mario on 2015-04-18 20:39:08 |
Other Links: | manifest | tags |
Context
2015-04-18
| ||
20:41 | Add [feeds] support for Jamendo. (Pretty much only albums, so track lookup works.) Better support for configured audio format/mime. Ogg Vorbis now works in albums and playlists. check-in: 90b1672010 user: mario tags: trunk | |
20:39 | Fix homepage url issue / quote() checks for list first now. Restructure playlist extraction into ordered dict; use in conver_playlist as probe formats. Make xml/json url decoding explicit, prepare for custom extractors (e.g. real json or xml traversal, full row/title extraction). check-in: 58d2981ca9 user: mario tags: trunk | |
20:37 | Guard appstate_init channels.current setting for absent plugins. check-in: 24fb9b895e user: mario tags: trunk | |
Changes
Modified action.py from [fc5074bffb] to [820487cd33].
︙ | ︙ | |||
53 54 55 56 57 58 59 60 61 62 63 64 65 66 | "video/x-ms-asf": "asx", "application/xspf+xml": "xspf", "*/*": "href", # "href" for unknown responses "url/direct": "srv", "url/youtube": "href", "url/http": "href", "audio/x-pn-realaudio": "ram", "application/smil": "smil", "application/vnd.ms-wpl":"smil", "audio/x-ms-wax": "asx", "video/x-ms-asf": "asx", "x-urn/st2-script": "script", # unused "application/x-shockwave-flash": "href", # fallback } | > | 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | "video/x-ms-asf": "asx", "application/xspf+xml": "xspf", "*/*": "href", # "href" for unknown responses "url/direct": "srv", "url/youtube": "href", "url/http": "href", "audio/x-pn-realaudio": "ram", "application/json": "json", "application/smil": "smil", "application/vnd.ms-wpl":"smil", "audio/x-ms-wax": "asx", "video/x-ms-asf": "asx", "x-urn/st2-script": "script", # unused "application/x-shockwave-flash": "href", # fallback } |
︙ | ︙ | |||
89 90 91 92 93 94 95 | srv = "(%srv | %d | %s) \\b", ) # Playlist format content probing (assert type) playlist_content_map = [ ("pls", r""" (?i)\[playlist\].*NumberOfEntries """), ("xspf", r""" <\?xml .* <playlist .* ((?i)http://xspf\.org)/ns/0/ """), | | | 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | srv = "(%srv | %d | %s) \\b", ) # Playlist format content probing (assert type) playlist_content_map = [ ("pls", r""" (?i)\[playlist\].*NumberOfEntries """), ("xspf", r""" <\?xml .* <playlist .* ((?i)http://xspf\.org)/ns/0/ """), ("m3u", r""" ^ \s* \#(EXT)?M3U """), ("asx" , r""" <asx\b """), ("smil", r""" <smil[^>]*> .* <seq> """), ("html", r""" (?i)<(audio|video)\b[^>]+\bsrc\s*=\s*["']?https?:// """), ("wpl", r""" <\?wpl \s+ version="1\.0" \s* \?> """), ("b4s", r""" <WinampXML> """), # http://gonze.com/playlists/playlist-format-survey.html ("jspf", r""" ^ \s* \{ \s* "playlist": \s* \{ """), ("asf", r""" ^ \[Reference\] .*? ^Ref\d+= """), |
︙ | ︙ | |||
117 118 119 120 121 122 123 124 125 126 127 128 129 130 | except: debug(dbg.ERR, "Command not found:", cmd) # Start web browser # def browser(url): bin = conf.play.get("url/http", "sensible-browser") run(bin + " " + quote(url)) # Open help browser, streamtuner2 pages # def help(*args): run("yelp /usr/share/doc/streamtuner2/help/") | > | 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | except: debug(dbg.ERR, "Command not found:", cmd) # Start web browser # def browser(url): bin = conf.play.get("url/http", "sensible-browser") print url run(bin + " " + quote(url)) # Open help browser, streamtuner2 pages # def help(*args): run("yelp /usr/share/doc/streamtuner2/help/") |
︙ | ︙ | |||
145 146 147 148 149 150 151 | cmd = interpol(cmd, url or row["url"], source, row) run(cmd) # OS shell command escaping # def quote(ins): | | | | | 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | cmd = interpol(cmd, url or row["url"], source, row) run(cmd) # OS shell command escaping # def quote(ins): if type(ins) is list: return " ".join(["%r" % str(s) for s in ins]) else: return "%r" % str(ins) # Convert e.g. "text/x-scpls" MIME types to just "pls" monikers # def listfmt(t = "pls"): return listfmt_t.get(t, t) # e.g. "pls" or still "text/x-unknown" |
︙ | ︙ | |||
230 231 232 233 234 235 236 | for probe,rx in playlist_content_map: if re.search(rx, cnt, re.X|re.S): probe = listfmt(probe) break # with `probe` set # Check ambiguity (except pseudo extension) if len(set([source, mime, probe])) > 1: | | | | 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 | for probe,rx in playlist_content_map: if re.search(rx, cnt, re.X|re.S): probe = listfmt(probe) break # with `probe` set # Check ambiguity (except pseudo extension) if len(set([source, mime, probe])) > 1: debug(dbg.ERR, "Possible playlist format mismatch:", "listformat={}, http_mime={}, rx_probe={}, ext={}".format(source, mime, probe, ext)) # Extract URLs from content for fmt in [id[0] for id in extract_playlist.extr_urls]: if not urls and fmt in (source, mime, probe, ext, "raw"): urls = extract_playlist(cnt).format(fmt) debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls) # Return original, or asis for srv targets if not urls: return [url] |
︙ | ︙ | |||
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | content = "\n".join(str.decode(errors='replace') for str in r.iter_lines()) return (mime, content) # Extract URLs from playlist formats: # class extract_playlist(object): # Content of playlist file src = "" def __init__(self, text): self.src = text # Extract only URLs from given source type def format(self, fmt): | > > > > | > > > > > | > | | > | | | | | < | | | | | | | | | | | | < > | 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 | content = "\n".join(str.decode(errors='replace') for str in r.iter_lines()) return (mime, content) # Extract URLs from playlist formats: # # It's entirely regex-based at the moment, because that's more # resilient against mailformed XSPF or JSON. # Needs proper extractors later for real playlist *imports*. # class extract_playlist(object): # Content of playlist file src = "" def __init__(self, text): self.src = text # Extract only URLs from given source type def format(self, fmt): debug(dbg.DATA, "input extractor/regex:", fmt, len(self.src)) # find extractor if fmt in dir(self): return self.__dict__[fmt]() # regex scheme rx, decode = dict(self.extr_urls)[fmt] urls = re.findall(rx, self.src, re.X) # decode urls if decode in ("xml", "*"): urls = [xmlunescape(url) for url in urls] if decode in ("json", "*"): urls = [url.replace("\\/", "/") for url in urls] # only uniques return list(set(urls)) # Only look out for URLs, not local file paths, nor titles extr_urls = ( ("pls", (r"(?im) ^ \s*File\d* \s*=\s* (\w+://[^\s]+) ", None)), ("m3u", (r" (?m) ^( \w+:// [^#\n]+ )", None)), ("xspf", (r" (?x) <location> (\w+://[^<>\s]+) </location> ", "xml")), ("asx", (r" (?x) <ref \b[^>]+\b href \s*=\s* [\'\"] (\w+://[^\s\"\']+) [\'\"] ", "xml")), ("smil", (r" (?x) <(?:audio|video|media)\b [^>]+ \b src \s*=\s* [^\"\']? \s* (\w+://[^\"\'\s]+) ", "xml")), ("jspf", (r" (?x) \"location\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "json")), ("jamj", (r" (?x) \"audio\" \s*:\s* \"(\w+:\\?/\\?/[^\"\s]+)\" ", "json")), ("json", (r" (?x) \"url\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "json")), ("asf", (r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ", "xml")), ("raw", (r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ", "*")), ) # Save rows in one of the export formats. # # The export() version uses urls[]+row/title= as input, converts it into # a list of rows{} beforehand. # |
︙ | ︙ |