Check-in [7149d92fe1]
Overview
Comment: | Updated Jamendo plugin audioformat and listformat descriptors. Attempted to use v3.0 API for playlist tracks. Still no playlist API endpoint. So using a separate track requests now. Implemented a new action/playlist_convert URL extractor, which shall henceforth be known as "jamj" (JamJibberish). Fixed XML url extraction in regex mode, trivial backslash deescaping for JSON formats; and fixed multiply URL bug by copying row{} dict during conversion. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
7149d92fe1ad67fa50c497d964fe5b07 |
User & Date: | mario on 2015-04-18 17:19:38 |
Other Links: | manifest | tags |
Context
2015-04-18
| ||
20:37 | Guard appstate_init channels.current setting for absent plugins. check-in: 24fb9b895e user: mario tags: trunk | |
17:19 | Updated Jamendo plugin audioformat and listformat descriptors. Attempted to use v3.0 API for playlist tracks. Still no playlist API endpoint. So using a separate track requests now. Implemented a new action/playlist_convert URL extractor, which shall henceforth be known as "jamj" (JamJibberish). Fixed XML url extraction in regex mode, trivial backslash deescaping for JSON formats; and fixed multiply URL bug by copying row{} dict during conversion. check-in: 7149d92fe1 user: mario tags: trunk | |
17:16 | Removed dnd code snippets. check-in: 1e268b6422 user: mario tags: trunk | |
Changes
Modified Makefile from [9e648740b2] to [75cd4eda77].
1 2 3 4 5 6 7 8 9 | # Requires # ยท http://fossil.include-once.org/versionnum/ # ยท http://fossil.include-once.org/xpm/ SHELL := /bin/bash #(for brace expansion) NAME := streamtuner2 VERSION := $(shell version get:plugin st2.py || echo 2.1dev) DEST := /usr/share/streamtuner2 INST := install -m 644 | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | # Requires # ยท http://fossil.include-once.org/versionnum/ # ยท http://fossil.include-once.org/xpm/ SHELL := /bin/bash #(for brace expansion) NAME := streamtuner2 VERSION := $(shell version get:plugin st2.py || echo 2.1dev) DEST := /usr/share/streamtuner2 INST := install -m 644 PACK := xpm DEPS := -n $(NAME) -d python -d python-pyquery -d python-gtk2 -d python-requests -d python-keybinder OPTS := -s src -u packfile,man,fixperms -f --prefix=$(DEST) --deb-compression xz --rpm-compression xz --exe-autoextract # targets .PHONY: bin all: gtk3 #(most used) pack: all ver docs xpm src |
︙ | ︙ | |||
47 48 49 50 51 52 53 | tar: $(PACK) $(OPTS) $(DEPS) -t $@ -p "$(NAME)-VERSION.bin.txz" st2.py exe: $(PACK) $(OPTS) $(DEPS) -t $@ -p "$(NAME)-VERSION.exe" st2.py pyz: #@BUG: relative package references leave a /tmp/doc/ folder $(PACK) -u packfile -s src -t zip --zip-shebang "/usr/bin/env python" \ | | | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | tar: $(PACK) $(OPTS) $(DEPS) -t $@ -p "$(NAME)-VERSION.bin.txz" st2.py exe: $(PACK) $(OPTS) $(DEPS) -t $@ -p "$(NAME)-VERSION.exe" st2.py pyz: #@BUG: relative package references leave a /tmp/doc/ folder $(PACK) -u packfile -s src -t zip --zip-shebang "/usr/bin/env python" \ -f -p "$(NAME)-$(VERSION).pyz" --prefix=./ .zip.py st2.py src: cd .. && pax -wvJf streamtuner2/streamtuner2-$(VERSION).src.txz \ streamtuner2/*.{py,png,svg,desktop} streamtuner2/channels/*.{py,png} \ streamtuner2/{bundle/,help/,gtk,NEWS,READ,PACK,PKG,CRED,Make,bin,.zip}* # test .deb contents check: |
︙ | ︙ |
Modified action.py from [f8a0134421] to [fc5074bffb].
︙ | ︙ | |||
32 33 34 35 36 37 38 39 40 41 42 43 44 45 | import os from ahttp import fix_url as http_fix_url, session from config import conf, __print__ as debug, dbg import platform import copy import json from datetime import datetime # Coupling to main window # main = None | > | 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import os from ahttp import fix_url as http_fix_url, session from config import conf, __print__ as debug, dbg import platform import copy import json from datetime import datetime from xml.sax.saxutils import escape as xmlentities, unescape as xmlunescape # Coupling to main window # main = None |
︙ | ︙ | |||
96 97 98 99 100 101 102 | ("asx" , r""" <asx\b """), ("smil", r""" <smil[^>]*> .* <seq> """), ("html", r""" (?i)<(audio|video)\b[^>]+\bsrc\s*=\s*["']?https?:// """), ("wpl", r""" <\?wpl \s+ version="1\.0" \s* \?> """), ("b4s", r""" <WinampXML> """), # http://gonze.com/playlists/playlist-format-survey.html ("jspf", r""" ^ \s* \{ \s* "playlist": \s* \{ """), ("asf", r""" ^ \[Reference\] .*? ^Ref\d+= """), | | > | 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | ("asx" , r""" <asx\b """), ("smil", r""" <smil[^>]*> .* <seq> """), ("html", r""" (?i)<(audio|video)\b[^>]+\bsrc\s*=\s*["']?https?:// """), ("wpl", r""" <\?wpl \s+ version="1\.0" \s* \?> """), ("b4s", r""" <WinampXML> """), # http://gonze.com/playlists/playlist-format-survey.html ("jspf", r""" ^ \s* \{ \s* "playlist": \s* \{ """), ("asf", r""" ^ \[Reference\] .*? ^Ref\d+= """), ("json", r""" "url": \s* "\w+:\\?/\\?/ """), ("jamj", r""" "audio": \s* "\w+:\\?/\\?/ """), ("gvp", r""" ^gvp_version:1\.\d+$ """), ("href", r""" .* """), ] # Exec wrapper |
︙ | ︙ | |||
231 232 233 234 235 236 237 | break # with `probe` set # Check ambiguity (except pseudo extension) if len(set([source, mime, probe])) > 1: debug(dbg.ERR, "Possible playlist format mismatch:", (source, mime, probe, ext)) # Extract URLs from content | | | 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | break # with `probe` set # Check ambiguity (except pseudo extension) if len(set([source, mime, probe])) > 1: debug(dbg.ERR, "Possible playlist format mismatch:", (source, mime, probe, ext)) # Extract URLs from content for fmt in ["pls", "xspf", "asx", "smil", "jspf", "m3u", "json", "asf", "jamj", "raw"]: if not urls and fmt in (source, mime, probe, ext, "raw"): urls = extract_playlist(cnt).format(fmt) debug(dbg.DATA, "conversion from:", source, " with extractor:", fmt, "got URLs=", urls) # Return original, or asis for srv targets if not urls: return [url] |
︙ | ︙ | |||
295 296 297 298 299 300 301 | src = "" def __init__(self, text): self.src = text # Extract only URLs from given source type def format(self, fmt): debug(dbg.DATA, "input regex:", fmt, len(self.src)) | > | > > > > > > > > | 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 | src = "" def __init__(self, text): self.src = text # Extract only URLs from given source type def format(self, fmt): debug(dbg.DATA, "input regex:", fmt, len(self.src)) # regex urls = re.findall(self.extr_urls[fmt], self.src, re.X) # xml entities urls = [xmlunescape(url) for url in urls] # json escaping urls = [url.replace("\\/", "/") for url in urls] # uniques urls = list(set(urls)) return urls # Only look out for URLs, not local file paths extr_urls = { "pls": r"(?im) ^ \s*File\d* \s*=\s* (\w+://[^\s]+) ", "m3u": r" (?m) ^( \w+:// [^#\n]+ )", "xspf": r" (?x) <location> (\w+://[^<>\s]+) </location> ", "asx": r" (?x) <ref \b[^>]+\b href \s*=\s* [\'\"] (\w+://[^\s\"\']+) [\'\"] ", "smil": r" (?x) <(?:audio|video|media)\b [^>]+ \b src \s*=\s* [^\"\']? \s* (\w+://[^\"\'\s]+) ", "jspf": r" (?x) \"location\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "jamj": r" (?x) \"audio\" \s*:\s* \"(\w+:\\?/\\?/[^\"\s]+)\" ", "json": r" (?x) \"url\" \s*:\s* \"(\w+://[^\"\s]+)\" ", "asf": r" (?m) ^ \s*Ref\d+ = (\w+://[^\s]+) ", "raw": r" (?i) ( [\w+]+:// [^\s\"\'\>\#]+ ) ", } # Save rows in one of the export formats. |
︙ | ︙ | |||
356 357 358 359 360 361 362 363 364 365 366 367 368 369 | # Expand contained stream urls if not self.source in ("srv", "raw", "asis"): new_rows = [] for i,row in enumerate(rows): # Preferrably convert to direct server addresses for url in convert_playlist(row["url"], self.source, "srv", local_file=False): row["url"] = url new_rows.append(row) # Or just allow one stream per station in a playlist entry if not self.multiply: break rows = new_rows | > | 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 | # Expand contained stream urls if not self.source in ("srv", "raw", "asis"): new_rows = [] for i,row in enumerate(rows): # Preferrably convert to direct server addresses for url in convert_playlist(row["url"], self.source, "srv", local_file=False): row = dict(row.items()) row["url"] = url new_rows.append(row) # Or just allow one stream per station in a playlist entry if not self.multiply: break rows = new_rows |
︙ | ︙ | |||
439 440 441 442 443 444 445 | for row in rows: if row.get("url"): txt += """\t\t<audio src="%s"/>\n""" % row["url"] txt += """\t</seq>\n</body>\n</smil>\n""" return txt | < < < < < < < | 451 452 453 454 455 456 457 458 459 460 461 462 463 464 | for row in rows: if row.get("url"): txt += """\t\t<audio src="%s"/>\n""" % row["url"] txt += """\t</seq>\n</body>\n</smil>\n""" return txt # Generate filename for temporary .m3u, if possible with unique id def tmp_fn(pls, ext="m3u"): # use shoutcast unique stream id if available stream_id = re.search("http://.+?/.*?(\d+)", pls, re.M) stream_id = stream_id and stream_id.group(1) or "XXXXXX" |
︙ | ︙ |
Modified channels/jamendo.py from [55defe1856] to [13bc7ee950].
︙ | ︙ | |||
53 54 55 56 57 58 59 60 61 62 63 64 65 66 | # Seem to resolve to OGG Vorbis each. # class jamendo (ChannelPlugin): # control flags has_search = True base = "http://www.jamendo.com/en/" listformat = "srv" api_base = "http://api.jamendo.com/v3.0/" cid = "49daa4f5" categories = [] titles = dict( title="Title", playing="Album/Artist/User", bitrate=False, listeners=False ) | > | 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | # Seem to resolve to OGG Vorbis each. # class jamendo (ChannelPlugin): # control flags has_search = True base = "http://www.jamendo.com/en/" audioformat = "ogg" listformat = "srv" api_base = "http://api.jamendo.com/v3.0/" cid = "49daa4f5" categories = [] titles = dict( title="Title", playing="Album/Artist/User", bitrate=False, listeners=False ) |
︙ | ︙ | |||
256 257 258 259 260 261 262 | # retrieve category or search def update_streams(self, cat, search=None): entries = [] fmt = self.stream_mime(conf.jamendo_stream_format) | | | | | > > | > > > > | > | > | 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 | # retrieve category or search def update_streams(self, cat, search=None): entries = [] fmt = self.stream_mime(conf.jamendo_stream_format) # Static list of Radios if cat == "radios": for radio in ["BestOf", "Pop", "Rock", "Lounge", "Electro", "HipHop", "World", "Jazz", "Metal", "Soundtrack", "Relaxation", "Classical"]: entries.append({ "genre": radio, "title": radio, "url": "http://streaming.radionomy.com/Jam" + radio, # optional +".m3u" "playing": "various artists", "format": "audio/mpeg", "homepage": "http://www.jamendo.com/en/radios", "img": "http://imgjam1.jamendo.com/new_jamendo_radios/%s30.jpg" % radio.lower(), }) # Playlist elif cat == "playlists": for e in self.api(method="playlists", order="creationdate_desc"): entries.append({ "title": e["name"], "playing": e["user_name"], "homepage": e["shareurl"], "extra": e["creationdate"], "format": "audio/mpeg", #"listformat": "xspf", # deprecated #"url": "http://api.jamendo.com/get2/stream/track/xspf/?playlist_id=%s&n=all&order=random&from=app-%s" % (e["id"], self.cid), #"listformat": "href", # raw ZIP redirect #"url": "http://api.jamendo.com/v3.0/playlists/file?client_id={}&audioformat=mp32&id={}".format(self.cid, e["id"]), #"listformat": "href", # raw ZIP direct #"url": e["zip"], "listformat": "jamj", "url": "http://api.jamendo.com/v3.0/playlists/tracks?client_id={}&audioformat=mp32&id={}".format(self.cid, e["id"]), }) # Albums elif cat in ["albums", "newest"]: if cat == "albums": order = "popularity_week" else: order = "releasedate_desc" for e in self.api(method = "albums/musicinfo", order = order, include = "musicinfo"): entries.append({ "genre": " ".join(e["musicinfo"]["tags"]), "title": e["name"], "playing": e["artist_name"], "img": e["image"], "homepage": e["shareurl"], #"url": "http://api.jamendo.com/v3.0/playlists/file?client_id=%s&id=%s" % (self.cid, e["id"]), "url": "http://api.jamendo.com/get2/stream/track/xspf/?album_id=%s&streamencoding=ogg2&n=all&from=app-%s" % (e["id"], self.cid), "format": "audio/ogg", "listformat": "xspf", }) # Genre list, or Search else: if cat: data = self.api(method = "tracks", order = "popularity_week", include = "musicinfo", fuzzytags = cat, audioformat = conf.jamendo_stream_format) |
︙ | ︙ | |||
321 322 323 324 325 326 327 | "extra": ", ".join(e["musicinfo"]["tags"]["vartags"]), "title": e["name"], "playing": e["album_name"] + " / " + e["artist_name"], "img": e["album_image"], "homepage": e["shareurl"], #"url": e["audio"], "url": "http://storage-new.newjamendo.com/?trackid=%s&format=ogg2&u=0&from=app-%s" % (e["id"], self.cid), | | > | 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 | "extra": ", ".join(e["musicinfo"]["tags"]["vartags"]), "title": e["name"], "playing": e["album_name"] + " / " + e["artist_name"], "img": e["album_image"], "homepage": e["shareurl"], #"url": e["audio"], "url": "http://storage-new.newjamendo.com/?trackid=%s&format=ogg2&u=0&from=app-%s" % (e["id"], self.cid), "format": fmt, "listformat": "srv", }) # done return entries # Collect data sets from Jamendo API |
︙ | ︙ | |||
361 362 363 364 365 366 367 | # audio/* def stream_mime(self, name): map = { "ogg": "audio/ogg", "ogg2": "audio/ogg", "mp3": "audio/mpeg", "mp31": "audio/mpeg", "mp32": "audio/mpeg", "flac": "audio/flac" } | < | < < | 371 372 373 374 375 376 377 378 379 | # audio/* def stream_mime(self, name): map = { "ogg": "audio/ogg", "ogg2": "audio/ogg", "mp3": "audio/mpeg", "mp31": "audio/mpeg", "mp32": "audio/mpeg", "flac": "audio/flac" } return map.get(name) or map["mp3"] |