Artifact ab1b538656297a743f30497b9d155091cb679ee9:

• File channels/__init__.py — part of check-in [f9c725e90b] at 2020-05-10 15:17:41 on branch trunk — Adapt Xiph plugin for BETA/dir-test.xoph.org rollout. It's fairly terrible: homepages gone, bitrate unavailable, case-sensitive category segregration. On the upside: direct streaming server urls.

  The 'cache' mode is likely broken soon, since the experimental JSON API is gone.

  For now using `.title()` on /genre/{} search. Adapted guess_format to recognize AAC. Subtitle is used in lieu of On Air: text for playing= (user: mario, size: 32629) [annotate] [blame] [check-ins using]

# encoding: UTF-8
# api: streamtuner2
# type: class
# category: ui
# title: Channel plugins
# description: Base implementation for channels and feature plugins
# version: 1.7
# license: public domain
# author: mario
# url: http://fossil.include-once.org/streamtuner2/
# pack:
#    *.py
# config: -
# priority: core
#
# GenericChannel implements the basic GUI functions and defines
# the default channel data structure. It implements fallback logic
# for all other channel implementations. Only `bookmarks` uses it
# directly.
#
# All other plugins don't have a pre-defined Notebook tab in the
# GtkBuilder description. They derive from ChannelPlugins therefore,
# which constructs and registers the required gtk widgets manually.


import gtk
from uikit import uikit, ver as gtk_ver
from config import *
from compat2and3 import *
import ahttp
import action
import os.path
import xml.sax.saxutils
import re
import copy
import inspect


# Only export plugin classes and a few utility functions
__all__ = [
    "FeaturePlugin", "GenericChannel", "ChannelPlugin", "use_rx", "mime_fmt",
    "stub_parent", "entity_decode", "strip_tags", "nl", "unhtml", "to_int"
]
__path__.insert(0, conf.dir + "/plugins")



# Base class for plugins (features or channels)
class FeaturePlugin(object):

    # plugin meta and object references
    module = ""
    meta = { "config": [] }
    parent = None     # main window

    # minimum setup for ChannelPlugins and feature hooks
    def __init__(self, parent, *k, **kw):

        # set up meta infos
        self.parent = parent
        self.module = self.__class__.__name__
        self.meta = plugin_meta(src = inspect.getcomments(inspect.getmodule(self)))
        self.config = self.meta.get("config", [])
        self.title = self.meta.get("title", self.module)

        # add default options values to config.conf.* dict
        conf.add_plugin_defaults(self.meta, self.module)

        # implicit action handler registration
        if hasattr(self, "resolve_urn"):
            action.handler["urn:%s" % self.module] = self.resolve_urn

        # secondary init function
        self.init2(parent)

    # optionally to be overriden by plugins (run after base __init__)
    def init2(self, parent, *k, **kw):
        pass

    # Statusbar stub (defers to parent/main window, if in GUI mode)
    def status(self, *args, **kw):
        if self.parent:
            self.parent.status(*args, **kw)
        else:
            log.INFO("status():", *v)
        
    # Statusbar with highlighting and default icon
    def warn(self, text, *args, **kw):
        if isinstance(text, (str, unicode)) and "status_color" in conf:
            text = "<span background='%s'>%s</span>" % (conf.status_color, text)
            kw["markup"] = 1
        if not "icon" in kw:
            kw["icon"] = "gtk-dialog-warning"
        self.status(text, *args, **kw)



# Generic channel module
class GenericChannel(FeaturePlugin):

    # control attributes
    meta = { "config": [] }
    base_url = ""
    listformat = "pls"
    audioformat = "audio/mpeg" # fallback value
    has_search = False

    # Categories
    categories = []   # Category names or subcategory groups in [] lists
    catmap = {}       # Map category names to channel/service-internal ids
    shown = None      # Just a state flag for .first_show() now

    # Stream list
    streams = {}      # Station list dict, associates each genre to a list of stream rows
    
    # Gtk widgets
    gtk_list = None   # Gtk widget for station treeview
    gtk_cat = None    # Gtk widget for category columns
    ls = None         # ListStore for station treeview
    rowmap = None     # Preserve streams-datamap
    pix_entry = None  # ListStore entry that contains favicon
    img_resize = None  # Rescale `img` references to icon size
    fixed_size = [24,24]  # Default height+width for favicons

    # mapping of stream{} data into gtk treeview/treestore representation
    datamap = [
       # coltitle   width	[ datasrc key, type, renderer, attrs ]	[cellrenderer2], ...
       ["",		20,	["state",	str,  "pixbuf",	{}],	],
       ["Genre",	65,	['genre',	str,	"t",	{}],	],
       ["Station Title",275,	["title",	str,    "text",	{"strikethrough":11, "cell-background":12, "cell-background-set":13}],  ["favicon", gtk.gdk.Pixbuf, "pixbuf", {}], ],
       ["Now Playing",	185,	["playing",	str,	"text",	{"strikethrough":11}],	],                                                                             #{"width":20, "expand":False}
       ["Listeners", 	45,	["listeners",	int,	"t",	{"strikethrough":11}],	],
      #["Max",		45,	["max",		int,	"t",	{}],	],
       ["Bitrate",	35,	["bitrate",	int,	"t",	{}],	],
       ["Homepage",	160,	["homepage",	str,	"t",	{"underline":10}],	],
       [False,		25,	["url",		str,	"t",	{"strikethrough":11}],	],
       [False,		20,	["format",	str,	None,	{}],	],
       [False,		0,	["favourite",	bool,	None,	{}],	],
       [False,		0,	["deleted",	bool,	None,	{}],	],
       [False,		0,	["search_col",	str,	None,	{}],	],
       [False,		0,	["search_set",	bool,	None,	{}],	],
    ]
    rowmap = []   # [state,genre,title,...] field enumeration still needed separately
    titles = {}   # For easier adapting of column titles in datamap

    # For empty grouping / categories
    placeholder = [dict(state="gtk-info", genre="./.", title="Subcategory placeholder", playing="./.", url="none:", listeners=0, bitrate=0, homepage="")]
    empty_stub = [dict(state="gtk-dialog-error", genre="./.", title="No categories found (website error)", playing="Try Channel→Reload Categories later..", url="none:", listeners=0, bitrate=0, homepage="")]
    nothing_found = [dict(state="gtk-no", genre="./.", title="No contents found on directory server", playing="Notice", listeners=0, bitrate=0)]
    
    # Title to homepage regex
    rx_www_url = re.compile("""(www(\.\w+[\w-]+){2,}|(\w+[\w-]+[ ]?\.)+(com|FM|net|org|de|PL|fr|uk))""", re.I)

    # Hooks for station list updating 
    prepare_filters = []      # run prior columns() display
    postprocess_filters = []  # called after update_streams()


    # Keep track of currently selected genre/category
    __current = None
    @property
    def current(self):
        return self.__current
    @current.setter
    def current(self, newcat):
        log.PROC("{}.current:={} ← from {}".format(self.module, newcat, [inspect.stack()[x][3] for x in range(1,4)]))
        self.__current = newcat
        return self.__current


    #--------------------------- initialization --------------------------------


    # constructor
    def __init__(self, parent=None):
    
        #self.streams = {}
        self.gtk_list = None
        self.gtk_cat = None
        
        # base init (meta infos, parent reference, init2)
        FeaturePlugin.__init__(self, parent)
        
        # Only if streamtuner2 is run in graphical mode        
        if (parent):
            # Update/display stream processors
            if not self.prepare_filters:
                self.prepare_filters += [
                    self.prepare_filter_icons,
                ]
            if not self.postprocess_filters:
                self.postprocess_filters += [
                    self.postprocess_filter_required_fields,
                    self.postprocess_filter_homepage,
                ]
            # Load cache, instantiate Gtk widgets
            self.cache()
            self.gui(parent)

        # Stub for ST2 main window / dispatcher
        else:
            self.parent = stub_parent(None)

        
    # initialize Gtk widgets / data objects
    def gui(self, parent):

        # save reference to main window/glade API
        self.parent = parent
        self.gtk_list = parent.get_widget(self.module+"_list")
        self.gtk_cat = parent.get_widget(self.module+"_cat")
        
        # last category, and prepare genre tree
        self.current = conf.state(self.module).get("current")
        self.display_categories()

        # update column names
        for field,title in list(self.titles.items()):
            self.update_datamap(field, title=title)
        
        # Initialize stations TreeView
        self.columns([])
        
        # add to main menu
        uikit.add_menu([parent.channelmenuitems], self.meta["title"], lambda w: parent.channel_switch_by_name(self.module) or 1)


    # Just wraps uikit.columns() to retain liststore, rowmap and pix_entry
    def columns(self, entries=None):
        self.ls, self.rowmap, self.pix_entry = uikit.columns(
            self.gtk_list, self.datamap, entries,
            show_favicons=True, fixed_size=self.fixed_size
        )
        # no longer using `conf.show_favicons`


        
    #--------------------- streams/model data accesss ---------------------------


    # Traverse category TreeModel to set current, expand parent nodes
    def select_current(self, name):
        log.UI("reselect .current category in treelist:", name)
        model = self.gtk_cat.get_model()
           # [Gtk3] Warning: g_object_ref_sink: assertion 'object->ref_count >= 1' failed
           # ERROR:../../gi/pygobject.c:688:pygobject_register_wrapper: assertion failed: (gself->obj->ref_count >= 1)
        iter = model.get_iter_first()
        self.iter_cats(name, model, iter)

    # Iterate over children to find current category
    def iter_cats(self, name, model, iter):
        while iter:
            val = model.get_value(iter, 0)
            if val == name:
                #log.UI("FOUND CATEGORY", name, "→select")
                self.gtk_cat.get_selection().select_iter(iter)
                self.gtk_cat.set_cursor(model.get_path(iter))
                self.gtk_cat.scroll_to_cell(model.get_path(iter), None)
                return True
            if model.iter_has_child(iter):
                found = self.iter_cats(name, model, model.iter_children(iter))
                if found:
                    self.gtk_cat.expand_row(model.get_path(iter), 0)
                    return True
            iter = model.iter_next(iter)
        
    # Selected category (current state from Gtk TreeModel)
    def currentcat(self):
        (model, iter) = self.gtk_cat.get_selection().get_selected()
        if (type(iter) == gtk.TreeIter):
            self.current = model.get_value(iter, 0)
        return self.current
        
    # Get list of stations in current category
    def stations(self):
        return self.streams.get(self.current, [])

    # Convert ListStore iter to row number
    def rowno(self):
        (model, iter) = self.model_iter()
        return model.get_path(iter)[0]

    # Return ListStore object and Iterator for currently selected row in gtk.TreeView station list
    def model_iter(self):
        return self.gtk_list.get_selection().get_selected()

    # Currently selected entry in stations list, return complete data dict
    def row(self):
        row = self.stations() [ self.rowno() ]
        # resolve stream url for some plugins
        if row.get("url") and row.get("url", "urn:x-streamtuner2:no").startswith("urn:"):
            self.status("Resolving actual stream URL for `%s`" % row["url"], timeout=2)
            # most urn: handlers update `row` - some return a new value - which is handled in action.run_fmt_url() however
            action.resolve_urn(row)  # row = ..() or row
        return row
        
    # Fetches a single varname from currently selected station entry
    def selected(self, name="url"):
        return self.row().get(name)
    
    # Inject status icon into currently selected row (used by main.bookmark() call)
    def row_icon(self, gtkIcon = gtk.STOCK_ABOUT):
        try:
            # Updates gtk_list store, set icon in current display.
            # Since it is used by bookmarks, would be reshown with next display() anyhow,
            # and there's no need to invalidate the ls cache, because that's referenced by model anyhow.
            (model,iter) = self.model_iter()
            model.set_value(iter, 0, gtkIcon)
        except Exception as e:
            log.ERR_UIKIT("Couldn't set row_icon()", e)

    

    #------------------------ base implementations -----------------------------


    # Read previous channel/stream data, if there's any
    def cache(self):
        # stream list
        cache = conf.load("cache/" + self.module)
        if (cache):
            self.streams = cache
        # categories
        cache = conf.load("cache/categories_" + self.module)
        if (cache):
            self.categories = cache
        # catmap (optional)
        cache = conf.load("cache/catmap_" + self.module)
        if (cache):
            self.catmap = cache
        pass

    # Store current streams data
    def save(self):
        conf.save("cache/" + self.module, self.streams, gz=1)

        
    # Create private copy of .datamap and modify entries (title= rewrites)
    def update_datamap(self, search="name", title=None):
        if self.datamap == GenericChannel.datamap:
            self.datamap = copy.deepcopy(self.datamap)
        for i,row in enumerate(self.datamap):
            if row[2][0] == search:
                row[0] = title


    # Reload current station list
    def reload(self):
        self.load(self.current, force=1)
    def switch(self):
        self.load(self.current, force=0)
    
    # Update streams pane if currently selected (used by bookmarks.links channel)
    def reload_if_current(self, category):
        if self.current == category:
            self.reload()


    # Called on switching genre/category, or loading a genre for the first time.
    # Either fetches new stream data, or displays list from cache.
    def load(self, category, force=False, y=None):

        # called to early
        if not category:
            log.ERR("load(None)")
            return
        self.current = category
        do_save = False

        # get data from cache or download
        if force or not category in self.streams:
            log.PROC("load", "update_streams")
            self.progress(None)
            self.status("Updating streams...")
            self.status(-0.1)
            if category == "empty":
                new_streams = self.empty_stub
            else:
                new_streams = self.update_streams(category)
  
            # Postprocess new list of streams (e.g. assert existing title and url)
            if new_streams:
                try:
                    new_streams = self.postprocess(new_streams)
                    do_save = True
                except Exception as e:
                    log.ERR("Updating new streams, postprocessing failed:", e)
  
                # don't lose forgotten streams
                if conf.retain_deleted:
                   self.streams[category] = new_streams + self.deleted_streams(new_streams, self.streams.get(category,[]))
                else:
                   self.streams[category] = new_streams
  
            else:
                # parse error
                self.status("Category parsed empty.")
                self.streams[category] = self.nothing_found
                log.INFO("Oooops, parser returned nothing for category " + category)
                
        # Update treeview/model (if category is still selected)
        if self.current == category:
            log.UI("load() → uikit.columns({}.streams[{}])".format(self.module, category), [inspect.stack()[x][3] for x in range(1,5)])
            uikit.do(self.columns, self.prepare(self.streams[category]))
            if y:
                uikit.do(self.gtk_list.scroll_to_point, 0, y + 1)   # scroll to previous position, +1 px, because
                # somehow Gtk.TreeView else stumbles over itself when scrolling to the same position the 2nd time
  
        # save in cache
        if do_save:
            self.save()

        # unset statusbar
        self.status()

        
    # Called occasionally (by some plugins) while updating station list
    def update_streams_partially_done(self, entries):
        if gtk_ver == 3 and not conf.nothreads:
            pass
        else:  # kills Gtk3 too easily
            uikit.do(self.columns, entries)
    # Prepare stream list for display (called immediataly before .columns() refreshing)
    def prepare(self, streams):
        for f in self.prepare_filters:
            map(f, streams)
        return streams

    # state icon: bookmark star, or deleted mark
    def prepare_filter_icons(self, row):
        if conf.show_bookmarks:
            # and "bookmarks" in self.parent.channels:
            row["favourite"] = self.parent.bookmarks.is_in(row.get("url", "file:///tmp/none"))
            # this should really go into bookmarks plugin itself,
            # disadvantage: would decelerate processing loop further
        if not row.get("state"):
            if row.get("url", "").find(" ") > 0:
                row["state"] = gtk.STOCK_UNINDENT
            if row.get("favourite"):
                row["state"] = gtk.STOCK_ABOUT
            if row.get("deleted"):
                row["state"] = gtk.STOCK_DELETE


    # Stream list cleanup - invoked directly after reload(),
    # callbacks can remove entries, or just update fields.
    def postprocess(self, streams):
        for f in self.postprocess_filters:
            streams = [row for row in streams if f(row, self)]
        return streams

    # Filter entries without title or url
    def postprocess_filter_required_fields(self, row, channel):
        return not len(set(["", None]) & set([row.get("title"), row.get("url")]))

    # Deduce homepage URLs from title
    # by looking for www.xyz.com domain names
    def postprocess_filter_homepage(self, row, channel):
        if not row.get("homepage"):
            url = self.rx_www_url.search(row.get("title", ""))
            if url:
                url = url.group(0).lower().replace(" ", "")
                url = (url if url.find("www.") == 0 else "www."+url)
                row["homepage"] = ahttp.fix_url(url)
        return True


    # Finds differences in new/old streamlist, marks deleted with flag
    def deleted_streams(self, new, old):
        diff = []
        new = [row.get("url","http://example.com/") for row in new]
        for row in old:
            if ("url" in row and (row.get("url") not in new)):
                row["deleted"] = 1
                diff.append(row)
        return diff

    # sets updating progress bar for .update_streams or .reload_categores
    def progress(self, max, i=None):
        if not max:
            self.status()
            self.status(1.0)
            self.progress_state = 1
            return
        if isinstance(max, (list, dict)):
            max = len(max)
        if not i:
            i = self.progress_state
        self.status( float(int(i)) / (float(int(max)) + 1.5) )
        self.progress_state = self.progress_state + 1
    progress_state = 1

        
    # Display .current category, once notebook/channel tab is first opened
    def first_show(self):

        # Already processed
        if (self.shown == 55555):
            return
        log.PROC(self.module, "→ first_show()", ", current=", self.current, ", categories=", len(self.categories))
    
        # if category tree is empty, initialize it
        if not self.categories:
            log.PROC(self.module, "→ first_show() → reload_categories()");
            try:
                self.reload_categories()
            except:
                log.ERR("HTTP error or extraction failure.")
                self.categories = ["empty"]
            self.display_categories()

        # Select first category
        if not self.current:
            log.STAT(self.module, "→ first_show(); use first category as current =", self.current)
            self.current = self.str_from_struct(self.categories) or None

        # put selection/cursor on last position
        if True:
            uikit.do(self.select_current, self.current)

        # Show current category in any case
        log.UI(self.module, "→ first_show(); station list → load(", self.current, ")")
        self.load(self.current)
            
        # Invoke only once
        self.shown = 55555


    # Retrieve first list value, or key from dict (-- used to get first category on init)
    def str_from_struct(self, d):
        if isinstance(d, (str)):
            return d
        elif isinstance(d, (dict)):
            return self.str_from_struct(d.keys()) or self.str_from_struct(d.values())
        elif isinstance(d, (list, tuple)):
            return d[0] if len(d) else None


    # Update categories, save, and display                
    def reload_categories(self):
    
        # get data and save
        self.update_categories()
        if self.categories:
            conf.save("cache/categories_"+self.module, self.categories)
        if self.catmap:
            conf.save("cache/catmap_" + self.module, self.catmap);

        # display outside of this non-main thread
        uikit.do(self.display_categories)


    # Refresh category treeview
    def display_categories(self):
        log.UI("{}.display_categories(), uikit.tree(#{}), expand_all(#<20), select_current(={})".format(self.module, len(self.categories), self.current))
    
        # rebuild gtk.TreeView
        uikit.tree(self.gtk_cat, self.categories, title="Category", icon=gtk.STOCK_OPEN)

        # if it's a short list of categories, there's probably subfolders
        if len(self.categories) < 20:
            self.gtk_cat.expand_all()
            
        # Select last .current or any first element
        if self.current:
            uikit.do(self.select_current, self.current)
            #self.currentcat()
        #else: self.gtk_cat.get_selection().select_path("0") #set_cursor

            

    
    # Insert/append new station rows - used by importing/drag'n'drop plugins
    def insert_rows(self, rows, y=None):
        streams = self.streams[self.current]
        tv = self.gtk_list
        
        # Inserting at correct row requires deducing index from dnd `y` position
        if y is not None:
            i_pos = (tv.get_path_at_pos(10, y) or [[len(streams) + 1]])[0][0]
            for row in rows:
                streams.insert(i_pos - 1, row)
                i_pos = i_pos + 1
        else:
            streams += rows

        # Now appending to the liststore directly would be even nicer
        y = int(tv.get_vadjustment().get_value())
        self.load(self.current, y=y)





    #--------------------------- actions ---------------------------------

    # Invoke action.play() for current station.
    # Can be overridden to provide channel-specific "play" alternative
    def play(self):
        row = self.row()
        if row and "url" in row:
            # playlist and audio type
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            # invoke audio player
            action.play(row, audioformat, listformat)
        elif row.get("homepage") and row.get("format") == "text/html":
            action.browser(row["homepage"])
        else:
            self.status("No station selected for playing.")
        return row

    # Start streamripper/youtube-dl/etc
    def record(self):
        row = self.row()
        if row and "url" in row:
            audioformat = row.get("format", self.audioformat)
            listformat = row.get("listformat", self.listformat)
            action.record(row, audioformat, listformat)
        return row



    


# channel plugin without glade-pre-defined notebook tab
#
class ChannelPlugin(GenericChannel):

    module = "abstract"

    def gui(self, parent):

        if not parent:
            return

        module = self.__class__.__name__
        # two panes
        vbox = gtk.HPaned()
        vbox.show()
        # category treeview
        sw1 = gtk.ScrolledWindow()
        sw1.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
        sw1.set_property("width_request", 150)
        sw1.show()
        tv1 = gtk.TreeView()
        tv1.set_property("width_request", 75)
        tv1.set_property("enable_tree_lines", True)
        uikit.tree_column(tv1, "Category")
        tv1.connect("button_release_event", parent.on_category_clicked)
        tv1.show()
        sw1.add(tv1)
        vbox.pack1(sw1, resize=False, shrink=True)
        # stream list
        sw2 = gtk.ScrolledWindow()
        sw2.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
        sw2.show()
        tv2 = gtk.TreeView()
        tv2.set_property("width_request", 200)
        tv2.set_property("enable_tree_lines", True)
        tv2.connect("row_activated", parent.on_play_clicked)
        tv2.show()
        sw2.add(tv2)
        vbox.pack2(sw2, resize=True, shrink=True)

        # prepare label
        pixbuf = None
        if "png" in self.meta:
            pixbuf = uikit.pixbuf(self.meta["png"])
        else:
            png = get_data("channels/" + self.module + ".png")
            pixbuf = uikit.pixbuf(png)
        if pixbuf:
            icon = gtk.image_new_from_pixbuf(pixbuf)
        else:
            icon = gtk.image_new_from_stock(gtk.STOCK_DIRECTORY, size=1)
        label = gtk.HBox()
        label.pack_start(icon, expand=False, fill=True)
        l = gtk.Label(self.meta.get("title", self.module))
        if self.meta.get("color"):
            l = uikit.bg(l, self.meta["color"])
        label.pack_start(l, expand=True, fill=True)
            
        # pack it into an event container to catch double-clicks
        ev_label = gtk.EventBox()
        ev_label.add(label)
        ev_label.connect('event', parent.on_homepage_channel_clicked)
        plain_label = gtk.Label(self.module)

        # to widgets
        self.gtk_cat = tv1
        parent.widgets[module + "_cat"] = tv1
        self.gtk_list = tv2
        parent.widgets[module + "_list"] = tv2
        ev_label.show_all()
        vbox.show_all()
        parent.widgets["v_" + module] = vbox
        parent.widgets["c_" + module] = ev_label
        tv2.connect('button-press-event', parent.station_context_menu)


        # try to initialize superclass now, before adding to channel tabs
        GenericChannel.gui(self, parent)

        # add notebook tab
        tab = parent.notebook_channels.insert_page_menu(vbox, ev_label, plain_label, -1)
        parent.notebook_channels.set_tab_reorderable(vbox, True)





# WORKAROUND for direct channel module imports,
# eases instantiations without GUI a little,
# reducing module dependencies (conf. / ahttp. / channels. / parent.) would be better
def stub_parent(object):
    def __setattr__(self, name, value):
        pass
    def __getattr__(self, name):
        return lambda *x: None
    def status(self, *x):
        pass


# Decorator
def use_rx(func):
    def try_both(*args, **kwargs):
        for method, use_rx in [("RX", not conf.pyquery), ("PQ", conf.pyquery)]:
            try:
                log.STAT(method)
                return func(*args, use_rx=not conf.pyquery, **kwargs)
            except Exception as e:
                log.ERR("{} extraction failed:".format(method), e)
                continue
        return []
    return try_both



#---------------- utility functions -------------------
# Used by raw page extraction in channel modules


# Strip html <tags> from string
def strip_tags(s):
    return re.sub("<.+?>", "", s)

# remove SGML/XML entities
def entity_decode(str):
    return re.sub('&(#?(x?))(\w+);', _entity, str)
def _entity(sym):
    num, hex, name = sym.groups()
    if hex:
        return unichr(int(name, base=16))
    elif num:
        return unichr(int(name))
    else:
        return unichr(htmlentitydefs_n2cp[name])

# Nay for the Py3 raft
htmlentitydefs_n2cp = {'aring': 229, 'gt': 62, 'sup': 8835, 'Ntilde': 209, 'upsih': 978, 'Yacute': 221,
'Atilde': 195, 'radic': 8730, 'otimes': 8855, 'aelig': 230, 'Psi': 936, 'Uuml': 220, 'Epsilon': 917, 'Icirc':
206, 'Eacute': 201, 'Lambda': 923, 'Prime': 8243, 'Kappa': 922, 'sigmaf': 962, 'lrm': 8206, 'cedil': 184,
'kappa': 954, 'AElig': 198, 'prime': 8242, 'Tau': 932, 'lceil': 8968, 'dArr': 8659, 'ge': 8805, 'sdot': 8901,
'lfloor': 8970, 'lArr': 8656, 'Auml': 196, 'brvbar': 166, 'Otilde': 213, 'Theta': 920, 'Pi': 928, 'OElig': 338,
'Scaron': 352, 'egrave': 232, 'sub': 8834, 'iexcl': 161, 'ordf': 170, 'sum': 8721, 'ntilde': 241, 'atilde':
227, 'theta': 952, 'nsub': 8836, 'hArr': 8660, 'Oslash': 216, 'THORN': 222, 'yuml': 255, 'Mu': 924, 'thinsp':
8201, 'ecirc': 234, 'bdquo': 8222, 'Aring': 197, 'nabla': 8711, 'permil': 8240, 'Ugrave': 217, 'eta': 951,
'Agrave': 192, 'forall': 8704, 'eth': 240, 'rceil': 8969, 'iuml': 239, 'Egrave': 200, 'divide': 247, 'igrave':
236, 'otilde': 245, 'pound': 163, 'frasl': 8260, 'ETH': 208, 'lowast': 8727, 'chi': 967, 'Aacute': 193, 'cent':
162, 'Beta': 914, 'perp': 8869, 'there4': 8756, 'pi': 960, 'empty': 8709, 'euml': 235, 'notin': 8713, 'uuml':
252, 'icirc': 238, 'bull': 8226, 'upsilon': 965, 'Oacute': 211, 'ensp': 8194, 'ccedil': 231, 'cap': 8745, 'mu':
956, 'deg': 176, 'tau': 964, 'emsp': 8195, 'hellip': 8230, 'ucirc': 251, 'ugrave': 249, 'cong': 8773, 'Iota':
921, 'quot': 34, 'rarr': 8594, 'Rho': 929, 'uacute': 250, 'acirc': 226, 'sim': 8764, 'phi': 966, 'diams': 9830,
'Euml': 203, 'Ccedil': 199, 'Eta': 919, 'Gamma': 915, 'euro': 8364, 'thetasym': 977, 'sect': 167, 'ldquo':
8220, 'hearts': 9829, 'oacute': 243, 'zwnj': 8204, 'yen': 165, 'ograve': 242, 'Chi': 935, 'trade': 8482, 'xi':
958, 'nbsp': 160, 'tilde': 732, 'lsaquo': 8249, 'oelig': 339, 'equiv': 8801, 'le': 8804, 'auml': 228, 'cup':
8746, 'Yuml': 376, 'lt': 60, 'Upsilon': 933, 'ndash': 8211, 'yacute': 253, 'real': 8476, 'psi': 968, 'rsaquo':
8250, 'darr': 8595, 'Alpha': 913, 'not': 172, 'amp': 38, 'oslash': 248, 'acute': 180, 'zwj': 8205, 'laquo':
171, 'rdquo': 8221, 'Igrave': 204, 'micro': 181, 'shy': 173, 'supe': 8839, 'szlig': 223, 'clubs': 9827,
'agrave': 224, 'Ocirc': 212, 'harr': 8596, 'larr': 8592, 'frac12': 189, 'prop': 8733, 'circ': 710, 'ocirc':
244, 'asymp': 8776, 'uml': 168, 'prod': 8719, 'reg': 174, 'rlm': 8207, 'infin': 8734, 'Sigma': 931, 'mdash':
8212, 'uarr': 8593, 'times': 215, 'rArr': 8658, 'or': 8744, 'gamma': 947, 'lambda': 955, 'rang': 9002, 'sup3':
179, 'dagger': 8224, 'Ouml': 214, 'image': 8465, 'alefsym': 8501, 'sube': 8838, 'alpha': 945, 'Nu': 925,
'plusmn': 177, 'sup1': 185, 'sup2': 178, 'frac34': 190, 'oline': 8254, 'Delta': 916, 'loz': 9674, 'iota': 953,
'iacute': 237, 'para': 182, 'ordm': 186, 'epsilon': 949, 'weierp': 8472, 'part': 8706, 'delta': 948, 'omicron':
959, 'copy': 169, 'Iuml': 207, 'Xi': 926, 'Dagger': 8225, 'Ograve': 210, 'Ucirc': 219, 'scaron': 353, 'lsquo':
8216, 'isin': 8712, 'Zeta': 918, 'minus': 8722, 'and': 8743, 'ang': 8736, 'curren': 164, 'int': 8747, 'rfloor':
8971, 'crarr': 8629, 'exist': 8707, 'oplus': 8853, 'Acirc': 194, 'piv': 982, 'ni': 8715, 'Phi': 934, 'Iacute':
205, 'Uacute': 218, 'Omicron': 927, 'ne': 8800, 'iquest': 191, 'sbquo': 8218, 'Ecirc': 202, 'zeta': 950,
'Omega': 937, 'nu': 957, 'macr': 175, 'frac14': 188, 'aacute': 225, 'uArr': 8657, 'beta': 946, 'fnof': 402,
'rho': 961, 'eacute': 233, 'omega': 969, 'middot': 183, 'lang': 9001, 'spades': 9824, 'rsquo': 8217, 'thorn':
254, 'ouml': 246, 'raquo': 187, 'sigma': 963, 'apos': 39}


# Extracts integer from string
def to_int(s):
    i = re.findall("\d+", s) or [0]
    return int(i[0])

# Strip newlines
rx_spc = re.compile("\s+")
def nl(str):
    return rx_spc.sub(" ", str).strip()

# Combine html tag, escapes and whitespace cleanup
def unhtml(str):
    return nl(entity_decode(strip_tags(str)))


# Convert audio format nick/shortnames to mime types, e.g. "OGG" to "audio/ogg"
# (only used by few plugin meanwhile, could be merged with action. module now)
def mime_fmt(s):
    # clean string
    s = s.lower().strip()
    # rename
    map = {
        "audio/mp3":"audio/mpeg",  # Note the real mime type is /mpeg, but /mp3 is more understandable in the GUI
        "ogg":"ogg", "ogm":"ogg", "xiph":"ogg", "vorbis":"ogg", "vnd.xiph.vorbis":"ogg",
        "mp3":"mpeg", "mp":"mpeg", "mp2":"mpeg", "mpc":"mpeg", "mps":"mpeg",
        "aac+":"aac", "aacp":"aac", "aac&#43;":"aac",
        "realaudio":"x-pn-realaudio", "real":"x-pn-realaudio", "ra":"x-pn-realaudio", "ram":"x-pn-realaudio", "rm":"x-pn-realaudio",
        # yes, we do video
        "flv":"video/flv", "mp4":"video/mp4",
    }
    #map.update(action.listfmt_t)   # list type formats (.m3u .pls and .xspf)
    if map.get(s):
        s = map[s]
    # add prefix:
    if s.find("/") < 1:
        s = "audio/" + s
    #
    return s