# encoding: utf-8
# api: streamtuner2
# title: favicon download
# description: retrieves favicons for station homepages, plus utility code for display preparation
# title: Favicons
# description: Display station favicons/logos. Instantly download them when ▸playing.
# config:
#    { name: favicon_google_first, type: bool, value: 1, description: "always use google favicon to png conversion service" }
#    { name: favicon_google_first, type: bool, value: 1, description: "Prefer faster Google favicon to PNG conversion service." }
#    { name: favicon_google_only,  type: bool, value: 1, description: "don't try other favicon retrieval methods, if google service fails" }
#    { name: favicon_delete_stub , type: bool, value: 1, description: "delete placeholder favicons" }
# type: function
#    { name: favicon_delete_stub , type: bool, value: 1, description: "Don't accept any placeholder favicons." }
#    [ main-name: google_homepage ]
#    [ main-name: load_favicon ]
# type: feature
# category: ui
# version: 1.7
# depends: streamtuner2 >= 2.1.9, python:pil
# priority: standard
#
# This module fetches favicon.ico files and prepares .png images for each domain
# This module fetches a favicon for each station, or a small banner
# in the stations list. Homepage URLs are used for this.
#
# or logo for some channel modules. It converts .ico image files and
# sanitizes .png or .jpeg images even prior display.
# 
# Files end up in:
#    /home/user/.config/streamtuner2/icons/www.example.org.png
# It prepares cache files in ~/.config/streamtuner2/icons/ in agreement
# with the station list display logic. Either uses station "homepage"
# or "img" URLs from entry rows{}.
#
# While it can often discover favicons directly from station homepages,
# Currently relies on Google conversion service, because urllib+PIL conversion
# it's often speedier to user the Google PNG conversion service. Both
# method is still flaky, and a bit slower. Future version might use imagemagick.


always_google = 1      # use favicon service for speed
only_google = 1        # if that fails, try our other/slower methods?
# depend on a recent Pillow2 python module (superseding the PIL module).
# Else may display images with fragments if converted from ICO files.
#
# Has recently been rewritten, is somewhat less entangled with other
# modules now:
#  · GenericChannel presets row["favicon"] with cache image filename
#    in any case, uses row["homepage"] or row["img"] as template
#  · the filename shortening functionality must be shared between
#    favicon and genericchannel.prepare() code
#  · uikit.columns() merely checks row["favicon"] for file existence
#    on redraws
delete_google_stub = 1   # don't keep placeholder images
google_placeholder_filesizes = (726,896)
#  · main.play() only calls .update_playing() or .update_all()
#  · urllib is no longer required, uses the main ahttp/requests API
#  · still might need unhtml() from channels/__init__ later
#  · Reduce config options → move main favicon options here?


import os, os.path
from compat2and3 import xrange, urllib
from compat2and3 import StringIO
import re
from config import *
from threading import Thread
import ahttp
import compat2and3
from PIL import Image
from uikit import gtk



# ensure that we don't try to download a single favicon twice per session,
# if it's not available the first time, we won't get it after switching stations back and forth
# Ensure that we don't try to download a single favicon twice per session.
# If it's not available the first time, we won't get it after switching
# stations back and forth either. So URLs are skipped simply.
tried_urls = []



# Hook up as feature plugin

# walk through entries
def download_all(*args, **kwargs):
  t = Thread(target=download_thread, args=args, kwargs=kwargs)
  t.start()
def download_thread(entries, pixstore=None):
    for i,e in enumerate(entries):
        # try just once
        if e.get("homepage") in tried_urls:
            continue
#
class favicon(object):

    # plugin attributes
    module = "favicon"
    meta = plugin_meta()
    
    
    # Register with main
    def __init__(self, parent):
    
        # Reference main, and register hook
        self.parent, self.main = parent, parent
        parent.hooks["play"].append(self.update_playing)

        # Prepare favicon cache directory
        conf.icon_dir = conf.dir + "/icons"
        if not os.path.exists(conf.icon_dir):
            os.mkdir(conf.icon_dir)
            open(icon_dir+"/.nobackup", "a").close()



    # Main callback: update favicon cache for complete list of station rows
    def update_all(self, *args, **kwargs):
        #kwargs[pixstore] = self.parent.channel()._ls, ...
        self.parent.thread(self.update_rows, *args, **kwargs)
        # retrieve specific img url as favicon
        elif e.get("img"):
            localcopy(e["img"], True)
            tried_urls.append(e.get("img"))
        # favicon from homepage URL
        elif e.get("homepage"):
            download(e["homepage"])
            tried_urls.append(e.get("homepage"))


    # Main callback for a single play() event
    def update_playing(self, row, pixstore=None):

        # Homepage search
        if conf.google_homepage and not len(row.get("homepage", "")):
            google_find_homepage(row)

        # Favicon only for currently playing station
        if conf.load_favicon:
            if row.get("homepage") or row.get("img"):
                self.update_all([row], pixstore=pixstore)

      
    # Run through rows[] to update "favicon" from "homepage" or "img",
    # optionally display new image right away in ListStore
    def update_rows(self, entries, pixstore=None):
        for i,row in enumerate(entries):
            ok = False

        # Update TreeView
            # Try just once
        update_pixstore(e, pixstore, i)
    pass
            if row.get("homepage") in tried_urls:
                continue
            # Ignore existing ["favicon"] filename
            if row.get("favicon") and False:
                pass

            # Cache image filename: have or can't have
            favicon_fn = row_to_fn(row)
            if not favicon_fn:
                continue
            if os.path.exists(favicon_fn):
                continue
# download a single favicon for currently playing station
def download_playing(row, pixstore=None):
    if conf.google_homepage and not row.get("homepage"):
        google_find_homepage(row)
    if conf.load_favicon and row.get("homepage"):
        download_all([row], pixstore=pixstore)
    pass

            # Custom "img" banner/logo as favicon
            if row.get("img"):
                tried_urls.append(row["img"])
                ok = banner_localcopy(row["img"], favicon_fn)

            # Homepage to favicon
            elif row.get("homepage"):
                tried_urls.append(row["homepage"])
                if conf.favicon_google_first:
                    ok = fav_google_ico2png(row["homepage"], favicon_fn)
                else:
                    ok = fav_from_homepage(row["homepage"], favicon_fn)

            # Update TreeView
            if ok:
                self.update_pixstore(row, pixstore, i)
        pass


# Update favicon in treeview/liststore
def update_pixstore(row, pixstore=None, row_i=None):
    log.PIXSTORE(pixstore, row_i)
    if pixstore:
    # Update favicon in treeview/liststore
    def update_pixstore(self, row, pixstore=None, row_i=None):
        log.FAVICON_UPDATE_PIXSTORE(pixstore, row_i)
        if not pixstore:
            return

        # Unpack ListStore, pixbuf column no, preset rowno
        ls, pix_entry, i = pixstore
        # Else rows-iteration rowno
        if i is None:
            i = row_i
        fn = None

        # Existing "favicon" cache filename
        if row.get("favicon"):
            fn = row["favicon"]
        elif row.get("img"):
            fn = localcopy(row["img"], False)
        elif row.get("homepage"):
        else:
            fn = row_to_fn(row)

            fn = file(row["homepage"])
        # Update pixbuf in active station liststore
        if fn and os.path.exists(fn):
            try:
            p = gtk.gdk.pixbuf_new_from_file(fn)
            ls[i][pix_entry] = p
                p = gtk.gdk.pixbuf_new_from_file(fn)
                ls[i][pix_entry] = p
            except Exception as e:
                log.ERR("Update_pixstore image", fn, "error:", e)




#--- unrelated ---
def google_find_homepage(row):
#--- somewhat unrelated ---
#
# Should become a distinct feature plugin. - It just depends on correct
# invocation order for plugins to work.
# Googling is often blocked anyway, because this is clearly a bot request.
# Tag requests with ?client=streamtuner2 purposefully still.
# 
def google_find_homepage(self, row):
    """ Searches for missing homepage URL via Google. """
    if row.get("url") not in tried_urls:
        tried_urls.append(row.get("url"))

    if row.get("title"):
        rx_t = re.compile('^(([^-:]+.?){1,2})')
        rx_u = re.compile('"(http://[^"]+)" class=l')
        rx_u = re.compile(r'/url\?q=(https?://[^"&/]+)')

        # extract first title parts
        title = rx_t.search(row["title"])
        if title:
            title = title.group(0).replace(" ", "%20")
            
            # do a google search
            html = ahttp.get("http://www.google.de/search?hl=de&q="+title, params={}, ajax=1)
            
            # find first URL hit
            url = rx_u.search(html)
            if url:
                row["homepage"] = ahttp.fix_url(url.group(1))
        # Use literal station title now
        title = row["title"]
        #title = title.group(0).replace(" ", "%20")
        
        # Do 'le google search
        html = ahttp.get("http://www.google.com/search", params=dict(hl="en", q=title, client="streamtuner2"), ajax=1)
                  
        # Find first URL hit
        url = rx_u.findall(html)
        if url:
            row["homepage"] = ahttp.fix_url(url[0])
    pass
#-----------------



# extract domain name
def domain(url):
    if url.startswith("http://"):

# Convert row["img"] or row["homepage"] into local favicon cache filename
def row_to_fn(row):
    url = row.get("img") or row.get("homepage") or None
    if url:
        return url[7:url.find("/", 8)]  # we assume our URLs are fixed already (http://example.org/ WITH trailing slash!)
    else:
        return "null"
         url = url.lower()
         url = re.sub("^\w+://|/$", "", url)   # strip proto:// and trailing /
         url = re.sub("[^\w._-]", "_", url)    # remove any non-word characters
         url = "{}/{}.png".format(conf.icon_dir, url)
    return url

# local filename
def name(url):
    return domain(url) + ".png"
  

# local filename
def file(url):
    icon_dir = conf.dir + "/icons"
    if not os.path.exists(icon_dir):
        os.mkdir(icon_dir)
        open(icon_dir+"/.nobackup", "w").close()
    return icon_dir + "/" + name(url)

    
# does the favicon exist
def available(url):
# Copy banner row["img"] into icons/ directory
def banner_localcopy(url, fn):
    return os.path.exists(file(url))
    
    

    # Check URL and target filename
# copy image from url into icons/ directory
def localcopy(url, download=False):
    if url and url.startswith("http"):
        fn = re.sub("[:/]", "_", url)
    if not re.match("^https?://[\w.-]{10}", url):
        fn = conf.dir + "/icons/" + fn
        if os.path.exists(fn):
            return fn
        elif download:
            imgdata = ahttp.get(url, binary=1, verify=False)
        return False

    # Fetch and save
    imgdata = ahttp.get(url, binary=1, verify=False)
            with open(fn, "wb") as f:
                f.write(imgdata)
    if imgdata:
                f.close()
        if os.path.exists(fn):    
            return fn
        return store_image(imgdata, fn)
    else:
        return url

    



    
# Check valid image, possibly convert, and save to cache filename
def store_image(imgdata, fn, resize=None):
# download favicon for given URL
def download(url):

  # skip if .png for domain already exists
  if available(url):
    return

    # Convert accepted formats -- even PNG for filtering now
    if re.match(br'^(.PNG|GIF\d+|.{0,15}JFIF|\x00\x00\x01\x00|.{0,255}<svg[^>]+svg)', imgdata):
        try:
            # Read from byte/str
            image = Image.open(StringIO(imgdata))
            log.FAVICON_IMAGE_TO_PNG(image, resize)

  # fastest method, so default to google for now
  if always_google:
      google_ico2png(url)
            # Resize
            if resize and image.size[0] > resize:
                image.resize((resize, resize), Image.ANTIALIAS)
      if available(url) or only_google:
         return

            # Convert to PNG via string buffer
            out = StringIO()
  try:    # look for /favicon.ico first
            image.save(out, "PNG", quality=98)
    log.FAVICON("try /favicon.ico")
    direct_download("http://"+domain(url)+"/favicon.ico", file(url))

            imgdata = out.getvalue()
  except:
    try:    # extract facicon filename from website <link rel>
      log.FAVICON("html <rel favicon>")
      html_download(url)

    except Exception as e:    # fallback
      log.ERR(e)
        except Exception as e:
            return log.ERR("favicon/logo conversion error:", e) and False
      google_ico2png(url)


    else:
        log.WARN("couldn't detect mime type")


    # PNG already?
# retrieve PNG via Google ico2png
def google_ico2png(url):
    log.FAVICON("google ico2png")

    if re.match(b"^.(PNG)", imgdata):
    GOOGLE = "http://www.google.com/s2/favicons?domain="
    (fn, headers) = urllib.urlretrieve(GOOGLE+domain(url), file(url))

        try:
    # test for stub image
    if delete_google_stub and (filesize(fn) in google_placeholder_filesizes):
       os.remove(fn)

  
            with open(fn, "wb") as f:
                f.write(imgdata)
def filesize(fn):
   return os.stat(fn).st_size

                return True
        except Exception as e:
            log.ERR("favicon.store_image() failure:", e)


# mime magic
def filetype(fn):
   f = open(fn, "rb")
   bin = f.read(4)
   f.close()
   if bin[1:3] == "PNG":
      return "image/png"
   else:
      return "*/*"



# favicon.ico
# PNG via Google ico2png
def fav_google_ico2png(url, fn):
    log.FAVICON("google ico2png")
def direct_download(favicon, fn):

    # URL download
    # Download from service
    r = urllib.urlopen(favicon)
    headers = r.info()
    domain = re.sub("^\w+://|/.*$", "", url).lower()
    log.HTTP(headers)
    
    geturl = "http://www.google.com/s2/favicons?domain={}".format(domain)
    # abort on
    if r.getcode() >= 300:
    imgdata = ahttp.get(geturl, binary=1, timeout=2.5)
       raise Exception("HTTP error %s" % r.getcode())
    if not headers["Content-Type"].lower().find("image/") == 0:
       raise Exception("can't use text/* content")
       
    # save file
    
    # Check for stub sizes
    fn_tmp = fn+".tmp"
    f = open(fn_tmp, "wb")
    f.write(r.read(32768))
    f.close()
        
    # check type
    if conf.favicon_delete_stub and len(imgdata) in (726,896): # google_placeholder_filesizes
        log.FAVICON("placeholder size, skipping")
        return False
    # Save
    if headers["Content-Type"].lower()=="image/png" and favicon.find(".png") and filetype(fn)=="image/png":
       pngresize(fn_tmp)
       os.mv(fn_tmp, fn)
    else:
       ico2png(fn_tmp, fn)
        return store_image(imgdata, fn)
       os.remove(fn_tmp)

    

  
# peek at URL, download favicon.ico <link rel>
# Peek at homepage URL, download favicon.ico <link rel>, convert to PNG file, resize to 16x16
def html_download(url):

def fav_from_homepage(url, fn):

  # <link rel>
    # Check for <link rel=icon>
  #try:
    # download html, look for @href in <link rel=shortcut icon>
    r = urllib.urlopen(url)
    html = r.read(4096)
    img = html_link_icon(url)
    if not img:
        return False
    r.close()
    rx = re.compile("""<link[^<>]+rel\s*=\s*"?\s*(?:shortcut\s+|fav)?icon[^<>]+href=["'](?P<href>[^<>"']+)["'<>\s].""")
    favicon = "".join(rx.findall(html))
    log.DATA(favicon)
    
    # url or
        
    # Fetc image, verify MIMEE type
    if favicon.startswith("http://"):
       None
    # just /pathname
    else:
       favicon = compat2and3.urlparse.urljoin(url, favicon)
       log.FAVICON(favicon)
       #favicon = "http://" + domain(url) + "/" + favicon
    r = ahttp.get(img, binary=1, content=0, timeout=2.75)
    if not re.match('image/(png|jpe?g|png|ico|x-ico|vnd.microsoft.ico)', r.headers["content-type"], re.I):
        log.WARN("content-type wrong", r.headers)
        return False
        
    # Convert, resize and save
    return store_image(r.content, fn, resize=16)



    # download
# Download HTML, look for favicon name in <link rel=shortcut icon>.
    direct_download(favicon, file(url))



# convert .ico file to .png format
def ico2png(ico, png_fn):
#
# Very rough, doesn't respect any <base href=> and manually patches
# icon path to homepage url; nor does any entity decoding.
#
def html_link_icon(url, href="/favicon.png"):
    html = ahttp.get(url, encoding="iso-8859-1", timeout=3.5)
    image = Image.open(ico)
    log.FAVICON_ICO2PNG(ico, png, image)
    # resize
    if image.size[0] > 16:
        image.resize((16, 16), Image.ANTIALIAS)
    # .png format
    # Extract
    for link in re.findall(r"""  <link ([^<>]+) >  """, html, re.X):
        pair = re.findall(r"""  \b(rel|href) \s*=\s* ["']? ([^<>"']+) ["']? """, link, re.X)
        pair = { name: val for name, val in pair }
        for name in ("shortcut icon", "favicon", "icon", "icon shortcut"):
            if name == pair.get("rel", "ignore") and pair.get("href"):
                href = pair["href"] # unhtml()
                break
    image.save(png_fn, "PNG", quality=98)


    # Patch URL together (strip double proto/domain, or double slash)
    return re.sub("^(https?://\w[^/]+\w)?/?(https?://\w[^/]+\w)/?(/.+)$", "\g<2>\g<3>", url+href)
    
# resize an image
def pngresize(fn, x=16, y=16):
    image = Image.open(fn)
    if image.size[0] > x:
        image.resize((x, y), Image.ANTIALIAS)
        image.save(fn, "PNG", quality=98)




#-- test
if __name__ == "__main__":
    import sys
    download(sys.argv[1])
    favicon(None).download(sys.argv[1])