11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
-
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-
+
-
-
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
|
#
#
from compat2and3 import urllib2, urlencode, urlparse, cookielib, StringIO, xrange, PY3
from gzip import GzipFile
from config import conf, __print__, dbg
import requests
import copy
#-- url download ---------------------------------------------
def get(url, maxsize=1<<19, feedback="old"):
def get(url, params={}, referer="", post=0, ajax=0, binary=0):
__print__( dbg.HTTP, "GET", url)
# statusbar info
progress_feedback(url, 0.0)
progress_feedback(url, 0.1)
# read
content = ""
f = urllib2.urlopen(url)
max = 222000 # mostly it's 200K, but we don't get any real information
read_size = 1
# combine headers
headers = copy.copy(default_headers)
if ajax:
headers["X-Requested-With"] = "XMLHttpRequest"
if referer:
headers["Referer"] = (referer if referer else url)
# multiple steps
while (read_size and len(content) < maxsize):
# partial read
add = f.read(8192)
content = content + add
read_size = len(add)
# set progress meter
progress_feedback(float(len(content)) / float(max))
# read
if post:
__print__("POST")
r = requests.post(url, params=params, headers=headers)
else:
__print__("GET")
r = requests.get(url, params=params, headers=headers)
# result
progress_feedback(0.9)
content = (r.content if binary else r.text)
# done
# clean statusbar
# finish, clean statusbar
progress_feedback()
# fin
# default HTTP headers for AJAX/POST request
default_headers = {
"User-Agent": "streamtuner2/2.1 (X11; U; Linux AMD64; en; rv:1.5.0.1) like WinAmp/2.1 but not like Googlebot/2.1", #"Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.6) Gecko/20100628 Ubuntu/10.04 (lucid) Firefox/3.6.6",
"Accept": "*/*;q=0.5, audio/*, url/*",
"Accept-Language": "en-US,en,de,es,fr,it,*;q=0.1",
"Accept-Encoding": "gzip,deflate",
"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.1",
"Keep-Alive": "115",
"Connection": "keep-alive",
#"Content-Length", "56",
#"Cookie": "s_pers=%20s_getnr%3D1278607170446-Repeat%7C1341679170446%3B%20s_nrgvo%3DRepeat%7C1341679170447%3B; s_sess=%20s_cc%3Dtrue%3B%20s_sq%3Daolshtcst%252Caolsvc%253D%252526pid%25253Dsht%25252520%2525253A%25252520SHOUTcast%25252520Radio%25252520%2525257C%25252520Search%25252520Results%252526pidt%25253D1%252526oid%25253Dfunctiononclick%25252528event%25252529%2525257BshowMoreGenre%25252528%25252529%2525253B%2525257D%252526oidt%25253D2%252526ot%25253DDIV%3B; aolDemoChecked=1.849061",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
}
# simulate ajax calls
def ajax(url, post, referer=""):
# request
headers = default_headers
headers.update({
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"X-Requested-With": "XMLHttpRequest",
"Referer": (referer if referer else url),
})
if type(post) == dict:
post = urlencode(post)
request = urllib2.Request(url, post, headers)
# open url
__print__( dbg.INFO, vars(request) )
progress_feedback(url, 0.2)
r = urllib2.urlopen(request)
# get data
__print__( dbg.HTTP, r.info() )
progress_feedback(0.5)
data = r.read()
progress_feedback()
return data
# http://techknack.net/python-urllib2-handlers/
class ContentEncodingProcessor(urllib2.BaseHandler):
"""A handler to add gzip capabilities to urllib2 requests """
# add headers to requests
def http_request(self, req):
req.add_header("Accept-Encoding", "gzip, deflate")
return req
# decode
def http_response(self, req, resp):
old_resp = resp
# gzip
if resp.headers.get("content-encoding") == "gzip":
gz = GzipFile(
fileobj=StringIO(resp.read()),
mode="r"
)
resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
# deflate
if resp.headers.get("content-encoding") == "deflate":
gz = StringIO( deflate(resp.read()) )
resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and geturl() methods to an open file.'
resp.msg = old_resp.msg
return resp
# deflate support
import zlib
def deflate(data): # zlib only provides the zlib compress format, not the deflate format;
try: # so on top of all there's this workaround:
return zlib.decompress(data, -zlib.MAX_WBITS)
except zlib.error:
return zlib.decompress(data)
#-- init for later use
if urllib2:
# config 1
handlers = [None, None, None]
# base
handlers[0] = urllib2.HTTPHandler()
if conf.debug:
handlers[0].set_http_debuglevel(3)
# content-encoding
handlers[1] = ContentEncodingProcessor()
# store cookies at runtime
cj = cookielib.CookieJar()
handlers[2] = urllib2.HTTPCookieProcessor( cj )
# inject into urllib2
urllib2.install_opener( urllib2.build_opener(*handlers) )
# alternative function names
AJAX=ajax
POST=ajax
GET=get
URL=fix_url
|