8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 | 8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 |
-
+
| #
# Provides a http GET method with gtk.statusbar() callback.
# And a function to add trailings slashes on http URLs.
#
#
from compat2and3 import urllib2, urlencode, urlparse, cookielib, StringIO, xrange
from compat2and3 import urllib2, urlencode, urlparse, cookielib, StringIO, xrange, PY3
from gzip import GzipFile
from config import conf, __print__, dbg
#-- url download ---------------------------------------------
|
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51 | 37
38
39
40
41
42
43
44
45
46
47
48
49
50
51 |
-
+
| except: pass
#-- GET
def get(url, maxsize=1<<19, feedback="old"):
__print__("GET", url)
__print__( dbg.HTTP, "GET", url)
# statusbar info
progress_feedback(url, 0.0)
# read
content = ""
f = urllib2.urlopen(url) |
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 | 65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 |
-
+
|
# done
# clean statusbar
progress_feedback()
# fin
__print__(len(content))
__print__( dbg.INFO, "Content-Length", len(content) )
return content
#-- fix invalid URLs |
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 | 122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 |
-
+
-
+
| "Referer": (referer if referer else url),
})
if type(post) == dict:
post = urlencode(post)
request = urllib2.Request(url, post, headers)
# open url
__print__( vars(request) )
__print__( dbg.INFO, vars(request) )
progress_feedback(url, 0.2)
r = urllib2.urlopen(request)
# get data
__print__( r.info() )
__print__( dbg.HTTP, r.info() )
progress_feedback(0.5)
data = r.read()
progress_feedback()
return data
|