13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
# And can normalize URLs to always carry a trailing slash
# after the domain name.
from config import *
import requests
#-- hooks to progress meter and status bar in main window
feedback = None
# Sets either text or percentage of main windows' status bar.
#
# Can either take a float parameter (e.g. 0.99 for % indicator)
|
>
>
>
>
>
>
>
>
>
>
|
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
# And can normalize URLs to always carry a trailing slash
# after the domain name.
from config import *
import requests
#-- extra debugging, http://stackoverflow.com/questions/10588644/how-can-i-see-the-entire-http-request-thats-being-sent-by-my-python-application
if conf.debug >= 2:
import logging
import httplib
httplib.HTTPConnection.debuglevel = 1
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
#-- hooks to progress meter and status bar in main window
feedback = None
# Sets either text or percentage of main windows' status bar.
#
# Can either take a float parameter (e.g. 0.99 for % indicator)
|
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
#-- Retrieve data via HTTP
#
# Well, it says "get", but it actually does POST and AJAXish GET requests too.
#
def get(
url, params={}, referer="", post=0, ajax=0,
binary=0, content=True, encoding=None, verify=False,
statusmsg=None, timeout=9.25, quieter=0
):
# statusbar info
if not quieter:
progress_feedback(url, timeout=timeout/1.5)
# combine headers
headers = {}
if ajax:
headers["X-Requested-With"] = "XMLHttpRequest"
if referer:
headers["Referer"] = (referer if referer else url)
#ifdef BLD_DEBUG
#srcout raise Exception("Simulated HTTP error")
#endif
# read
if post:
log.HTTP("POST", url, params)
r = session.post(url, params=params, headers=headers, timeout=timeout)
else:
log.HTTP("GET"+(" AJAX" if ajax else ""), url, params )
r = session.get(url, params=params, headers=headers, verify=verify, timeout=timeout)
if not quieter:
log.HTTP(">>>", r.request.headers );
log.HTTP("<<<", r.headers );
|
|
|
|
|
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
#-- Retrieve data via HTTP
#
# Well, it says "get", but it actually does POST and AJAXish GET requests too.
#
def get(
url, params={}, referer=None, post=0, ajax=0,
binary=0, content=True, encoding=None, verify=False,
statusmsg=None, timeout=9.25, quieter=0
):
# statusbar info
if not quieter:
progress_feedback(url, timeout=timeout/1.5)
# combine headers
headers = {}
if ajax:
headers["X-Requested-With"] = "XMLHttpRequest"
if referer:
headers["Referer"] = (referer if referer not in [True, 1] else url)
#ifdef BLD_DEBUG
#srcout raise Exception("Simulated HTTP error")
#endif
# read
if post:
log.HTTP("POST", url, params)
r = session.post(url, data=params, headers=headers, timeout=timeout)
else:
log.HTTP("GET"+(" AJAX" if ajax else ""), url, params )
r = session.get(url, params=params, headers=headers, verify=verify, timeout=timeout)
if not quieter:
log.HTTP(">>>", r.request.headers );
log.HTTP("<<<", r.headers );
|