Lines Matching refs:url
1 """Open an arbitrary URL.
12 - RFC1808: the "relative URL" spec. (authoritative status)
13 - RFC1738 - the "URL standard". (authoritative status)
54 """OS-specific conversion from a relative URL of the 'file' scheme
59 """OS-specific conversion from a file system path to a relative URL
72 def urlopen(url, data=None, proxies=None):
73 """Create a file-like object for the specified URL to read from."""
87 return opener.open(url)
89 return opener.open(url, data)
90 def urlretrieve(url, filename=None, reporthook=None, data=None):
94 return _urlopener.retrieve(url, filename, reporthook, data)
149 # in case you want logically independent URL openers
181 # percent encode url, fixing lame server errors for e.g, like space
182 # within url paths.
188 urltype, url = splittype(fullurl)
195 url = (host, fullurl) # Signal special case to open_*()
208 return getattr(self, name)(url)
210 return getattr(self, name)(url, data)
215 """Overridable interface to open unknown URL type."""
216 type, url = splittype(fullurl)
217 raise IOError, ('url error', 'unknown url type', type)
220 """Overridable interface to open unknown URL type."""
221 type, url = splittype(fullurl)
222 raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
225 def retrieve(self, url, filename=None, reporthook=None, data=None):
226 """retrieve(url) returns (filename, headers) for a local object
228 url = unwrap(toBytes(url))
229 if self.tempcache and url in self.tempcache:
230 return self.tempcache[url]
231 type, url1 = splittype(url)
240 fp = self.open(url, data)
247 garbage, path = splittype(url)
258 self.tempcache[url] = result
288 # Each method named open_<type> knows how to open that type of URL
290 def open_http(self, url, data=None):
295 if isinstance(url, str):
296 host, selector = splithost(url)
302 host, selector = url
305 # now we proceed with the url we want to obtain
307 url = rest
356 return addinfourl(fp, headers, "http:" + url, errcode)
359 return self.http_error(url, fp, errcode, errmsg, headers)
361 return self.http_error(url, fp, errcode, errmsg, headers, data)
363 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
372 result = method(url, fp, errcode, errmsg, headers)
374 result = method(url, fp, errcode, errmsg, headers, data)
376 return self.http_error_default(url, fp, errcode, errmsg, headers)
378 def http_error_default(self, url, fp, errcode, errmsg, headers):
384 def open_https(self, url, data=None):
390 if isinstance(url, str):
391 host, selector = splithost(url)
397 host, selector = url
401 url = rest
448 return addinfourl(fp, headers, "https:" + url, errcode)
451 return self.http_error(url, fp, errcode, errmsg, headers)
453 return self.http_error(url, fp, errcode, errmsg, headers,
456 def open_file(self, url):
457 """Use local file or FTP depending on form of URL."""
458 if not isinstance(url, str):
460 if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
461 return self.open_ftp(url)
463 return self.open_local_file(url)
465 def open_local_file(self, url):
472 host, file = splithost(url)
480 mtype = mimetypes.guess_type(url)[0]
489 raise ValueError("local file url may start with / or file:. Unknown url of type: %s" % url)
502 def open_ftp(self, url):
504 if not isinstance(url, str):
511 host, path = splithost(url)
553 mtype = mimetypes.guess_type("ftp:" + url)[0]
560 return addinfourl(fp, headers, "ftp:" + url)
564 def open_data(self, url, data=None):
565 """Use "data" URL."""
566 if not isinstance(url, str):
581 [type, data] = url.split(',', 1)
583 raise IOError, ('data error', 'bad data URL')
607 return addinfourl(f, headers, url)
619 def http_error_default(self, url, fp, errcode, errmsg, headers):
621 return addinfourl(fp, headers, "http:" + url, errcode)
623 def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
632 return meth(url, fp, 500,
634 result = self.redirect_internal(url, fp, errcode, errmsg, headers,
639 def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
647 # In case the server sent a relative URL, join with original:
648 newurl = basejoin(self.type + ":" + url, newurl)
657 errmsg + " - Redirection to url '%s' is not allowed" %
663 def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
665 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
667 def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
669 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
671 def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
674 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
676 return self.http_error_default(url, fp, errcode, errmsg, headers)
678 def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
682 URLopener.http_error_default(self, url, fp,
688 URLopener.http_error_default(self, url, fp,
692 URLopener.http_error_default(self, url, fp,
696 return getattr(self,name)(url, realm)
698 return getattr(self,name)(url, realm, data)
700 def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
704 URLopener.http_error_default(self, url, fp,
710 URLopener.http_error_default(self, url, fp,
714 URLopener.http_error_default(self, url, fp,
718 return getattr(self,name)(url, realm)
720 return getattr(self,name)(url, realm, data)
722 def retry_proxy_http_basic_auth(self, url, realm, data=None):
723 host, selector = splithost(url)
739 def retry_proxy_https_basic_auth(self, url, realm, data=None):
740 host, selector = splithost(url)
756 def retry_http_basic_auth(self, url, realm, data=None):
757 host, selector = splithost(url)
769 def retry_https_basic_auth(self, url, realm, data=None):
770 host, selector = splithost(url)
1003 def __init__(self, fp, headers, url, code=None):
1006 self.url = url
1016 return self.url
1020 # unwrap('<URL:type://host/path>') --> 'type://host/path'
1043 def toBytes(url):
1044 """toBytes(u"URL") --> 'URL'."""
1045 # Most URL schemes require ASCII. If that changes, the conversion
1047 if _is_unicode(url):
1049 url = url.encode("ASCII")
1051 raise UnicodeError("URL " + repr(url) +
1053 return url
1055 def unwrap(url):
1056 """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
1057 url = url.strip()
1058 if url[:1] == '<' and url[-1:] == '>':
1059 url = url[1:-1].strip()
1060 if url[:4] == 'URL:': url = url[4:].strip()
1061 return url
1064 def splittype(url):
1071 match = _typeprog.match(url)
1074 return scheme.lower(), url[len(scheme) + 1:]
1075 return None, url
1078 def splithost(url):
1085 match = _hostprog.match(url)
1092 return None, url
1154 def splitquery(url):
1161 match = _queryprog.match(url)
1163 return url, None
1166 def splittag(url):
1173 match = _tagprog.match(url)
1175 return url, None
1177 def splitattr(url):
1180 words = url.split(';')
1248 Each part of a URL, e.g. the path info, the query, etc., has a
1257 Each of these characters is reserved in some component of a URL,
1261 section of a URL. Thus, it will not encode '/'. This character
1285 """Quote the query fragment of a URL; replacing ' ' with '+'"""
1292 """Encode a sequence of two-element tuples or dictionary into a URL query string.
1356 """Return a dictionary of scheme -> proxy server URL mappings.
1455 """Return a dictionary of scheme -> proxy server URL mappings.
1473 """Return a dictionary of scheme -> proxy server URL mappings.
1519 """Return a dictionary of scheme -> proxy server URL mappings.
1580 """Return a dictionary of scheme -> proxy server URL mappings.
