googleapiclient.http

1 # Copyright 2014 Google Inc. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 """Classes to encapsulate a single HTTP request. 16 17 The classes implement a command pattern, with every 18 object supporting an execute() method that does the 19 actuall HTTP request. 20 """ 21 from __future__ import absolute_import 22 import six 23 from six.moves import http_client 24 from six.moves import range 25 26 __author__ = 'jcgregorio@google.com (Joe Gregorio)' 27 28 from six import BytesIO, StringIO 29 from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 30 31 import base64 32 import copy 33 import gzip 34 import httplib2 35 import json 36 import logging 37 import mimetypes 38 import os 39 import random 40 import socket 41 import sys 42 import time 43 import uuid 44 45 # TODO(issue 221): Remove this conditional import jibbajabba. 46 try: 47 import ssl 48 except ImportError: 49 _ssl_SSLError = object() 50 else: 51 _ssl_SSLError = ssl.SSLError 52 53 from email.generator import Generator 54 from email.mime.multipart import MIMEMultipart 55 from email.mime.nonmultipart import MIMENonMultipart 56 from email.parser import FeedParser 57 58 # Oauth2client < 3 has the positional helper in 'util', >= 3 has it 59 # in '_helpers'. 60 try: 61 from oauth2client import util 62 except ImportError: 63 from oauth2client import _helpers as util 64 65 from googleapiclient import mimeparse 66 from googleapiclient.errors import BatchError 67 from googleapiclient.errors import HttpError 68 from googleapiclient.errors import InvalidChunkSizeError 69 from googleapiclient.errors import ResumableUploadError 70 from googleapiclient.errors import UnexpectedBodyError 71 from googleapiclient.errors import UnexpectedMethodError 72 from googleapiclient.model import JsonModel 73 74 75 LOGGER = logging.getLogger(__name__) 76 77 DEFAULT_CHUNK_SIZE = 512*1024 78 79 MAX_URI_LENGTH = 2048 80 81 _TOO_MANY_REQUESTS = 429 82 83 DEFAULT_HTTP_TIMEOUT_SEC = 60

84 85 86 -def _should_retry_response(resp_status, content):

87 """Determines whether a response should be retried. 88 89 Args: 90 resp_status: The response status received. 91 content: The response content body. 92 93 Returns: 94 True if the response should be retried, otherwise False. 95 """ 96 # Retry on 5xx errors. 97 if resp_status >= 500: 98 return True 99 100 # Retry on 429 errors. 101 if resp_status == _TOO_MANY_REQUESTS: 102 return True 103 104 # For 403 errors, we have to check for the `reason` in the response to 105 # determine if we should retry. 106 if resp_status == six.moves.http_client.FORBIDDEN: 107 # If there's no details about the 403 type, don't retry. 108 if not content: 109 return False 110 111 # Content is in JSON format. 112 try: 113 data = json.loads(content.decode('utf-8')) 114 reason = data['error']['errors'][0]['reason'] 115 except (UnicodeDecodeError, ValueError, KeyError): 116 LOGGER.warning('Invalid JSON content from response: %s', content) 117 return False 118 119 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 120 121 # Only retry on rate limit related failures. 122 if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): 123 return True 124 125 # Everything else is a success or non-retriable so break. 126 return False

127

128 129 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, 130 **kwargs):

131 """Retries an HTTP request multiple times while handling errors. 132 133 If after all retries the request still fails, last error is either returned as 134 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 135 136 Args: 137 http: Http object to be used to execute request. 138 num_retries: Maximum number of retries. 139 req_type: Type of the request (used for logging retries). 140 sleep, rand: Functions to sleep for random time between retries. 141 uri: URI to be requested. 142 method: HTTP method to be used. 143 args, kwargs: Additional arguments passed to http.request. 144 145 Returns: 146 resp, content - Response from the http request (may be HTTP 5xx). 147 """ 148 resp = None 149 content = None 150 for retry_num in range(num_retries + 1): 151 if retry_num > 0: 152 # Sleep before retrying. 153 sleep_time = rand() * 2 ** retry_num 154 LOGGER.warning( 155 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', 156 sleep_time, retry_num, num_retries, req_type, method, uri, 157 resp.status if resp else exception) 158 sleep(sleep_time) 159 160 try: 161 exception = None 162 resp, content = http.request(uri, method, *args, **kwargs) 163 # Retry on SSL errors and socket timeout errors. 164 except _ssl_SSLError as ssl_error: 165 exception = ssl_error 166 except socket.error as socket_error: 167 # errno's contents differ by platform, so we have to match by name. 168 if socket.errno.errorcode.get(socket_error.errno) not in ( 169 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ): 170 raise 171 exception = socket_error 172 173 if exception: 174 if retry_num == num_retries: 175 raise exception 176 else: 177 continue 178 179 if not _should_retry_response(resp.status, content): 180 break 181 182 return resp, content

183

184 185 -class MediaUploadProgress(object):

186 """Status of a resumable upload.""" 187

188 - def __init__(self, resumable_progress, total_size):

189 """Constructor. 190 191 Args: 192 resumable_progress: int, bytes sent so far. 193 total_size: int, total bytes in complete upload, or None if the total 194 upload size isn't known ahead of time. 195 """ 196 self.resumable_progress = resumable_progress 197 self.total_size = total_size

198

199 - def progress(self):

200 """Percent of upload completed, as a float. 201 202 Returns: 203 the percentage complete as a float, returning 0.0 if the total size of 204 the upload is unknown. 205 """ 206 if self.total_size is not None and self.total_size != 0: 207 return float(self.resumable_progress) / float(self.total_size) 208 else: 209 return 0.0

210

211 212 -class MediaDownloadProgress(object):

213 """Status of a resumable download.""" 214

215 - def __init__(self, resumable_progress, total_size):

216 """Constructor. 217 218 Args: 219 resumable_progress: int, bytes received so far. 220 total_size: int, total bytes in complete download. 221 """ 222 self.resumable_progress = resumable_progress 223 self.total_size = total_size

224

225 - def progress(self):

226 """Percent of download completed, as a float. 227 228 Returns: 229 the percentage complete as a float, returning 0.0 if the total size of 230 the download is unknown. 231 """ 232 if self.total_size is not None and self.total_size != 0: 233 return float(self.resumable_progress) / float(self.total_size) 234 else: 235 return 0.0

236

237 238 -class MediaUpload(object):

239 """Describes a media object to upload. 240 241 Base class that defines the interface of MediaUpload subclasses. 242 243 Note that subclasses of MediaUpload may allow you to control the chunksize 244 when uploading a media object. It is important to keep the size of the chunk 245 as large as possible to keep the upload efficient. Other factors may influence 246 the size of the chunk you use, particularly if you are working in an 247 environment where individual HTTP requests may have a hardcoded time limit, 248 such as under certain classes of requests under Google App Engine. 249 250 Streams are io.Base compatible objects that support seek(). Some MediaUpload 251 subclasses support using streams directly to upload data. Support for 252 streaming may be indicated by a MediaUpload sub-class and if appropriate for a 253 platform that stream will be used for uploading the media object. The support 254 for streaming is indicated by has_stream() returning True. The stream() method 255 should return an io.Base object that supports seek(). On platforms where the 256 underlying httplib module supports streaming, for example Python 2.6 and 257 later, the stream will be passed into the http library which will result in 258 less memory being used and possibly faster uploads. 259 260 If you need to upload media that can't be uploaded using any of the existing 261 MediaUpload sub-class then you can sub-class MediaUpload for your particular 262 needs. 263 """ 264

265 - def chunksize(self):

266 """Chunk size for resumable uploads. 267 268 Returns: 269 Chunk size in bytes. 270 """ 271 raise NotImplementedError()

272

273 - def mimetype(self):

274 """Mime type of the body. 275 276 Returns: 277 Mime type. 278 """ 279 return 'application/octet-stream'

280

281 - def size(self):

282 """Size of upload. 283 284 Returns: 285 Size of the body, or None of the size is unknown. 286 """ 287 return None

288

289 - def resumable(self):

290 """Whether this upload is resumable. 291 292 Returns: 293 True if resumable upload or False. 294 """ 295 return False

296

297 - def getbytes(self, begin, end):

298 """Get bytes from the media. 299 300 Args: 301 begin: int, offset from beginning of file. 302 length: int, number of bytes to read, starting at begin. 303 304 Returns: 305 A string of bytes read. May be shorter than length if EOF was reached 306 first. 307 """ 308 raise NotImplementedError()

309

310 - def has_stream(self):

311 """Does the underlying upload support a streaming interface. 312 313 Streaming means it is an io.IOBase subclass that supports seek, i.e. 314 seekable() returns True. 315 316 Returns: 317 True if the call to stream() will return an instance of a seekable io.Base 318 subclass. 319 """ 320 return False

321

322 - def stream(self):

323 """A stream interface to the data being uploaded. 324 325 Returns: 326 The returned value is an io.IOBase subclass that supports seek, i.e. 327 seekable() returns True. 328 """ 329 raise NotImplementedError()

330 331 @util.positional(1)

332 - def _to_json(self, strip=None):

333 """Utility function for creating a JSON representation of a MediaUpload. 334 335 Args: 336 strip: array, An array of names of members to not include in the JSON. 337 338 Returns: 339 string, a JSON representation of this instance, suitable to pass to 340 from_json(). 341 """ 342 t = type(self) 343 d = copy.copy(self.__dict__) 344 if strip is not None: 345 for member in strip: 346 del d[member] 347 d['_class'] = t.__name__ 348 d['_module'] = t.__module__ 349 return json.dumps(d)

350

351 - def to_json(self):

352 """Create a JSON representation of an instance of MediaUpload. 353 354 Returns: 355 string, a JSON representation of this instance, suitable to pass to 356 from_json(). 357 """ 358 return self._to_json()

359 360 @classmethod

361 - def new_from_json(cls, s):

362 """Utility class method to instantiate a MediaUpload subclass from a JSON 363 representation produced by to_json(). 364 365 Args: 366 s: string, JSON from to_json(). 367 368 Returns: 369 An instance of the subclass of MediaUpload that was serialized with 370 to_json(). 371 """ 372 data = json.loads(s) 373 # Find and call the right classmethod from_json() to restore the object. 374 module = data['_module'] 375 m = __import__(module, fromlist=module.split('.')[:-1]) 376 kls = getattr(m, data['_class']) 377 from_json = getattr(kls, 'from_json') 378 return from_json(s)

379

380 381 -class MediaIoBaseUpload(MediaUpload):

382 """A MediaUpload for a io.Base objects. 383 384 Note that the Python file object is compatible with io.Base and can be used 385 with this class also. 386 387 fh = BytesIO('...Some data to upload...') 388 media = MediaIoBaseUpload(fh, mimetype='image/png', 389 chunksize=1024*1024, resumable=True) 390 farm.animals().insert( 391 id='cow', 392 name='cow.png', 393 media_body=media).execute() 394 395 Depending on the platform you are working on, you may pass -1 as the 396 chunksize, which indicates that the entire file should be uploaded in a single 397 request. If the underlying platform supports streams, such as Python 2.6 or 398 later, then this can be very efficient as it avoids multiple connections, and 399 also avoids loading the entire file into memory before sending it. Note that 400 Google App Engine has a 5MB limit on request size, so you should never set 401 your chunksize larger than 5MB, or to -1. 402 """ 403 404 @util.positional(3)

405 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 406 resumable=False):

407 """Constructor. 408 409 Args: 410 fd: io.Base or file object, The source of the bytes to upload. MUST be 411 opened in blocking mode, do not use streams opened in non-blocking mode. 412 The given stream must be seekable, that is, it must be able to call 413 seek() on fd. 414 mimetype: string, Mime-type of the file. 415 chunksize: int, File will be uploaded in chunks of this many bytes. Only 416 used if resumable=True. Pass in a value of -1 if the file is to be 417 uploaded as a single chunk. Note that Google App Engine has a 5MB limit 418 on request size, so you should never set your chunksize larger than 5MB, 419 or to -1. 420 resumable: bool, True if this is a resumable upload. False means upload 421 in a single request. 422 """ 423 super(MediaIoBaseUpload, self).__init__() 424 self._fd = fd 425 self._mimetype = mimetype 426 if not (chunksize == -1 or chunksize > 0): 427 raise InvalidChunkSizeError() 428 self._chunksize = chunksize 429 self._resumable = resumable 430 431 self._fd.seek(0, os.SEEK_END) 432 self._size = self._fd.tell()

433

434 - def chunksize(self):

435 """Chunk size for resumable uploads. 436 437 Returns: 438 Chunk size in bytes. 439 """ 440 return self._chunksize

441

442 - def mimetype(self):

443 """Mime type of the body. 444 445 Returns: 446 Mime type. 447 """ 448 return self._mimetype

449

450 - def size(self):

451 """Size of upload. 452 453 Returns: 454 Size of the body, or None of the size is unknown. 455 """ 456 return self._size

457

458 - def resumable(self):

459 """Whether this upload is resumable. 460 461 Returns: 462 True if resumable upload or False. 463 """ 464 return self._resumable

465

466 - def getbytes(self, begin, length):

467 """Get bytes from the media. 468 469 Args: 470 begin: int, offset from beginning of file. 471 length: int, number of bytes to read, starting at begin. 472 473 Returns: 474 A string of bytes read. May be shorted than length if EOF was reached 475 first. 476 """ 477 self._fd.seek(begin) 478 return self._fd.read(length)

479

480 - def has_stream(self):

481 """Does the underlying upload support a streaming interface. 482 483 Streaming means it is an io.IOBase subclass that supports seek, i.e. 484 seekable() returns True. 485 486 Returns: 487 True if the call to stream() will return an instance of a seekable io.Base 488 subclass. 489 """ 490 return True

491

492 - def stream(self):

493 """A stream interface to the data being uploaded. 494 495 Returns: 496 The returned value is an io.IOBase subclass that supports seek, i.e. 497 seekable() returns True. 498 """ 499 return self._fd

500

501 - def to_json(self):

502 """This upload type is not serializable.""" 503 raise NotImplementedError('MediaIoBaseUpload is not serializable.')

504

505 506 -class MediaFileUpload(MediaIoBaseUpload):

507 """A MediaUpload for a file. 508 509 Construct a MediaFileUpload and pass as the media_body parameter of the 510 method. For example, if we had a service that allowed uploading images: 511 512 513 media = MediaFileUpload('cow.png', mimetype='image/png', 514 chunksize=1024*1024, resumable=True) 515 farm.animals().insert( 516 id='cow', 517 name='cow.png', 518 media_body=media).execute() 519 520 Depending on the platform you are working on, you may pass -1 as the 521 chunksize, which indicates that the entire file should be uploaded in a single 522 request. If the underlying platform supports streams, such as Python 2.6 or 523 later, then this can be very efficient as it avoids multiple connections, and 524 also avoids loading the entire file into memory before sending it. Note that 525 Google App Engine has a 5MB limit on request size, so you should never set 526 your chunksize larger than 5MB, or to -1. 527 """ 528 529 @util.positional(2)

530 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, 531 resumable=False):

532 """Constructor. 533 534 Args: 535 filename: string, Name of the file. 536 mimetype: string, Mime-type of the file. If None then a mime-type will be 537 guessed from the file extension. 538 chunksize: int, File will be uploaded in chunks of this many bytes. Only 539 used if resumable=True. Pass in a value of -1 if the file is to be 540 uploaded in a single chunk. Note that Google App Engine has a 5MB limit 541 on request size, so you should never set your chunksize larger than 5MB, 542 or to -1. 543 resumable: bool, True if this is a resumable upload. False means upload 544 in a single request. 545 """ 546 self._filename = filename 547 fd = open(self._filename, 'rb') 548 if mimetype is None: 549 # No mimetype provided, make a guess. 550 mimetype, _ = mimetypes.guess_type(filename) 551 if mimetype is None: 552 # Guess failed, use octet-stream. 553 mimetype = 'application/octet-stream' 554 super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, 555 resumable=resumable)

556

557 - def to_json(self):

558 """Creating a JSON representation of an instance of MediaFileUpload. 559 560 Returns: 561 string, a JSON representation of this instance, suitable to pass to 562 from_json(). 563 """ 564 return self._to_json(strip=['_fd'])

565 566 @staticmethod

567 - def from_json(s):

568 d = json.loads(s) 569 return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'], 570 chunksize=d['_chunksize'], resumable=d['_resumable'])

571

572 573 -class MediaInMemoryUpload(MediaIoBaseUpload):

574 """MediaUpload for a chunk of bytes. 575 576 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 577 the stream. 578 """ 579 580 @util.positional(2)

581 - def __init__(self, body, mimetype='application/octet-stream', 582 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):

583 """Create a new MediaInMemoryUpload. 584 585 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 586 the stream. 587 588 Args: 589 body: string, Bytes of body content. 590 mimetype: string, Mime-type of the file or default of 591 'application/octet-stream'. 592 chunksize: int, File will be uploaded in chunks of this many bytes. Only 593 used if resumable=True. 594 resumable: bool, True if this is a resumable upload. False means upload 595 in a single request. 596 """ 597 fd = BytesIO(body) 598 super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize, 599 resumable=resumable)

600

601 602 -class MediaIoBaseDownload(object):

603 """"Download media resources. 604 605 Note that the Python file object is compatible with io.Base and can be used 606 with this class also. 607 608 609 Example: 610 request = farms.animals().get_media(id='cow') 611 fh = io.FileIO('cow.png', mode='wb') 612 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 613 614 done = False 615 while done is False: 616 status, done = downloader.next_chunk() 617 if status: 618 print "Download %d%%." % int(status.progress() * 100) 619 print "Download Complete!" 620 """ 621 622 @util.positional(3)

623 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):

624 """Constructor. 625 626 Args: 627 fd: io.Base or file object, The stream in which to write the downloaded 628 bytes. 629 request: googleapiclient.http.HttpRequest, the media request to perform in 630 chunks. 631 chunksize: int, File will be downloaded in chunks of this many bytes. 632 """ 633 self._fd = fd 634 self._request = request 635 self._uri = request.uri 636 self._chunksize = chunksize 637 self._progress = 0 638 self._total_size = None 639 self._done = False 640 641 # Stubs for testing. 642 self._sleep = time.sleep 643 self._rand = random.random

644 645 @util.positional(1)

646 - def next_chunk(self, num_retries=0):

647 """Get the next chunk of the download. 648 649 Args: 650 num_retries: Integer, number of times to retry with randomized 651 exponential backoff. If all retries fail, the raised HttpError 652 represents the last request. If zero (default), we attempt the 653 request only once. 654 655 Returns: 656 (status, done): (MediaDownloadStatus, boolean) 657 The value of 'done' will be True when the media has been fully 658 downloaded or the total size of the media is unknown. 659 660 Raises: 661 googleapiclient.errors.HttpError if the response was not a 2xx. 662 httplib2.HttpLib2Error if a transport error has occured. 663 """ 664 headers = { 665 'range': 'bytes=%d-%d' % ( 666 self._progress, self._progress + self._chunksize) 667 } 668 http = self._request.http 669 670 resp, content = _retry_request( 671 http, num_retries, 'media download', self._sleep, self._rand, self._uri, 672 'GET', headers=headers) 673 674 if resp.status in [200, 206]: 675 if 'content-location' in resp and resp['content-location'] != self._uri: 676 self._uri = resp['content-location'] 677 self._progress += len(content) 678 self._fd.write(content) 679 680 if 'content-range' in resp: 681 content_range = resp['content-range'] 682 length = content_range.rsplit('/', 1)[1] 683 self._total_size = int(length) 684 elif 'content-length' in resp: 685 self._total_size = int(resp['content-length']) 686 687 if self._total_size is None or self._progress == self._total_size: 688 self._done = True 689 return MediaDownloadProgress(self._progress, self._total_size), self._done 690 else: 691 raise HttpError(resp, content, uri=self._uri)

692

693 694 -class _StreamSlice(object):

695 """Truncated stream. 696 697 Takes a stream and presents a stream that is a slice of the original stream. 698 This is used when uploading media in chunks. In later versions of Python a 699 stream can be passed to httplib in place of the string of data to send. The 700 problem is that httplib just blindly reads to the end of the stream. This 701 wrapper presents a virtual stream that only reads to the end of the chunk. 702 """ 703

704 - def __init__(self, stream, begin, chunksize):

705 """Constructor. 706 707 Args: 708 stream: (io.Base, file object), the stream to wrap. 709 begin: int, the seek position the chunk begins at. 710 chunksize: int, the size of the chunk. 711 """ 712 self._stream = stream 713 self._begin = begin 714 self._chunksize = chunksize 715 self._stream.seek(begin)

716

717 - def read(self, n=-1):

718 """Read n bytes. 719 720 Args: 721 n, int, the number of bytes to read. 722 723 Returns: 724 A string of length 'n', or less if EOF is reached. 725 """ 726 # The data left available to read sits in [cur, end) 727 cur = self._stream.tell() 728 end = self._begin + self._chunksize 729 if n == -1 or cur + n > end: 730 n = end - cur 731 return self._stream.read(n)

732

733 734 -class HttpRequest(object):

735 """Encapsulates a single HTTP request.""" 736 737 @util.positional(4)

738 - def __init__(self, http, postproc, uri, 739 method='GET', 740 body=None, 741 headers=None, 742 methodId=None, 743 resumable=None):

744 """Constructor for an HttpRequest. 745 746 Args: 747 http: httplib2.Http, the transport object to use to make a request 748 postproc: callable, called on the HTTP response and content to transform 749 it into a data object before returning, or raising an exception 750 on an error. 751 uri: string, the absolute URI to send the request to 752 method: string, the HTTP method to use 753 body: string, the request body of the HTTP request, 754 headers: dict, the HTTP request headers 755 methodId: string, a unique identifier for the API method being called. 756 resumable: MediaUpload, None if this is not a resumbale request. 757 """ 758 self.uri = uri 759 self.method = method 760 self.body = body 761 self.headers = headers or {} 762 self.methodId = methodId 763 self.http = http 764 self.postproc = postproc 765 self.resumable = resumable 766 self.response_callbacks = [] 767 self._in_error_state = False 768 769 # Pull the multipart boundary out of the content-type header. 770 major, minor, params = mimeparse.parse_mime_type( 771 self.headers.get('content-type', 'application/json')) 772 773 # The size of the non-media part of the request. 774 self.body_size = len(self.body or '') 775 776 # The resumable URI to send chunks to. 777 self.resumable_uri = None 778 779 # The bytes that have been uploaded. 780 self.resumable_progress = 0 781 782 # Stubs for testing. 783 self._rand = random.random 784 self._sleep = time.sleep

785 786 @util.positional(1)

787 - def execute(self, http=None, num_retries=0):

788 """Execute the request. 789 790 Args: 791 http: httplib2.Http, an http object to be used in place of the 792 one the HttpRequest request object was constructed with. 793 num_retries: Integer, number of times to retry with randomized 794 exponential backoff. If all retries fail, the raised HttpError 795 represents the last request. If zero (default), we attempt the 796 request only once. 797 798 Returns: 799 A deserialized object model of the response body as determined 800 by the postproc. 801 802 Raises: 803 googleapiclient.errors.HttpError if the response was not a 2xx. 804 httplib2.HttpLib2Error if a transport error has occured. 805 """ 806 if http is None: 807 http = self.http 808 809 if self.resumable: 810 body = None 811 while body is None: 812 _, body = self.next_chunk(http=http, num_retries=num_retries) 813 return body 814 815 # Non-resumable case. 816 817 if 'content-length' not in self.headers: 818 self.headers['content-length'] = str(self.body_size) 819 # If the request URI is too long then turn it into a POST request. 820 # Assume that a GET request never contains a request body. 821 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': 822 self.method = 'POST' 823 self.headers['x-http-method-override'] = 'GET' 824 self.headers['content-type'] = 'application/x-www-form-urlencoded' 825 parsed = urlparse(self.uri) 826 self.uri = urlunparse( 827 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, 828 None) 829 ) 830 self.body = parsed.query 831 self.headers['content-length'] = str(len(self.body)) 832 833 # Handle retries for server-side errors. 834 resp, content = _retry_request( 835 http, num_retries, 'request', self._sleep, self._rand, str(self.uri), 836 method=str(self.method), body=self.body, headers=self.headers) 837 838 for callback in self.response_callbacks: 839 callback(resp) 840 if resp.status >= 300: 841 raise HttpError(resp, content, uri=self.uri) 842 return self.postproc(resp, content)

843 844 @util.positional(2)

845 - def add_response_callback(self, cb):

846 """add_response_headers_callback 847 848 Args: 849 cb: Callback to be called on receiving the response headers, of signature: 850 851 def cb(resp): 852 # Where resp is an instance of httplib2.Response 853 """ 854 self.response_callbacks.append(cb)

855 856 @util.positional(1)

857 - def next_chunk(self, http=None, num_retries=0):

858 """Execute the next step of a resumable upload. 859 860 Can only be used if the method being executed supports media uploads and 861 the MediaUpload object passed in was flagged as using resumable upload. 862 863 Example: 864 865 media = MediaFileUpload('cow.png', mimetype='image/png', 866 chunksize=1000, resumable=True) 867 request = farm.animals().insert( 868 id='cow', 869 name='cow.png', 870 media_body=media) 871 872 response = None 873 while response is None: 874 status, response = request.next_chunk() 875 if status: 876 print "Upload %d%% complete." % int(status.progress() * 100) 877 878 879 Args: 880 http: httplib2.Http, an http object to be used in place of the 881 one the HttpRequest request object was constructed with. 882 num_retries: Integer, number of times to retry with randomized 883 exponential backoff. If all retries fail, the raised HttpError 884 represents the last request. If zero (default), we attempt the 885 request only once. 886 887 Returns: 888 (status, body): (ResumableMediaStatus, object) 889 The body will be None until the resumable media is fully uploaded. 890 891 Raises: 892 googleapiclient.errors.HttpError if the response was not a 2xx. 893 httplib2.HttpLib2Error if a transport error has occured. 894 """ 895 if http is None: 896 http = self.http 897 898 if self.resumable.size() is None: 899 size = '*' 900 else: 901 size = str(self.resumable.size()) 902 903 if self.resumable_uri is None: 904 start_headers = copy.copy(self.headers) 905 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 906 if size != '*': 907 start_headers['X-Upload-Content-Length'] = size 908 start_headers['content-length'] = str(self.body_size) 909 910 resp, content = _retry_request( 911 http, num_retries, 'resumable URI request', self._sleep, self._rand, 912 self.uri, method=self.method, body=self.body, headers=start_headers) 913 914 if resp.status == 200 and 'location' in resp: 915 self.resumable_uri = resp['location'] 916 else: 917 raise ResumableUploadError(resp, content) 918 elif self._in_error_state: 919 # If we are in an error state then query the server for current state of 920 # the upload by sending an empty PUT and reading the 'range' header in 921 # the response. 922 headers = { 923 'Content-Range': 'bytes */%s' % size, 924 'content-length': '0' 925 } 926 resp, content = http.request(self.resumable_uri, 'PUT', 927 headers=headers) 928 status, body = self._process_response(resp, content) 929 if body: 930 # The upload was complete. 931 return (status, body) 932 933 if self.resumable.has_stream(): 934 data = self.resumable.stream() 935 if self.resumable.chunksize() == -1: 936 data.seek(self.resumable_progress) 937 chunk_end = self.resumable.size() - self.resumable_progress - 1 938 else: 939 # Doing chunking with a stream, so wrap a slice of the stream. 940 data = _StreamSlice(data, self.resumable_progress, 941 self.resumable.chunksize()) 942 chunk_end = min( 943 self.resumable_progress + self.resumable.chunksize() - 1, 944 self.resumable.size() - 1) 945 else: 946 data = self.resumable.getbytes( 947 self.resumable_progress, self.resumable.chunksize()) 948 949 # A short read implies that we are at EOF, so finish the upload. 950 if len(data) < self.resumable.chunksize(): 951 size = str(self.resumable_progress + len(data)) 952 953 chunk_end = self.resumable_progress + len(data) - 1 954 955 headers = { 956 'Content-Range': 'bytes %d-%d/%s' % ( 957 self.resumable_progress, chunk_end, size), 958 # Must set the content-length header here because httplib can't 959 # calculate the size when working with _StreamSlice. 960 'Content-Length': str(chunk_end - self.resumable_progress + 1) 961 } 962 963 for retry_num in range(num_retries + 1): 964 if retry_num > 0: 965 self._sleep(self._rand() * 2**retry_num) 966 LOGGER.warning( 967 'Retry #%d for media upload: %s %s, following status: %d' 968 % (retry_num, self.method, self.uri, resp.status)) 969 970 try: 971 resp, content = http.request(self.resumable_uri, method='PUT', 972 body=data, 973 headers=headers) 974 except: 975 self._in_error_state = True 976 raise 977 if not _should_retry_response(resp.status, content): 978 break 979 980 return self._process_response(resp, content)

981

982 - def _process_response(self, resp, content):

983 """Process the response from a single chunk upload. 984 985 Args: 986 resp: httplib2.Response, the response object. 987 content: string, the content of the response. 988 989 Returns: 990 (status, body): (ResumableMediaStatus, object) 991 The body will be None until the resumable media is fully uploaded. 992 993 Raises: 994 googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 995 """ 996 if resp.status in [200, 201]: 997 self._in_error_state = False 998 return None, self.postproc(resp, content) 999 elif resp.status == 308: 1000 self._in_error_state = False 1001 # A "308 Resume Incomplete" indicates we are not done. 1002 try: 1003 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 1004 except KeyError: 1005 # If resp doesn't contain range header, resumable progress is 0 1006 self.resumable_progress = 0 1007 if 'location' in resp: 1008 self.resumable_uri = resp['location'] 1009 else: 1010 self._in_error_state = True 1011 raise HttpError(resp, content, uri=self.uri) 1012 1013 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 1014 None)

1015

1016 - def to_json(self):

1017 """Returns a JSON representation of the HttpRequest.""" 1018 d = copy.copy(self.__dict__) 1019 if d['resumable'] is not None: 1020 d['resumable'] = self.resumable.to_json() 1021 del d['http'] 1022 del d['postproc'] 1023 del d['_sleep'] 1024 del d['_rand'] 1025 1026 return json.dumps(d)

1027 1028 @staticmethod

1029 - def from_json(s, http, postproc):

1030 """Returns an HttpRequest populated with info from a JSON object.""" 1031 d = json.loads(s) 1032 if d['resumable'] is not None: 1033 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 1034 return HttpRequest( 1035 http, 1036 postproc, 1037 uri=d['uri'], 1038 method=d['method'], 1039 body=d['body'], 1040 headers=d['headers'], 1041 methodId=d['methodId'], 1042 resumable=d['resumable'])

1043

1044 1045 -class BatchHttpRequest(object):

1046 """Batches multiple HttpRequest objects into a single HTTP request. 1047 1048 Example: 1049 from googleapiclient.http import BatchHttpRequest 1050 1051 def list_animals(request_id, response, exception): 1052 \"\"\"Do something with the animals list response.\"\"\" 1053 if exception is not None: 1054 # Do something with the exception. 1055 pass 1056 else: 1057 # Do something with the response. 1058 pass 1059 1060 def list_farmers(request_id, response, exception): 1061 \"\"\"Do something with the farmers list response.\"\"\" 1062 if exception is not None: 1063 # Do something with the exception. 1064 pass 1065 else: 1066 # Do something with the response. 1067 pass 1068 1069 service = build('farm', 'v2') 1070 1071 batch = BatchHttpRequest() 1072 1073 batch.add(service.animals().list(), list_animals) 1074 batch.add(service.farmers().list(), list_farmers) 1075 batch.execute(http=http) 1076 """ 1077 1078 @util.positional(1)

1079 - def __init__(self, callback=None, batch_uri=None):

1080 """Constructor for a BatchHttpRequest. 1081 1082 Args: 1083 callback: callable, A callback to be called for each response, of the 1084 form callback(id, response, exception). The first parameter is the 1085 request id, and the second is the deserialized response object. The 1086 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1087 occurred while processing the request, or None if no error occurred. 1088 batch_uri: string, URI to send batch requests to. 1089 """ 1090 if batch_uri is None: 1091 batch_uri = 'https://www.googleapis.com/batch' 1092 self._batch_uri = batch_uri 1093 1094 # Global callback to be called for each individual response in the batch. 1095 self._callback = callback 1096 1097 # A map from id to request. 1098 self._requests = {} 1099 1100 # A map from id to callback. 1101 self._callbacks = {} 1102 1103 # List of request ids, in the order in which they were added. 1104 self._order = [] 1105 1106 # The last auto generated id. 1107 self._last_auto_id = 0 1108 1109 # Unique ID on which to base the Content-ID headers. 1110 self._base_id = None 1111 1112 # A map from request id to (httplib2.Response, content) response pairs 1113 self._responses = {} 1114 1115 # A map of id(Credentials) that have been refreshed. 1116 self._refreshed_credentials = {}

1117

1118 - def _refresh_and_apply_credentials(self, request, http):

1119 """Refresh the credentials and apply to the request. 1120 1121 Args: 1122 request: HttpRequest, the request. 1123 http: httplib2.Http, the global http object for the batch. 1124 """ 1125 # For the credentials to refresh, but only once per refresh_token 1126 # If there is no http per the request then refresh the http passed in 1127 # via execute() 1128 creds = None 1129 if request.http is not None and hasattr(request.http.request, 1130 'credentials'): 1131 creds = request.http.request.credentials 1132 elif http is not None and hasattr(http.request, 'credentials'): 1133 creds = http.request.credentials 1134 if creds is not None: 1135 if id(creds) not in self._refreshed_credentials: 1136 creds.refresh(http) 1137 self._refreshed_credentials[id(creds)] = 1 1138 1139 # Only apply the credentials if we are using the http object passed in, 1140 # otherwise apply() will get called during _serialize_request(). 1141 if request.http is None or not hasattr(request.http.request, 1142 'credentials'): 1143 creds.apply(request.headers)

1144

1145 - def _id_to_header(self, id_):

1146 """Convert an id to a Content-ID header value. 1147 1148 Args: 1149 id_: string, identifier of individual request. 1150 1151 Returns: 1152 A Content-ID header with the id_ encoded into it. A UUID is prepended to 1153 the value because Content-ID headers are supposed to be universally 1154 unique. 1155 """ 1156 if self._base_id is None: 1157 self._base_id = uuid.uuid4() 1158 1159 return '<%s+%s>' % (self._base_id, quote(id_))

1160

1161 - def _header_to_id(self, header):

1162 """Convert a Content-ID header value to an id. 1163 1164 Presumes the Content-ID header conforms to the format that _id_to_header() 1165 returns. 1166 1167 Args: 1168 header: string, Content-ID header value. 1169 1170 Returns: 1171 The extracted id value. 1172 1173 Raises: 1174 BatchError if the header is not in the expected format. 1175 """ 1176 if header[0] != '<' or header[-1] != '>': 1177 raise BatchError("Invalid value for Content-ID: %s" % header) 1178 if '+' not in header: 1179 raise BatchError("Invalid value for Content-ID: %s" % header) 1180 base, id_ = header[1:-1].rsplit('+', 1) 1181 1182 return unquote(id_)

1183

1184 - def _serialize_request(self, request):

1185 """Convert an HttpRequest object into a string. 1186 1187 Args: 1188 request: HttpRequest, the request to serialize. 1189 1190 Returns: 1191 The request as a string in application/http format. 1192 """ 1193 # Construct status line 1194 parsed = urlparse(request.uri) 1195 request_line = urlunparse( 1196 ('', '', parsed.path, parsed.params, parsed.query, '') 1197 ) 1198 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 1199 major, minor = request.headers.get('content-type', 'application/json').split('/') 1200 msg = MIMENonMultipart(major, minor) 1201 headers = request.headers.copy() 1202 1203 if request.http is not None and hasattr(request.http.request, 1204 'credentials'): 1205 request.http.request.credentials.apply(headers) 1206 1207 # MIMENonMultipart adds its own Content-Type header. 1208 if 'content-type' in headers: 1209 del headers['content-type'] 1210 1211 for key, value in six.iteritems(headers): 1212 msg[key] = value 1213 msg['Host'] = parsed.netloc 1214 msg.set_unixfrom(None) 1215 1216 if request.body is not None: 1217 msg.set_payload(request.body) 1218 msg['content-length'] = str(len(request.body)) 1219 1220 # Serialize the mime message. 1221 fp = StringIO() 1222 # maxheaderlen=0 means don't line wrap headers. 1223 g = Generator(fp, maxheaderlen=0) 1224 g.flatten(msg, unixfrom=False) 1225 body = fp.getvalue() 1226 1227 return status_line + body

1228

1229 - def _deserialize_response(self, payload):

1230 """Convert string into httplib2 response and content. 1231 1232 Args: 1233 payload: string, headers and body as a string. 1234 1235 Returns: 1236 A pair (resp, content), such as would be returned from httplib2.request. 1237 """ 1238 # Strip off the status line 1239 status_line, payload = payload.split('\n', 1) 1240 protocol, status, reason = status_line.split(' ', 2) 1241 1242 # Parse the rest of the response 1243 parser = FeedParser() 1244 parser.feed(payload) 1245 msg = parser.close() 1246 msg['status'] = status 1247 1248 # Create httplib2.Response from the parsed headers. 1249 resp = httplib2.Response(msg) 1250 resp.reason = reason 1251 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1252 1253 content = payload.split('\r\n\r\n', 1)[1] 1254 1255 return resp, content

1256

1257 - def _new_id(self):

1258 """Create a new id. 1259 1260 Auto incrementing number that avoids conflicts with ids already used. 1261 1262 Returns: 1263 string, a new unique id. 1264 """ 1265 self._last_auto_id += 1 1266 while str(self._last_auto_id) in self._requests: 1267 self._last_auto_id += 1 1268 return str(self._last_auto_id)

1269 1270 @util.positional(2)

1271 - def add(self, request, callback=None, request_id=None):

1272 """Add a new request. 1273 1274 Every callback added will be paired with a unique id, the request_id. That 1275 unique id will be passed back to the callback when the response comes back 1276 from the server. The default behavior is to have the library generate it's 1277 own unique id. If the caller passes in a request_id then they must ensure 1278 uniqueness for each request_id, and if they are not an exception is 1279 raised. Callers should either supply all request_ids or nevery supply a 1280 request id, to avoid such an error. 1281 1282 Args: 1283 request: HttpRequest, Request to add to the batch. 1284 callback: callable, A callback to be called for this response, of the 1285 form callback(id, response, exception). The first parameter is the 1286 request id, and the second is the deserialized response object. The 1287 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1288 occurred while processing the request, or None if no errors occurred. 1289 request_id: string, A unique id for the request. The id will be passed to 1290 the callback with the response. 1291 1292 Returns: 1293 None 1294 1295 Raises: 1296 BatchError if a media request is added to a batch. 1297 KeyError is the request_id is not unique. 1298 """ 1299 if request_id is None: 1300 request_id = self._new_id() 1301 if request.resumable is not None: 1302 raise BatchError("Media requests cannot be used in a batch request.") 1303 if request_id in self._requests: 1304 raise KeyError("A request with this ID already exists: %s" % request_id) 1305 self._requests[request_id] = request 1306 self._callbacks[request_id] = callback 1307 self._order.append(request_id)

1308

1309 - def _execute(self, http, order, requests):

1310 """Serialize batch request, send to server, process response. 1311 1312 Args: 1313 http: httplib2.Http, an http object to be used to make the request with. 1314 order: list, list of request ids in the order they were added to the 1315 batch. 1316 request: list, list of request objects to send. 1317 1318 Raises: 1319 httplib2.HttpLib2Error if a transport error has occured. 1320 googleapiclient.errors.BatchError if the response is the wrong format. 1321 """ 1322 message = MIMEMultipart('mixed') 1323 # Message should not write out it's own headers. 1324 setattr(message, '_write_headers', lambda self: None) 1325 1326 # Add all the individual requests. 1327 for request_id in order: 1328 request = requests[request_id] 1329 1330 msg = MIMENonMultipart('application', 'http') 1331 msg['Content-Transfer-Encoding'] = 'binary' 1332 msg['Content-ID'] = self._id_to_header(request_id) 1333 1334 body = self._serialize_request(request) 1335 msg.set_payload(body) 1336 message.attach(msg) 1337 1338 # encode the body: note that we can't use `as_string`, because 1339 # it plays games with `From ` lines. 1340 fp = StringIO() 1341 g = Generator(fp, mangle_from_=False) 1342 g.flatten(message, unixfrom=False) 1343 body = fp.getvalue() 1344 1345 headers = {} 1346 headers['content-type'] = ('multipart/mixed; ' 1347 'boundary="%s"') % message.get_boundary() 1348 1349 resp, content = http.request(self._batch_uri, method='POST', body=body, 1350 headers=headers) 1351 1352 if resp.status >= 300: 1353 raise HttpError(resp, content, uri=self._batch_uri) 1354 1355 # Prepend with a content-type header so FeedParser can handle it. 1356 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1357 # PY3's FeedParser only accepts unicode. So we should decode content 1358 # here, and encode each payload again. 1359 if six.PY3: 1360 content = content.decode('utf-8') 1361 for_parser = header + content 1362 1363 parser = FeedParser() 1364 parser.feed(for_parser) 1365 mime_response = parser.close() 1366 1367 if not mime_response.is_multipart(): 1368 raise BatchError("Response not in multipart/mixed format.", resp=resp, 1369 content=content) 1370 1371 for part in mime_response.get_payload(): 1372 request_id = self._header_to_id(part['Content-ID']) 1373 response, content = self._deserialize_response(part.get_payload()) 1374 # We encode content here to emulate normal http response. 1375 if isinstance(content, six.text_type): 1376 content = content.encode('utf-8') 1377 self._responses[request_id] = (response, content)

1378 1379 @util.positional(1)

1380 - def execute(self, http=None):

1381 """Execute all the requests as a single batched HTTP request. 1382 1383 Args: 1384 http: httplib2.Http, an http object to be used in place of the one the 1385 HttpRequest request object was constructed with. If one isn't supplied 1386 then use a http object from the requests in this batch. 1387 1388 Returns: 1389 None 1390 1391 Raises: 1392 httplib2.HttpLib2Error if a transport error has occured. 1393 googleapiclient.errors.BatchError if the response is the wrong format. 1394 """ 1395 # If we have no requests return 1396 if len(self._order) == 0: 1397 return None 1398 1399 # If http is not supplied use the first valid one given in the requests. 1400 if http is None: 1401 for request_id in self._order: 1402 request = self._requests[request_id] 1403 if request is not None: 1404 http = request.http 1405 break 1406 1407 if http is None: 1408 raise ValueError("Missing a valid http object.") 1409 1410 # Special case for OAuth2Credentials-style objects which have not yet been 1411 # refreshed with an initial access_token. 1412 if getattr(http.request, 'credentials', None) is not None: 1413 creds = http.request.credentials 1414 if not getattr(creds, 'access_token', None): 1415 LOGGER.info('Attempting refresh to obtain initial access_token') 1416 creds.refresh(http) 1417 1418 self._execute(http, self._order, self._requests) 1419 1420 # Loop over all the requests and check for 401s. For each 401 request the 1421 # credentials should be refreshed and then sent again in a separate batch. 1422 redo_requests = {} 1423 redo_order = [] 1424 1425 for request_id in self._order: 1426 resp, content = self._responses[request_id] 1427 if resp['status'] == '401': 1428 redo_order.append(request_id) 1429 request = self._requests[request_id] 1430 self._refresh_and_apply_credentials(request, http) 1431 redo_requests[request_id] = request 1432 1433 if redo_requests: 1434 self._execute(http, redo_order, redo_requests) 1435 1436 # Now process all callbacks that are erroring, and raise an exception for 1437 # ones that return a non-2xx response? Or add extra parameter to callback 1438 # that contains an HttpError? 1439 1440 for request_id in self._order: 1441 resp, content = self._responses[request_id] 1442 1443 request = self._requests[request_id] 1444 callback = self._callbacks[request_id] 1445 1446 response = None 1447 exception = None 1448 try: 1449 if resp.status >= 300: 1450 raise HttpError(resp, content, uri=request.uri) 1451 response = request.postproc(resp, content) 1452 except HttpError as e: 1453 exception = e 1454 1455 if callback is not None: 1456 callback(request_id, response, exception) 1457 if self._callback is not None: 1458 self._callback(request_id, response, exception)

1459

1460 1461 -class HttpRequestMock(object):

1462 """Mock of HttpRequest. 1463 1464 Do not construct directly, instead use RequestMockBuilder. 1465 """ 1466

1467 - def __init__(self, resp, content, postproc):

1468 """Constructor for HttpRequestMock 1469 1470 Args: 1471 resp: httplib2.Response, the response to emulate coming from the request 1472 content: string, the response body 1473 postproc: callable, the post processing function usually supplied by 1474 the model class. See model.JsonModel.response() as an example. 1475 """ 1476 self.resp = resp 1477 self.content = content 1478 self.postproc = postproc 1479 if resp is None: 1480 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1481 if 'reason' in self.resp: 1482 self.resp.reason = self.resp['reason']

1483

1484 - def execute(self, http=None):

1485 """Execute the request. 1486 1487 Same behavior as HttpRequest.execute(), but the response is 1488 mocked and not really from an HTTP request/response. 1489 """ 1490 return self.postproc(self.resp, self.content)

1491

1492 1493 -class RequestMockBuilder(object):

1494 """A simple mock of HttpRequest 1495 1496 Pass in a dictionary to the constructor that maps request methodIds to 1497 tuples of (httplib2.Response, content, opt_expected_body) that should be 1498 returned when that method is called. None may also be passed in for the 1499 httplib2.Response, in which case a 200 OK response will be generated. 1500 If an opt_expected_body (str or dict) is provided, it will be compared to 1501 the body and UnexpectedBodyError will be raised on inequality. 1502 1503 Example: 1504 response = '{"data": {"id": "tag:google.c...' 1505 requestBuilder = RequestMockBuilder( 1506 { 1507 'plus.activities.get': (None, response), 1508 } 1509 ) 1510 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1511 1512 Methods that you do not supply a response for will return a 1513 200 OK with an empty string as the response content or raise an excpetion 1514 if check_unexpected is set to True. The methodId is taken from the rpcName 1515 in the discovery document. 1516 1517 For more details see the project wiki. 1518 """ 1519

1520 - def __init__(self, responses, check_unexpected=False):

1521 """Constructor for RequestMockBuilder 1522 1523 The constructed object should be a callable object 1524 that can replace the class HttpResponse. 1525 1526 responses - A dictionary that maps methodIds into tuples 1527 of (httplib2.Response, content). The methodId 1528 comes from the 'rpcName' field in the discovery 1529 document. 1530 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1531 should be raised on unsupplied method. 1532 """ 1533 self.responses = responses 1534 self.check_unexpected = check_unexpected

1535

1536 - def __call__(self, http, postproc, uri, method='GET', body=None, 1537 headers=None, methodId=None, resumable=None):

1538 """Implements the callable interface that discovery.build() expects 1539 of requestBuilder, which is to build an object compatible with 1540 HttpRequest.execute(). See that method for the description of the 1541 parameters and the expected response. 1542 """ 1543 if methodId in self.responses: 1544 response = self.responses[methodId] 1545 resp, content = response[:2] 1546 if len(response) > 2: 1547 # Test the body against the supplied expected_body. 1548 expected_body = response[2] 1549 if bool(expected_body) != bool(body): 1550 # Not expecting a body and provided one 1551 # or expecting a body and not provided one. 1552 raise UnexpectedBodyError(expected_body, body) 1553 if isinstance(expected_body, str): 1554 expected_body = json.loads(expected_body) 1555 body = json.loads(body) 1556 if body != expected_body: 1557 raise UnexpectedBodyError(expected_body, body) 1558 return HttpRequestMock(resp, content, postproc) 1559 elif self.check_unexpected: 1560 raise UnexpectedMethodError(methodId=methodId) 1561 else: 1562 model = JsonModel(False) 1563 return HttpRequestMock(None, '{}', model.response)

1564

1565 1566 -class HttpMock(object):

1567 """Mock of httplib2.Http""" 1568

1569 - def __init__(self, filename=None, headers=None):

1570 """ 1571 Args: 1572 filename: string, absolute filename to read response from 1573 headers: dict, header to return with response 1574 """ 1575 if headers is None: 1576 headers = {'status': '200'} 1577 if filename: 1578 f = open(filename, 'rb') 1579 self.data = f.read() 1580 f.close() 1581 else: 1582 self.data = None 1583 self.response_headers = headers 1584 self.headers = None 1585 self.uri = None 1586 self.method = None 1587 self.body = None 1588 self.headers = None

1589 1590

1591 - def request(self, uri, 1592 method='GET', 1593 body=None, 1594 headers=None, 1595 redirections=1, 1596 connection_type=None):

1597 self.uri = uri 1598 self.method = method 1599 self.body = body 1600 self.headers = headers 1601 return httplib2.Response(self.response_headers), self.data

1602

1603 1604 -class HttpMockSequence(object):

1605 """Mock of httplib2.Http 1606 1607 Mocks a sequence of calls to request returning different responses for each 1608 call. Create an instance initialized with the desired response headers 1609 and content and then use as if an httplib2.Http instance. 1610 1611 http = HttpMockSequence([ 1612 ({'status': '401'}, ''), 1613 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1614 ({'status': '200'}, 'echo_request_headers'), 1615 ]) 1616 resp, content = http.request("http://examples.com") 1617 1618 There are special values you can pass in for content to trigger 1619 behavours that are helpful in testing. 1620 1621 'echo_request_headers' means return the request headers in the response body 1622 'echo_request_headers_as_json' means return the request headers in 1623 the response body 1624 'echo_request_body' means return the request body in the response body 1625 'echo_request_uri' means return the request uri in the response body 1626 """ 1627

1628 - def __init__(self, iterable):

1629 """ 1630 Args: 1631 iterable: iterable, a sequence of pairs of (headers, body) 1632 """ 1633 self._iterable = iterable 1634 self.follow_redirects = True

1635

1636 - def request(self, uri, 1637 method='GET', 1638 body=None, 1639 headers=None, 1640 redirections=1, 1641 connection_type=None):

1642 resp, content = self._iterable.pop(0) 1643 if content == 'echo_request_headers': 1644 content = headers 1645 elif content == 'echo_request_headers_as_json': 1646 content = json.dumps(headers) 1647 elif content == 'echo_request_body': 1648 if hasattr(body, 'read'): 1649 content = body.read() 1650 else: 1651 content = body 1652 elif content == 'echo_request_uri': 1653 content = uri 1654 if isinstance(content, six.text_type): 1655 content = content.encode('utf-8') 1656 return httplib2.Response(resp), content

1657

1658 1659 -def set_user_agent(http, user_agent):

1660 """Set the user-agent on every request. 1661 1662 Args: 1663 http - An instance of httplib2.Http 1664 or something that acts like it. 1665 user_agent: string, the value for the user-agent header. 1666 1667 Returns: 1668 A modified instance of http that was passed in. 1669 1670 Example: 1671 1672 h = httplib2.Http() 1673 h = set_user_agent(h, "my-app-name/6.0") 1674 1675 Most of the time the user-agent will be set doing auth, this is for the rare 1676 cases where you are accessing an unauthenticated endpoint. 1677 """ 1678 request_orig = http.request 1679 1680 # The closure that will replace 'httplib2.Http.request'. 1681 def new_request(uri, method='GET', body=None, headers=None, 1682 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1683 connection_type=None): 1684 """Modify the request headers to add the user-agent.""" 1685 if headers is None: 1686 headers = {} 1687 if 'user-agent' in headers: 1688 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1689 else: 1690 headers['user-agent'] = user_agent 1691 resp, content = request_orig(uri, method, body, headers, 1692 redirections, connection_type) 1693 return resp, content

1694 1695 http.request = new_request 1696 return http 1697

1698 1699 -def tunnel_patch(http):

1700 """Tunnel PATCH requests over POST. 1701 Args: 1702 http - An instance of httplib2.Http 1703 or something that acts like it. 1704 1705 Returns: 1706 A modified instance of http that was passed in. 1707 1708 Example: 1709 1710 h = httplib2.Http() 1711 h = tunnel_patch(h, "my-app-name/6.0") 1712 1713 Useful if you are running on a platform that doesn't support PATCH. 1714 Apply this last if you are using OAuth 1.0, as changing the method 1715 will result in a different signature. 1716 """ 1717 request_orig = http.request 1718 1719 # The closure that will replace 'httplib2.Http.request'. 1720 def new_request(uri, method='GET', body=None, headers=None, 1721 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1722 connection_type=None): 1723 """Modify the request headers to add the user-agent.""" 1724 if headers is None: 1725 headers = {} 1726 if method == 'PATCH': 1727 if 'oauth_token' in headers.get('authorization', ''): 1728 LOGGER.warning( 1729 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1730 headers['x-http-method-override'] = "PATCH" 1731 method = 'POST' 1732 resp, content = request_orig(uri, method, body, headers, 1733 redirections, connection_type) 1734 return resp, content

1735 1736 http.request = new_request 1737 return http 1738

1739 1740 -def build_http():

1741 """Builds httplib2.Http object 1742 1743 Returns: 1744 A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 1745 To override default timeout call 1746 1747 socket.setdefaulttimeout(timeout_in_sec) 1748 1749 before interacting with this method. 1750 """ 1751 if socket.getdefaulttimeout() is not None: 1752 http_timeout = socket.getdefaulttimeout() 1753 else: 1754 http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 1755 return httplib2.Http(timeout=http_timeout)

1756

Source Code for Module googleapiclient.http