googleapiclient.http

Package googleapiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.http

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actuall HTTP request.
      20  """
      21  from __future__ import absolute_import
      22  import six
      23  from six.moves import http_client
      24  from six.moves import range
      25
      26  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      27
      28  from six import BytesIO, StringIO
      29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
      30
      31  import base64
      32  import copy
      33  import gzip
      34  import httplib2
      35  import json
      36  import logging
      37  import mimetypes
      38  import os
      39  import random
      40  import socket
      41  import sys
      42  import time
      43  import uuid
      44
      45  # TODO(issue 221): Remove this conditional import jibbajabba.
      46  try:
      47    import ssl
      48  except ImportError:
      49    _ssl_SSLError = object()
      50  else:
      51    _ssl_SSLError = ssl.SSLError
      52
      53  from email.generator import Generator
      54  from email.mime.multipart import MIMEMultipart
      55  from email.mime.nonmultipart import MIMENonMultipart
      56  from email.parser import FeedParser
      57
      58  # Oauth2client < 3 has the positional helper in 'util', >= 3 has it
      59  # in '_helpers'.
      60  try:
      61    from oauth2client import util
      62  except ImportError:
      63    from oauth2client import _helpers as util
      64
      65  from googleapiclient import mimeparse
      66  from googleapiclient.errors import BatchError
      67  from googleapiclient.errors import HttpError
      68  from googleapiclient.errors import InvalidChunkSizeError
      69  from googleapiclient.errors import ResumableUploadError
      70  from googleapiclient.errors import UnexpectedBodyError
      71  from googleapiclient.errors import UnexpectedMethodError
      72  from googleapiclient.model import JsonModel
      73
      74
      75  LOGGER = logging.getLogger(__name__)
      76
      77  DEFAULT_CHUNK_SIZE = 512*1024
      78
      79  MAX_URI_LENGTH = 2048
      80
      81  _TOO_MANY_REQUESTS = 429
      82
      83  DEFAULT_HTTP_TIMEOUT_SEC = 60
      84
      85
      86 -def _should_retry_response(resp_status, content):
      87    """Determines whether a response should be retried.
      88
      89    Args:
      90      resp_status: The response status received.
      91      content: The response content body.
      92
      93    Returns:
      94      True if the response should be retried, otherwise False.
      95    """
      96    # Retry on 5xx errors.
      97    if resp_status >= 500:
      98      return True
      99
     100    # Retry on 429 errors.
     101    if resp_status == _TOO_MANY_REQUESTS:
     102      return True
     103
     104    # For 403 errors, we have to check for the `reason` in the response to
     105    # determine if we should retry.
     106    if resp_status == six.moves.http_client.FORBIDDEN:
     107      # If there's no details about the 403 type, don't retry.
     108      if not content:
     109        return False
     110
     111      # Content is in JSON format.
     112      try:
     113        data = json.loads(content.decode('utf-8'))
     114        reason = data['error']['errors'][0]['reason']
     115      except (UnicodeDecodeError, ValueError, KeyError):
     116        LOGGER.warning('Invalid JSON content from response: %s', content)
     117        return False
     118
     119      LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
     120
     121      # Only retry on rate limit related failures.
     122      if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
     123        return True
     124
     125    # Everything else is a success or non-retriable so break.
     126    return False
     127
     128
     129 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
     130                     **kwargs):
     131    """Retries an HTTP request multiple times while handling errors.
     132
     133    If after all retries the request still fails, last error is either returned as
     134    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
     135
     136    Args:
     137      http: Http object to be used to execute request.
     138      num_retries: Maximum number of retries.
     139      req_type: Type of the request (used for logging retries).
     140      sleep, rand: Functions to sleep for random time between retries.
     141      uri: URI to be requested.
     142      method: HTTP method to be used.
     143      args, kwargs: Additional arguments passed to http.request.
     144
     145    Returns:
     146      resp, content - Response from the http request (may be HTTP 5xx).
     147    """
     148    resp = None
     149    content = None
     150    for retry_num in range(num_retries + 1):
     151      if retry_num > 0:
     152        # Sleep before retrying.
     153        sleep_time = rand() * 2 ** retry_num
     154        LOGGER.warning(
     155            'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
     156            sleep_time, retry_num, num_retries, req_type, method, uri,
     157            resp.status if resp else exception)
     158        sleep(sleep_time)
     159
     160      try:
     161        exception = None
     162        resp, content = http.request(uri, method, *args, **kwargs)
     163      # Retry on SSL errors and socket timeout errors.
     164      except _ssl_SSLError as ssl_error:
     165        exception = ssl_error
     166      except socket.error as socket_error:
     167        # errno's contents differ by platform, so we have to match by name.
     168        if socket.errno.errorcode.get(socket_error.errno) not in (
     169            'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
     170          raise
     171        exception = socket_error
     172
     173      if exception:
     174        if retry_num == num_retries:
     175          raise exception
     176        else:
     177          continue
     178
     179      if not _should_retry_response(resp.status, content):
     180        break
     181
     182    return resp, content
     183
     184
     185 -class MediaUploadProgress(object):
     186    """Status of a resumable upload."""
     187
     188 -  def __init__(self, resumable_progress, total_size):
     189      """Constructor.
     190
     191      Args:
     192        resumable_progress: int, bytes sent so far.
     193        total_size: int, total bytes in complete upload, or None if the total
     194          upload size isn't known ahead of time.
     195      """
     196      self.resumable_progress = resumable_progress
     197      self.total_size = total_size
     198
     199 -  def progress(self):
     200      """Percent of upload completed, as a float.
     201
     202      Returns:
     203        the percentage complete as a float, returning 0.0 if the total size of
     204        the upload is unknown.
     205      """
     206      if self.total_size is not None and self.total_size != 0:
     207        return float(self.resumable_progress) / float(self.total_size)
     208      else:
     209        return 0.0
     210
     211
     212 -class MediaDownloadProgress(object):
     213    """Status of a resumable download."""
     214
     215 -  def __init__(self, resumable_progress, total_size):
     216      """Constructor.
     217
     218      Args:
     219        resumable_progress: int, bytes received so far.
     220        total_size: int, total bytes in complete download.
     221      """
     222      self.resumable_progress = resumable_progress
     223      self.total_size = total_size
     224
     225 -  def progress(self):
     226      """Percent of download completed, as a float.
     227
     228      Returns:
     229        the percentage complete as a float, returning 0.0 if the total size of
     230        the download is unknown.
     231      """
     232      if self.total_size is not None and self.total_size != 0:
     233        return float(self.resumable_progress) / float(self.total_size)
     234      else:
     235        return 0.0
     236
     237
     238 -class MediaUpload(object):
     239    """Describes a media object to upload.
     240
     241    Base class that defines the interface of MediaUpload subclasses.
     242
     243    Note that subclasses of MediaUpload may allow you to control the chunksize
     244    when uploading a media object. It is important to keep the size of the chunk
     245    as large as possible to keep the upload efficient. Other factors may influence
     246    the size of the chunk you use, particularly if you are working in an
     247    environment where individual HTTP requests may have a hardcoded time limit,
     248    such as under certain classes of requests under Google App Engine.
     249
     250    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     251    subclasses support using streams directly to upload data. Support for
     252    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     253    platform that stream will be used for uploading the media object. The support
     254    for streaming is indicated by has_stream() returning True. The stream() method
     255    should return an io.Base object that supports seek(). On platforms where the
     256    underlying httplib module supports streaming, for example Python 2.6 and
     257    later, the stream will be passed into the http library which will result in
     258    less memory being used and possibly faster uploads.
     259
     260    If you need to upload media that can't be uploaded using any of the existing
     261    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     262    needs.
     263    """
     264
     265 -  def chunksize(self):
     266      """Chunk size for resumable uploads.
     267
     268      Returns:
     269        Chunk size in bytes.
     270      """
     271      raise NotImplementedError()
     272
     273 -  def mimetype(self):
     274      """Mime type of the body.
     275
     276      Returns:
     277        Mime type.
     278      """
     279      return 'application/octet-stream'
     280
     281 -  def size(self):
     282      """Size of upload.
     283
     284      Returns:
     285        Size of the body, or None of the size is unknown.
     286      """
     287      return None
     288
     289 -  def resumable(self):
     290      """Whether this upload is resumable.
     291
     292      Returns:
     293        True if resumable upload or False.
     294      """
     295      return False
     296
     297 -  def getbytes(self, begin, end):
     298      """Get bytes from the media.
     299
     300      Args:
     301        begin: int, offset from beginning of file.
     302        length: int, number of bytes to read, starting at begin.
     303
     304      Returns:
     305        A string of bytes read. May be shorter than length if EOF was reached
     306        first.
     307      """
     308      raise NotImplementedError()
     309
     310 -  def has_stream(self):
     311      """Does the underlying upload support a streaming interface.
     312
     313      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     314      seekable() returns True.
     315
     316      Returns:
     317        True if the call to stream() will return an instance of a seekable io.Base
     318        subclass.
     319      """
     320      return False
     321
     322 -  def stream(self):
     323      """A stream interface to the data being uploaded.
     324
     325      Returns:
     326        The returned value is an io.IOBase subclass that supports seek, i.e.
     327        seekable() returns True.
     328      """
     329      raise NotImplementedError()
     330
     331    @util.positional(1)
     332 -  def _to_json(self, strip=None):
     333      """Utility function for creating a JSON representation of a MediaUpload.
     334
     335      Args:
     336        strip: array, An array of names of members to not include in the JSON.
     337
     338      Returns:
     339         string, a JSON representation of this instance, suitable to pass to
     340         from_json().
     341      """
     342      t = type(self)
     343      d = copy.copy(self.__dict__)
     344      if strip is not None:
     345        for member in strip:
     346          del d[member]
     347      d['_class'] = t.__name__
     348      d['_module'] = t.__module__
     349      return json.dumps(d)
     350
     351 -  def to_json(self):
     352      """Create a JSON representation of an instance of MediaUpload.
     353
     354      Returns:
     355         string, a JSON representation of this instance, suitable to pass to
     356         from_json().
     357      """
     358      return self._to_json()
     359
     360    @classmethod
     361 -  def new_from_json(cls, s):
     362      """Utility class method to instantiate a MediaUpload subclass from a JSON
     363      representation produced by to_json().
     364
     365      Args:
     366        s: string, JSON from to_json().
     367
     368      Returns:
     369        An instance of the subclass of MediaUpload that was serialized with
     370        to_json().
     371      """
     372      data = json.loads(s)
     373      # Find and call the right classmethod from_json() to restore the object.
     374      module = data['_module']
     375      m = __import__(module, fromlist=module.split('.')[:-1])
     376      kls = getattr(m, data['_class'])
     377      from_json = getattr(kls, 'from_json')
     378      return from_json(s)
     379
     380
     381 -class MediaIoBaseUpload(MediaUpload):
     382    """A MediaUpload for a io.Base objects.
     383
     384    Note that the Python file object is compatible with io.Base and can be used
     385    with this class also.
     386
     387      fh = BytesIO('...Some data to upload...')
     388      media = MediaIoBaseUpload(fh, mimetype='image/png',
     389        chunksize=1024*1024, resumable=True)
     390      farm.animals().insert(
     391          id='cow',
     392          name='cow.png',
     393          media_body=media).execute()
     394
     395    Depending on the platform you are working on, you may pass -1 as the
     396    chunksize, which indicates that the entire file should be uploaded in a single
     397    request. If the underlying platform supports streams, such as Python 2.6 or
     398    later, then this can be very efficient as it avoids multiple connections, and
     399    also avoids loading the entire file into memory before sending it. Note that
     400    Google App Engine has a 5MB limit on request size, so you should never set
     401    your chunksize larger than 5MB, or to -1.
     402    """
     403
     404    @util.positional(3)
     405 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     406        resumable=False):
     407      """Constructor.
     408
     409      Args:
     410        fd: io.Base or file object, The source of the bytes to upload. MUST be
     411          opened in blocking mode, do not use streams opened in non-blocking mode.
     412          The given stream must be seekable, that is, it must be able to call
     413          seek() on fd.
     414        mimetype: string, Mime-type of the file.
     415        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     416          used if resumable=True. Pass in a value of -1 if the file is to be
     417          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     418          on request size, so you should never set your chunksize larger than 5MB,
     419          or to -1.
     420        resumable: bool, True if this is a resumable upload. False means upload
     421          in a single request.
     422      """
     423      super(MediaIoBaseUpload, self).__init__()
     424      self._fd = fd
     425      self._mimetype = mimetype
     426      if not (chunksize == -1 or chunksize > 0):
     427        raise InvalidChunkSizeError()
     428      self._chunksize = chunksize
     429      self._resumable = resumable
     430
     431      self._fd.seek(0, os.SEEK_END)
     432      self._size = self._fd.tell()
     433
     434 -  def chunksize(self):
     435      """Chunk size for resumable uploads.
     436
     437      Returns:
     438        Chunk size in bytes.
     439      """
     440      return self._chunksize
     441
     442 -  def mimetype(self):
     443      """Mime type of the body.
     444
     445      Returns:
     446        Mime type.
     447      """
     448      return self._mimetype
     449
     450 -  def size(self):
     451      """Size of upload.
     452
     453      Returns:
     454        Size of the body, or None of the size is unknown.
     455      """
     456      return self._size
     457
     458 -  def resumable(self):
     459      """Whether this upload is resumable.
     460
     461      Returns:
     462        True if resumable upload or False.
     463      """
     464      return self._resumable
     465
     466 -  def getbytes(self, begin, length):
     467      """Get bytes from the media.
     468
     469      Args:
     470        begin: int, offset from beginning of file.
     471        length: int, number of bytes to read, starting at begin.
     472
     473      Returns:
     474        A string of bytes read. May be shorted than length if EOF was reached
     475        first.
     476      """
     477      self._fd.seek(begin)
     478      return self._fd.read(length)
     479
     480 -  def has_stream(self):
     481      """Does the underlying upload support a streaming interface.
     482
     483      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     484      seekable() returns True.
     485
     486      Returns:
     487        True if the call to stream() will return an instance of a seekable io.Base
     488        subclass.
     489      """
     490      return True
     491
     492 -  def stream(self):
     493      """A stream interface to the data being uploaded.
     494
     495      Returns:
     496        The returned value is an io.IOBase subclass that supports seek, i.e.
     497        seekable() returns True.
     498      """
     499      return self._fd
     500
     501 -  def to_json(self):
     502      """This upload type is not serializable."""
     503      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     504
     505
     506 -class MediaFileUpload(MediaIoBaseUpload):
     507    """A MediaUpload for a file.
     508
     509    Construct a MediaFileUpload and pass as the media_body parameter of the
     510    method. For example, if we had a service that allowed uploading images:
     511
     512
     513      media = MediaFileUpload('cow.png', mimetype='image/png',
     514        chunksize=1024*1024, resumable=True)
     515      farm.animals().insert(
     516          id='cow',
     517          name='cow.png',
     518          media_body=media).execute()
     519
     520    Depending on the platform you are working on, you may pass -1 as the
     521    chunksize, which indicates that the entire file should be uploaded in a single
     522    request. If the underlying platform supports streams, such as Python 2.6 or
     523    later, then this can be very efficient as it avoids multiple connections, and
     524    also avoids loading the entire file into memory before sending it. Note that
     525    Google App Engine has a 5MB limit on request size, so you should never set
     526    your chunksize larger than 5MB, or to -1.
     527    """
     528
     529    @util.positional(2)
     530 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     531                 resumable=False):
     532      """Constructor.
     533
     534      Args:
     535        filename: string, Name of the file.
     536        mimetype: string, Mime-type of the file. If None then a mime-type will be
     537          guessed from the file extension.
     538        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     539          used if resumable=True. Pass in a value of -1 if the file is to be
     540          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     541          on request size, so you should never set your chunksize larger than 5MB,
     542          or to -1.
     543        resumable: bool, True if this is a resumable upload. False means upload
     544          in a single request.
     545      """
     546      self._filename = filename
     547      fd = open(self._filename, 'rb')
     548      if mimetype is None:
     549        # No mimetype provided, make a guess.
     550        mimetype, _ = mimetypes.guess_type(filename)
     551        if mimetype is None:
     552          # Guess failed, use octet-stream.
     553          mimetype = 'application/octet-stream'
     554      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     555                                            resumable=resumable)
     556
     557 -  def to_json(self):
     558      """Creating a JSON representation of an instance of MediaFileUpload.
     559
     560      Returns:
     561         string, a JSON representation of this instance, suitable to pass to
     562         from_json().
     563      """
     564      return self._to_json(strip=['_fd'])
     565
     566    @staticmethod
     567 -  def from_json(s):
     568      d = json.loads(s)
     569      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     570                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     571
     572
     573 -class MediaInMemoryUpload(MediaIoBaseUpload):
     574    """MediaUpload for a chunk of bytes.
     575
     576    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     577    the stream.
     578    """
     579
     580    @util.positional(2)
     581 -  def __init__(self, body, mimetype='application/octet-stream',
     582                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     583      """Create a new MediaInMemoryUpload.
     584
     585    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     586    the stream.
     587
     588    Args:
     589      body: string, Bytes of body content.
     590      mimetype: string, Mime-type of the file or default of
     591        'application/octet-stream'.
     592      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     593        used if resumable=True.
     594      resumable: bool, True if this is a resumable upload. False means upload
     595        in a single request.
     596      """
     597      fd = BytesIO(body)
     598      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     599                                                resumable=resumable)
     600
     601
     602 -class MediaIoBaseDownload(object):
     603    """"Download media resources.
     604
     605    Note that the Python file object is compatible with io.Base and can be used
     606    with this class also.
     607
     608
     609    Example:
     610      request = farms.animals().get_media(id='cow')
     611      fh = io.FileIO('cow.png', mode='wb')
     612      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     613
     614      done = False
     615      while done is False:
     616        status, done = downloader.next_chunk()
     617        if status:
     618          print "Download %d%%." % int(status.progress() * 100)
     619      print "Download Complete!"
     620    """
     621
     622    @util.positional(3)
     623 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     624      """Constructor.
     625
     626      Args:
     627        fd: io.Base or file object, The stream in which to write the downloaded
     628          bytes.
     629        request: googleapiclient.http.HttpRequest, the media request to perform in
     630          chunks.
     631        chunksize: int, File will be downloaded in chunks of this many bytes.
     632      """
     633      self._fd = fd
     634      self._request = request
     635      self._uri = request.uri
     636      self._chunksize = chunksize
     637      self._progress = 0
     638      self._total_size = None
     639      self._done = False
     640
     641      # Stubs for testing.
     642      self._sleep = time.sleep
     643      self._rand = random.random
     644
     645    @util.positional(1)
     646 -  def next_chunk(self, num_retries=0):
     647      """Get the next chunk of the download.
     648
     649      Args:
     650        num_retries: Integer, number of times to retry with randomized
     651              exponential backoff. If all retries fail, the raised HttpError
     652              represents the last request. If zero (default), we attempt the
     653              request only once.
     654
     655      Returns:
     656        (status, done): (MediaDownloadStatus, boolean)
     657           The value of 'done' will be True when the media has been fully
     658           downloaded or the total size of the media is unknown.
     659
     660      Raises:
     661        googleapiclient.errors.HttpError if the response was not a 2xx.
     662        httplib2.HttpLib2Error if a transport error has occured.
     663      """
     664      headers = {
     665          'range': 'bytes=%d-%d' % (
     666              self._progress, self._progress + self._chunksize)
     667          }
     668      http = self._request.http
     669
     670      resp, content = _retry_request(
     671          http, num_retries, 'media download', self._sleep, self._rand, self._uri,
     672          'GET', headers=headers)
     673
     674      if resp.status in [200, 206]:
     675        if 'content-location' in resp and resp['content-location'] != self._uri:
     676          self._uri = resp['content-location']
     677        self._progress += len(content)
     678        self._fd.write(content)
     679
     680        if 'content-range' in resp:
     681          content_range = resp['content-range']
     682          length = content_range.rsplit('/', 1)[1]
     683          self._total_size = int(length)
     684        elif 'content-length' in resp:
     685          self._total_size = int(resp['content-length'])
     686
     687        if self._total_size is None or self._progress == self._total_size:
     688          self._done = True
     689        return MediaDownloadProgress(self._progress, self._total_size), self._done
     690      else:
     691        raise HttpError(resp, content, uri=self._uri)
     692
     693
     694 -class _StreamSlice(object):
     695    """Truncated stream.
     696
     697    Takes a stream and presents a stream that is a slice of the original stream.
     698    This is used when uploading media in chunks. In later versions of Python a
     699    stream can be passed to httplib in place of the string of data to send. The
     700    problem is that httplib just blindly reads to the end of the stream. This
     701    wrapper presents a virtual stream that only reads to the end of the chunk.
     702    """
     703
     704 -  def __init__(self, stream, begin, chunksize):
     705      """Constructor.
     706
     707      Args:
     708        stream: (io.Base, file object), the stream to wrap.
     709        begin: int, the seek position the chunk begins at.
     710        chunksize: int, the size of the chunk.
     711      """
     712      self._stream = stream
     713      self._begin = begin
     714      self._chunksize = chunksize
     715      self._stream.seek(begin)
     716
     717 -  def read(self, n=-1):
     718      """Read n bytes.
     719
     720      Args:
     721        n, int, the number of bytes to read.
     722
     723      Returns:
     724        A string of length 'n', or less if EOF is reached.
     725      """
     726      # The data left available to read sits in [cur, end)
     727      cur = self._stream.tell()
     728      end = self._begin + self._chunksize
     729      if n == -1 or cur + n > end:
     730        n = end - cur
     731      return self._stream.read(n)
     732
     733
     734 -class HttpRequest(object):
     735    """Encapsulates a single HTTP request."""
     736
     737    @util.positional(4)
     738 -  def __init__(self, http, postproc, uri,
     739                 method='GET',
     740                 body=None,
     741                 headers=None,
     742                 methodId=None,
     743                 resumable=None):
     744      """Constructor for an HttpRequest.
     745
     746      Args:
     747        http: httplib2.Http, the transport object to use to make a request
     748        postproc: callable, called on the HTTP response and content to transform
     749                  it into a data object before returning, or raising an exception
     750                  on an error.
     751        uri: string, the absolute URI to send the request to
     752        method: string, the HTTP method to use
     753        body: string, the request body of the HTTP request,
     754        headers: dict, the HTTP request headers
     755        methodId: string, a unique identifier for the API method being called.
     756        resumable: MediaUpload, None if this is not a resumbale request.
     757      """
     758      self.uri = uri
     759      self.method = method
     760      self.body = body
     761      self.headers = headers or {}
     762      self.methodId = methodId
     763      self.http = http
     764      self.postproc = postproc
     765      self.resumable = resumable
     766      self.response_callbacks = []
     767      self._in_error_state = False
     768
     769      # Pull the multipart boundary out of the content-type header.
     770      major, minor, params = mimeparse.parse_mime_type(
     771          self.headers.get('content-type', 'application/json'))
     772
     773      # The size of the non-media part of the request.
     774      self.body_size = len(self.body or '')
     775
     776      # The resumable URI to send chunks to.
     777      self.resumable_uri = None
     778
     779      # The bytes that have been uploaded.
     780      self.resumable_progress = 0
     781
     782      # Stubs for testing.
     783      self._rand = random.random
     784      self._sleep = time.sleep
     785
     786    @util.positional(1)
     787 -  def execute(self, http=None, num_retries=0):
     788      """Execute the request.
     789
     790      Args:
     791        http: httplib2.Http, an http object to be used in place of the
     792              one the HttpRequest request object was constructed with.
     793        num_retries: Integer, number of times to retry with randomized
     794              exponential backoff. If all retries fail, the raised HttpError
     795              represents the last request. If zero (default), we attempt the
     796              request only once.
     797
     798      Returns:
     799        A deserialized object model of the response body as determined
     800        by the postproc.
     801
     802      Raises:
     803        googleapiclient.errors.HttpError if the response was not a 2xx.
     804        httplib2.HttpLib2Error if a transport error has occured.
     805      """
     806      if http is None:
     807        http = self.http
     808
     809      if self.resumable:
     810        body = None
     811        while body is None:
     812          _, body = self.next_chunk(http=http, num_retries=num_retries)
     813        return body
     814
     815      # Non-resumable case.
     816
     817      if 'content-length' not in self.headers:
     818        self.headers['content-length'] = str(self.body_size)
     819      # If the request URI is too long then turn it into a POST request.
     820      # Assume that a GET request never contains a request body.
     821      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     822        self.method = 'POST'
     823        self.headers['x-http-method-override'] = 'GET'
     824        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     825        parsed = urlparse(self.uri)
     826        self.uri = urlunparse(
     827            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     828             None)
     829            )
     830        self.body = parsed.query
     831        self.headers['content-length'] = str(len(self.body))
     832
     833      # Handle retries for server-side errors.
     834      resp, content = _retry_request(
     835            http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
     836            method=str(self.method), body=self.body, headers=self.headers)
     837
     838      for callback in self.response_callbacks:
     839        callback(resp)
     840      if resp.status >= 300:
     841        raise HttpError(resp, content, uri=self.uri)
     842      return self.postproc(resp, content)
     843
     844    @util.positional(2)
     845 -  def add_response_callback(self, cb):
     846      """add_response_headers_callback
     847
     848      Args:
     849        cb: Callback to be called on receiving the response headers, of signature:
     850
     851        def cb(resp):
     852          # Where resp is an instance of httplib2.Response
     853      """
     854      self.response_callbacks.append(cb)
     855
     856    @util.positional(1)
     857 -  def next_chunk(self, http=None, num_retries=0):
     858      """Execute the next step of a resumable upload.
     859
     860      Can only be used if the method being executed supports media uploads and
     861      the MediaUpload object passed in was flagged as using resumable upload.
     862
     863      Example:
     864
     865        media = MediaFileUpload('cow.png', mimetype='image/png',
     866                                chunksize=1000, resumable=True)
     867        request = farm.animals().insert(
     868            id='cow',
     869            name='cow.png',
     870            media_body=media)
     871
     872        response = None
     873        while response is None:
     874          status, response = request.next_chunk()
     875          if status:
     876            print "Upload %d%% complete." % int(status.progress() * 100)
     877
     878
     879      Args:
     880        http: httplib2.Http, an http object to be used in place of the
     881              one the HttpRequest request object was constructed with.
     882        num_retries: Integer, number of times to retry with randomized
     883              exponential backoff. If all retries fail, the raised HttpError
     884              represents the last request. If zero (default), we attempt the
     885              request only once.
     886
     887      Returns:
     888        (status, body): (ResumableMediaStatus, object)
     889           The body will be None until the resumable media is fully uploaded.
     890
     891      Raises:
     892        googleapiclient.errors.HttpError if the response was not a 2xx.
     893        httplib2.HttpLib2Error if a transport error has occured.
     894      """
     895      if http is None:
     896        http = self.http
     897
     898      if self.resumable.size() is None:
     899        size = '*'
     900      else:
     901        size = str(self.resumable.size())
     902
     903      if self.resumable_uri is None:
     904        start_headers = copy.copy(self.headers)
     905        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     906        if size != '*':
     907          start_headers['X-Upload-Content-Length'] = size
     908        start_headers['content-length'] = str(self.body_size)
     909
     910        resp, content = _retry_request(
     911            http, num_retries, 'resumable URI request', self._sleep, self._rand,
     912            self.uri, method=self.method, body=self.body, headers=start_headers)
     913
     914        if resp.status == 200 and 'location' in resp:
     915          self.resumable_uri = resp['location']
     916        else:
     917          raise ResumableUploadError(resp, content)
     918      elif self._in_error_state:
     919        # If we are in an error state then query the server for current state of
     920        # the upload by sending an empty PUT and reading the 'range' header in
     921        # the response.
     922        headers = {
     923            'Content-Range': 'bytes */%s' % size,
     924            'content-length': '0'
     925            }
     926        resp, content = http.request(self.resumable_uri, 'PUT',
     927                                     headers=headers)
     928        status, body = self._process_response(resp, content)
     929        if body:
     930          # The upload was complete.
     931          return (status, body)
     932
     933      if self.resumable.has_stream():
     934        data = self.resumable.stream()
     935        if self.resumable.chunksize() == -1:
     936          data.seek(self.resumable_progress)
     937          chunk_end = self.resumable.size() - self.resumable_progress - 1
     938        else:
     939          # Doing chunking with a stream, so wrap a slice of the stream.
     940          data = _StreamSlice(data, self.resumable_progress,
     941                              self.resumable.chunksize())
     942          chunk_end = min(
     943              self.resumable_progress + self.resumable.chunksize() - 1,
     944              self.resumable.size() - 1)
     945      else:
     946        data = self.resumable.getbytes(
     947            self.resumable_progress, self.resumable.chunksize())
     948
     949        # A short read implies that we are at EOF, so finish the upload.
     950        if len(data) < self.resumable.chunksize():
     951          size = str(self.resumable_progress + len(data))
     952
     953        chunk_end = self.resumable_progress + len(data) - 1
     954
     955      headers = {
     956          'Content-Range': 'bytes %d-%d/%s' % (
     957              self.resumable_progress, chunk_end, size),
     958          # Must set the content-length header here because httplib can't
     959          # calculate the size when working with _StreamSlice.
     960          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     961          }
     962
     963      for retry_num in range(num_retries + 1):
     964        if retry_num > 0:
     965          self._sleep(self._rand() * 2**retry_num)
     966          LOGGER.warning(
     967              'Retry #%d for media upload: %s %s, following status: %d'
     968              % (retry_num, self.method, self.uri, resp.status))
     969
     970        try:
     971          resp, content = http.request(self.resumable_uri, method='PUT',
     972                                       body=data,
     973                                       headers=headers)
     974        except:
     975          self._in_error_state = True
     976          raise
     977        if not _should_retry_response(resp.status, content):
     978          break
     979
     980      return self._process_response(resp, content)
     981
     982 -  def _process_response(self, resp, content):
     983      """Process the response from a single chunk upload.
     984
     985      Args:
     986        resp: httplib2.Response, the response object.
     987        content: string, the content of the response.
     988
     989      Returns:
     990        (status, body): (ResumableMediaStatus, object)
     991           The body will be None until the resumable media is fully uploaded.
     992
     993      Raises:
     994        googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
     995      """
     996      if resp.status in [200, 201]:
     997        self._in_error_state = False
     998        return None, self.postproc(resp, content)
     999      elif resp.status == 308:
    1000        self._in_error_state = False
    1001        # A "308 Resume Incomplete" indicates we are not done.
    1002        try:
    1003          self.resumable_progress = int(resp['range'].split('-')[1]) + 1
    1004        except KeyError:
    1005          # If resp doesn't contain range header, resumable progress is 0
    1006          self.resumable_progress = 0
    1007        if 'location' in resp:
    1008          self.resumable_uri = resp['location']
    1009      else:
    1010        self._in_error_state = True
    1011        raise HttpError(resp, content, uri=self.uri)
    1012
    1013      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
    1014              None)
    1015
    1016 -  def to_json(self):
    1017      """Returns a JSON representation of the HttpRequest."""
    1018      d = copy.copy(self.__dict__)
    1019      if d['resumable'] is not None:
    1020        d['resumable'] = self.resumable.to_json()
    1021      del d['http']
    1022      del d['postproc']
    1023      del d['_sleep']
    1024      del d['_rand']
    1025
    1026      return json.dumps(d)
    1027
    1028    @staticmethod
    1029 -  def from_json(s, http, postproc):
    1030      """Returns an HttpRequest populated with info from a JSON object."""
    1031      d = json.loads(s)
    1032      if d['resumable'] is not None:
    1033        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
    1034      return HttpRequest(
    1035          http,
    1036          postproc,
    1037          uri=d['uri'],
    1038          method=d['method'],
    1039          body=d['body'],
    1040          headers=d['headers'],
    1041          methodId=d['methodId'],
    1042          resumable=d['resumable'])
    1043
    1044
    1045 -class BatchHttpRequest(object):
    1046    """Batches multiple HttpRequest objects into a single HTTP request.
    1047
    1048    Example:
    1049      from googleapiclient.http import BatchHttpRequest
    1050
    1051      def list_animals(request_id, response, exception):
    1052        \"\"\"Do something with the animals list response.\"\"\"
    1053        if exception is not None:
    1054          # Do something with the exception.
    1055          pass
    1056        else:
    1057          # Do something with the response.
    1058          pass
    1059
    1060      def list_farmers(request_id, response, exception):
    1061        \"\"\"Do something with the farmers list response.\"\"\"
    1062        if exception is not None:
    1063          # Do something with the exception.
    1064          pass
    1065        else:
    1066          # Do something with the response.
    1067          pass
    1068
    1069      service = build('farm', 'v2')
    1070
    1071      batch = BatchHttpRequest()
    1072
    1073      batch.add(service.animals().list(), list_animals)
    1074      batch.add(service.farmers().list(), list_farmers)
    1075      batch.execute(http=http)
    1076    """
    1077
    1078    @util.positional(1)
    1079 -  def __init__(self, callback=None, batch_uri=None):
    1080      """Constructor for a BatchHttpRequest.
    1081
    1082      Args:
    1083        callback: callable, A callback to be called for each response, of the
    1084          form callback(id, response, exception). The first parameter is the
    1085          request id, and the second is the deserialized response object. The
    1086          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1087          occurred while processing the request, or None if no error occurred.
    1088        batch_uri: string, URI to send batch requests to.
    1089      """
    1090      if batch_uri is None:
    1091        batch_uri = 'https://www.googleapis.com/batch'
    1092      self._batch_uri = batch_uri
    1093
    1094      # Global callback to be called for each individual response in the batch.
    1095      self._callback = callback
    1096
    1097      # A map from id to request.
    1098      self._requests = {}
    1099
    1100      # A map from id to callback.
    1101      self._callbacks = {}
    1102
    1103      # List of request ids, in the order in which they were added.
    1104      self._order = []
    1105
    1106      # The last auto generated id.
    1107      self._last_auto_id = 0
    1108
    1109      # Unique ID on which to base the Content-ID headers.
    1110      self._base_id = None
    1111
    1112      # A map from request id to (httplib2.Response, content) response pairs
    1113      self._responses = {}
    1114
    1115      # A map of id(Credentials) that have been refreshed.
    1116      self._refreshed_credentials = {}
    1117
    1118 -  def _refresh_and_apply_credentials(self, request, http):
    1119      """Refresh the credentials and apply to the request.
    1120
    1121      Args:
    1122        request: HttpRequest, the request.
    1123        http: httplib2.Http, the global http object for the batch.
    1124      """
    1125      # For the credentials to refresh, but only once per refresh_token
    1126      # If there is no http per the request then refresh the http passed in
    1127      # via execute()
    1128      creds = None
    1129      if request.http is not None and hasattr(request.http.request,
    1130          'credentials'):
    1131        creds = request.http.request.credentials
    1132      elif http is not None and hasattr(http.request, 'credentials'):
    1133        creds = http.request.credentials
    1134      if creds is not None:
    1135        if id(creds) not in self._refreshed_credentials:
    1136          creds.refresh(http)
    1137          self._refreshed_credentials[id(creds)] = 1
    1138
    1139      # Only apply the credentials if we are using the http object passed in,
    1140      # otherwise apply() will get called during _serialize_request().
    1141      if request.http is None or not hasattr(request.http.request,
    1142          'credentials'):
    1143        creds.apply(request.headers)
    1144
    1145 -  def _id_to_header(self, id_):
    1146      """Convert an id to a Content-ID header value.
    1147
    1148      Args:
    1149        id_: string, identifier of individual request.
    1150
    1151      Returns:
    1152        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1153        the value because Content-ID headers are supposed to be universally
    1154        unique.
    1155      """
    1156      if self._base_id is None:
    1157        self._base_id = uuid.uuid4()
    1158
    1159      return '<%s+%s>' % (self._base_id, quote(id_))
    1160
    1161 -  def _header_to_id(self, header):
    1162      """Convert a Content-ID header value to an id.
    1163
    1164      Presumes the Content-ID header conforms to the format that _id_to_header()
    1165      returns.
    1166
    1167      Args:
    1168        header: string, Content-ID header value.
    1169
    1170      Returns:
    1171        The extracted id value.
    1172
    1173      Raises:
    1174        BatchError if the header is not in the expected format.
    1175      """
    1176      if header[0] != '<' or header[-1] != '>':
    1177        raise BatchError("Invalid value for Content-ID: %s" % header)
    1178      if '+' not in header:
    1179        raise BatchError("Invalid value for Content-ID: %s" % header)
    1180      base, id_ = header[1:-1].rsplit('+', 1)
    1181
    1182      return unquote(id_)
    1183
    1184 -  def _serialize_request(self, request):
    1185      """Convert an HttpRequest object into a string.
    1186
    1187      Args:
    1188        request: HttpRequest, the request to serialize.
    1189
    1190      Returns:
    1191        The request as a string in application/http format.
    1192      """
    1193      # Construct status line
    1194      parsed = urlparse(request.uri)
    1195      request_line = urlunparse(
    1196          ('', '', parsed.path, parsed.params, parsed.query, '')
    1197          )
    1198      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1199      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1200      msg = MIMENonMultipart(major, minor)
    1201      headers = request.headers.copy()
    1202
    1203      if request.http is not None and hasattr(request.http.request,
    1204          'credentials'):
    1205        request.http.request.credentials.apply(headers)
    1206
    1207      # MIMENonMultipart adds its own Content-Type header.
    1208      if 'content-type' in headers:
    1209        del headers['content-type']
    1210
    1211      for key, value in six.iteritems(headers):
    1212        msg[key] = value
    1213      msg['Host'] = parsed.netloc
    1214      msg.set_unixfrom(None)
    1215
    1216      if request.body is not None:
    1217        msg.set_payload(request.body)
    1218        msg['content-length'] = str(len(request.body))
    1219
    1220      # Serialize the mime message.
    1221      fp = StringIO()
    1222      # maxheaderlen=0 means don't line wrap headers.
    1223      g = Generator(fp, maxheaderlen=0)
    1224      g.flatten(msg, unixfrom=False)
    1225      body = fp.getvalue()
    1226
    1227      return status_line + body
    1228
    1229 -  def _deserialize_response(self, payload):
    1230      """Convert string into httplib2 response and content.
    1231
    1232      Args:
    1233        payload: string, headers and body as a string.
    1234
    1235      Returns:
    1236        A pair (resp, content), such as would be returned from httplib2.request.
    1237      """
    1238      # Strip off the status line
    1239      status_line, payload = payload.split('\n', 1)
    1240      protocol, status, reason = status_line.split(' ', 2)
    1241
    1242      # Parse the rest of the response
    1243      parser = FeedParser()
    1244      parser.feed(payload)
    1245      msg = parser.close()
    1246      msg['status'] = status
    1247
    1248      # Create httplib2.Response from the parsed headers.
    1249      resp = httplib2.Response(msg)
    1250      resp.reason = reason
    1251      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1252
    1253      content = payload.split('\r\n\r\n', 1)[1]
    1254
    1255      return resp, content
    1256
    1257 -  def _new_id(self):
    1258      """Create a new id.
    1259
    1260      Auto incrementing number that avoids conflicts with ids already used.
    1261
    1262      Returns:
    1263         string, a new unique id.
    1264      """
    1265      self._last_auto_id += 1
    1266      while str(self._last_auto_id) in self._requests:
    1267        self._last_auto_id += 1
    1268      return str(self._last_auto_id)
    1269
    1270    @util.positional(2)
    1271 -  def add(self, request, callback=None, request_id=None):
    1272      """Add a new request.
    1273
    1274      Every callback added will be paired with a unique id, the request_id. That
    1275      unique id will be passed back to the callback when the response comes back
    1276      from the server. The default behavior is to have the library generate it's
    1277      own unique id. If the caller passes in a request_id then they must ensure
    1278      uniqueness for each request_id, and if they are not an exception is
    1279      raised. Callers should either supply all request_ids or nevery supply a
    1280      request id, to avoid such an error.
    1281
    1282      Args:
    1283        request: HttpRequest, Request to add to the batch.
    1284        callback: callable, A callback to be called for this response, of the
    1285          form callback(id, response, exception). The first parameter is the
    1286          request id, and the second is the deserialized response object. The
    1287          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1288          occurred while processing the request, or None if no errors occurred.
    1289        request_id: string, A unique id for the request. The id will be passed to
    1290          the callback with the response.
    1291
    1292      Returns:
    1293        None
    1294
    1295      Raises:
    1296        BatchError if a media request is added to a batch.
    1297        KeyError is the request_id is not unique.
    1298      """
    1299      if request_id is None:
    1300        request_id = self._new_id()
    1301      if request.resumable is not None:
    1302        raise BatchError("Media requests cannot be used in a batch request.")
    1303      if request_id in self._requests:
    1304        raise KeyError("A request with this ID already exists: %s" % request_id)
    1305      self._requests[request_id] = request
    1306      self._callbacks[request_id] = callback
    1307      self._order.append(request_id)
    1308
    1309 -  def _execute(self, http, order, requests):
    1310      """Serialize batch request, send to server, process response.
    1311
    1312      Args:
    1313        http: httplib2.Http, an http object to be used to make the request with.
    1314        order: list, list of request ids in the order they were added to the
    1315          batch.
    1316        request: list, list of request objects to send.
    1317
    1318      Raises:
    1319        httplib2.HttpLib2Error if a transport error has occured.
    1320        googleapiclient.errors.BatchError if the response is the wrong format.
    1321      """
    1322      message = MIMEMultipart('mixed')
    1323      # Message should not write out it's own headers.
    1324      setattr(message, '_write_headers', lambda self: None)
    1325
    1326      # Add all the individual requests.
    1327      for request_id in order:
    1328        request = requests[request_id]
    1329
    1330        msg = MIMENonMultipart('application', 'http')
    1331        msg['Content-Transfer-Encoding'] = 'binary'
    1332        msg['Content-ID'] = self._id_to_header(request_id)
    1333
    1334        body = self._serialize_request(request)
    1335        msg.set_payload(body)
    1336        message.attach(msg)
    1337
    1338      # encode the body: note that we can't use `as_string`, because
    1339      # it plays games with `From ` lines.
    1340      fp = StringIO()
    1341      g = Generator(fp, mangle_from_=False)
    1342      g.flatten(message, unixfrom=False)
    1343      body = fp.getvalue()
    1344
    1345      headers = {}
    1346      headers['content-type'] = ('multipart/mixed; '
    1347                                 'boundary="%s"') % message.get_boundary()
    1348
    1349      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1350                                   headers=headers)
    1351
    1352      if resp.status >= 300:
    1353        raise HttpError(resp, content, uri=self._batch_uri)
    1354
    1355      # Prepend with a content-type header so FeedParser can handle it.
    1356      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1357      # PY3's FeedParser only accepts unicode. So we should decode content
    1358      # here, and encode each payload again.
    1359      if six.PY3:
    1360        content = content.decode('utf-8')
    1361      for_parser = header + content
    1362
    1363      parser = FeedParser()
    1364      parser.feed(for_parser)
    1365      mime_response = parser.close()
    1366
    1367      if not mime_response.is_multipart():
    1368        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1369                         content=content)
    1370
    1371      for part in mime_response.get_payload():
    1372        request_id = self._header_to_id(part['Content-ID'])
    1373        response, content = self._deserialize_response(part.get_payload())
    1374        # We encode content here to emulate normal http response.
    1375        if isinstance(content, six.text_type):
    1376          content = content.encode('utf-8')
    1377        self._responses[request_id] = (response, content)
    1378
    1379    @util.positional(1)
    1380 -  def execute(self, http=None):
    1381      """Execute all the requests as a single batched HTTP request.
    1382
    1383      Args:
    1384        http: httplib2.Http, an http object to be used in place of the one the
    1385          HttpRequest request object was constructed with. If one isn't supplied
    1386          then use a http object from the requests in this batch.
    1387
    1388      Returns:
    1389        None
    1390
    1391      Raises:
    1392        httplib2.HttpLib2Error if a transport error has occured.
    1393        googleapiclient.errors.BatchError if the response is the wrong format.
    1394      """
    1395      # If we have no requests return
    1396      if len(self._order) == 0:
    1397        return None
    1398
    1399      # If http is not supplied use the first valid one given in the requests.
    1400      if http is None:
    1401        for request_id in self._order:
    1402          request = self._requests[request_id]
    1403          if request is not None:
    1404            http = request.http
    1405            break
    1406
    1407      if http is None:
    1408        raise ValueError("Missing a valid http object.")
    1409
    1410      # Special case for OAuth2Credentials-style objects which have not yet been
    1411      # refreshed with an initial access_token.
    1412      if getattr(http.request, 'credentials', None) is not None:
    1413        creds = http.request.credentials
    1414        if not getattr(creds, 'access_token', None):
    1415          LOGGER.info('Attempting refresh to obtain initial access_token')
    1416          creds.refresh(http)
    1417
    1418      self._execute(http, self._order, self._requests)
    1419
    1420      # Loop over all the requests and check for 401s. For each 401 request the
    1421      # credentials should be refreshed and then sent again in a separate batch.
    1422      redo_requests = {}
    1423      redo_order = []
    1424
    1425      for request_id in self._order:
    1426        resp, content = self._responses[request_id]
    1427        if resp['status'] == '401':
    1428          redo_order.append(request_id)
    1429          request = self._requests[request_id]
    1430          self._refresh_and_apply_credentials(request, http)
    1431          redo_requests[request_id] = request
    1432
    1433      if redo_requests:
    1434        self._execute(http, redo_order, redo_requests)
    1435
    1436      # Now process all callbacks that are erroring, and raise an exception for
    1437      # ones that return a non-2xx response? Or add extra parameter to callback
    1438      # that contains an HttpError?
    1439
    1440      for request_id in self._order:
    1441        resp, content = self._responses[request_id]
    1442
    1443        request = self._requests[request_id]
    1444        callback = self._callbacks[request_id]
    1445
    1446        response = None
    1447        exception = None
    1448        try:
    1449          if resp.status >= 300:
    1450            raise HttpError(resp, content, uri=request.uri)
    1451          response = request.postproc(resp, content)
    1452        except HttpError as e:
    1453          exception = e
    1454
    1455        if callback is not None:
    1456          callback(request_id, response, exception)
    1457        if self._callback is not None:
    1458          self._callback(request_id, response, exception)
    1459
    1460
    1461 -class HttpRequestMock(object):
    1462    """Mock of HttpRequest.
    1463
    1464    Do not construct directly, instead use RequestMockBuilder.
    1465    """
    1466
    1467 -  def __init__(self, resp, content, postproc):
    1468      """Constructor for HttpRequestMock
    1469
    1470      Args:
    1471        resp: httplib2.Response, the response to emulate coming from the request
    1472        content: string, the response body
    1473        postproc: callable, the post processing function usually supplied by
    1474                  the model class. See model.JsonModel.response() as an example.
    1475      """
    1476      self.resp = resp
    1477      self.content = content
    1478      self.postproc = postproc
    1479      if resp is None:
    1480        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1481      if 'reason' in self.resp:
    1482        self.resp.reason = self.resp['reason']
    1483
    1484 -  def execute(self, http=None):
    1485      """Execute the request.
    1486
    1487      Same behavior as HttpRequest.execute(), but the response is
    1488      mocked and not really from an HTTP request/response.
    1489      """
    1490      return self.postproc(self.resp, self.content)
    1491
    1492
    1493 -class RequestMockBuilder(object):
    1494    """A simple mock of HttpRequest
    1495
    1496      Pass in a dictionary to the constructor that maps request methodIds to
    1497      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1498      returned when that method is called. None may also be passed in for the
    1499      httplib2.Response, in which case a 200 OK response will be generated.
    1500      If an opt_expected_body (str or dict) is provided, it will be compared to
    1501      the body and UnexpectedBodyError will be raised on inequality.
    1502
    1503      Example:
    1504        response = '{"data": {"id": "tag:google.c...'
    1505        requestBuilder = RequestMockBuilder(
    1506          {
    1507            'plus.activities.get': (None, response),
    1508          }
    1509        )
    1510        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1511
    1512      Methods that you do not supply a response for will return a
    1513      200 OK with an empty string as the response content or raise an excpetion
    1514      if check_unexpected is set to True. The methodId is taken from the rpcName
    1515      in the discovery document.
    1516
    1517      For more details see the project wiki.
    1518    """
    1519
    1520 -  def __init__(self, responses, check_unexpected=False):
    1521      """Constructor for RequestMockBuilder
    1522
    1523      The constructed object should be a callable object
    1524      that can replace the class HttpResponse.
    1525
    1526      responses - A dictionary that maps methodIds into tuples
    1527                  of (httplib2.Response, content). The methodId
    1528                  comes from the 'rpcName' field in the discovery
    1529                  document.
    1530      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1531                         should be raised on unsupplied method.
    1532      """
    1533      self.responses = responses
    1534      self.check_unexpected = check_unexpected
    1535
    1536 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1537                 headers=None, methodId=None, resumable=None):
    1538      """Implements the callable interface that discovery.build() expects
    1539      of requestBuilder, which is to build an object compatible with
    1540      HttpRequest.execute(). See that method for the description of the
    1541      parameters and the expected response.
    1542      """
    1543      if methodId in self.responses:
    1544        response = self.responses[methodId]
    1545        resp, content = response[:2]
    1546        if len(response) > 2:
    1547          # Test the body against the supplied expected_body.
    1548          expected_body = response[2]
    1549          if bool(expected_body) != bool(body):
    1550            # Not expecting a body and provided one
    1551            # or expecting a body and not provided one.
    1552            raise UnexpectedBodyError(expected_body, body)
    1553          if isinstance(expected_body, str):
    1554            expected_body = json.loads(expected_body)
    1555          body = json.loads(body)
    1556          if body != expected_body:
    1557            raise UnexpectedBodyError(expected_body, body)
    1558        return HttpRequestMock(resp, content, postproc)
    1559      elif self.check_unexpected:
    1560        raise UnexpectedMethodError(methodId=methodId)
    1561      else:
    1562        model = JsonModel(False)
    1563        return HttpRequestMock(None, '{}', model.response)
    1564
    1565
    1566 -class HttpMock(object):
    1567    """Mock of httplib2.Http"""
    1568
    1569 -  def __init__(self, filename=None, headers=None):
    1570      """
    1571      Args:
    1572        filename: string, absolute filename to read response from
    1573        headers: dict, header to return with response
    1574      """
    1575      if headers is None:
    1576        headers = {'status': '200'}
    1577      if filename:
    1578        f = open(filename, 'rb')
    1579        self.data = f.read()
    1580        f.close()
    1581      else:
    1582        self.data = None
    1583      self.response_headers = headers
    1584      self.headers = None
    1585      self.uri = None
    1586      self.method = None
    1587      self.body = None
    1588      self.headers = None
    1589
    1590
    1591 -  def request(self, uri,
    1592                method='GET',
    1593                body=None,
    1594                headers=None,
    1595                redirections=1,
    1596                connection_type=None):
    1597      self.uri = uri
    1598      self.method = method
    1599      self.body = body
    1600      self.headers = headers
    1601      return httplib2.Response(self.response_headers), self.data
    1602
    1603
    1604 -class HttpMockSequence(object):
    1605    """Mock of httplib2.Http
    1606
    1607    Mocks a sequence of calls to request returning different responses for each
    1608    call. Create an instance initialized with the desired response headers
    1609    and content and then use as if an httplib2.Http instance.
    1610
    1611      http = HttpMockSequence([
    1612        ({'status': '401'}, ''),
    1613        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1614        ({'status': '200'}, 'echo_request_headers'),
    1615        ])
    1616      resp, content = http.request("http://examples.com")
    1617
    1618    There are special values you can pass in for content to trigger
    1619    behavours that are helpful in testing.
    1620
    1621    'echo_request_headers' means return the request headers in the response body
    1622    'echo_request_headers_as_json' means return the request headers in
    1623       the response body
    1624    'echo_request_body' means return the request body in the response body
    1625    'echo_request_uri' means return the request uri in the response body
    1626    """
    1627
    1628 -  def __init__(self, iterable):
    1629      """
    1630      Args:
    1631        iterable: iterable, a sequence of pairs of (headers, body)
    1632      """
    1633      self._iterable = iterable
    1634      self.follow_redirects = True
    1635
    1636 -  def request(self, uri,
    1637                method='GET',
    1638                body=None,
    1639                headers=None,
    1640                redirections=1,
    1641                connection_type=None):
    1642      resp, content = self._iterable.pop(0)
    1643      if content == 'echo_request_headers':
    1644        content = headers
    1645      elif content == 'echo_request_headers_as_json':
    1646        content = json.dumps(headers)
    1647      elif content == 'echo_request_body':
    1648        if hasattr(body, 'read'):
    1649          content = body.read()
    1650        else:
    1651          content = body
    1652      elif content == 'echo_request_uri':
    1653        content = uri
    1654      if isinstance(content, six.text_type):
    1655        content = content.encode('utf-8')
    1656      return httplib2.Response(resp), content
    1657
    1658
    1659 -def set_user_agent(http, user_agent):
    1660    """Set the user-agent on every request.
    1661
    1662    Args:
    1663       http - An instance of httplib2.Http
    1664           or something that acts like it.
    1665       user_agent: string, the value for the user-agent header.
    1666
    1667    Returns:
    1668       A modified instance of http that was passed in.
    1669
    1670    Example:
    1671
    1672      h = httplib2.Http()
    1673      h = set_user_agent(h, "my-app-name/6.0")
    1674
    1675    Most of the time the user-agent will be set doing auth, this is for the rare
    1676    cases where you are accessing an unauthenticated endpoint.
    1677    """
    1678    request_orig = http.request
    1679
    1680    # The closure that will replace 'httplib2.Http.request'.
    1681    def new_request(uri, method='GET', body=None, headers=None,
    1682                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1683                    connection_type=None):
    1684      """Modify the request headers to add the user-agent."""
    1685      if headers is None:
    1686        headers = {}
    1687      if 'user-agent' in headers:
    1688        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1689      else:
    1690        headers['user-agent'] = user_agent
    1691      resp, content = request_orig(uri, method, body, headers,
    1692                          redirections, connection_type)
    1693      return resp, content
    1694
    1695    http.request = new_request
    1696    return http
    1697
    1698
    1699 -def tunnel_patch(http):
    1700    """Tunnel PATCH requests over POST.
    1701    Args:
    1702       http - An instance of httplib2.Http
    1703           or something that acts like it.
    1704
    1705    Returns:
    1706       A modified instance of http that was passed in.
    1707
    1708    Example:
    1709
    1710      h = httplib2.Http()
    1711      h = tunnel_patch(h, "my-app-name/6.0")
    1712
    1713    Useful if you are running on a platform that doesn't support PATCH.
    1714    Apply this last if you are using OAuth 1.0, as changing the method
    1715    will result in a different signature.
    1716    """
    1717    request_orig = http.request
    1718
    1719    # The closure that will replace 'httplib2.Http.request'.
    1720    def new_request(uri, method='GET', body=None, headers=None,
    1721                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1722                    connection_type=None):
    1723      """Modify the request headers to add the user-agent."""
    1724      if headers is None:
    1725        headers = {}
    1726      if method == 'PATCH':
    1727        if 'oauth_token' in headers.get('authorization', ''):
    1728          LOGGER.warning(
    1729              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1730        headers['x-http-method-override'] = "PATCH"
    1731        method = 'POST'
    1732      resp, content = request_orig(uri, method, body, headers,
    1733                          redirections, connection_type)
    1734      return resp, content
    1735
    1736    http.request = new_request
    1737    return http
    1738
    1739
    1740 -def build_http():
    1741    """Builds httplib2.Http object
    1742
    1743    Returns:
    1744    A httplib2.Http object, which is used to make http requests, and which has timeout set by default.
    1745    To override default timeout call
    1746
    1747      socket.setdefaulttimeout(timeout_in_sec)
    1748
    1749    before interacting with this method.
    1750    """
    1751    if socket.getdefaulttimeout() is not None:
    1752      http_timeout = socket.getdefaulttimeout()
    1753    else:
    1754      http_timeout = DEFAULT_HTTP_TIMEOUT_SEC
    1755    return httplib2.Http(timeout=http_timeout)
    1756