Home | History | Annotate | Download | only in contacts 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/chromeos/contacts/gdata_contacts_service.h"
      6 
      7 #include <cstring>
      8 #include <map>
      9 #include <string>
     10 #include <utility>
     11 
     12 #include "base/json/json_value_converter.h"
     13 #include "base/json/json_writer.h"
     14 #include "base/logging.h"
     15 #include "base/memory/weak_ptr.h"
     16 #include "base/metrics/histogram.h"
     17 #include "base/stl_util.h"
     18 #include "base/strings/string_util.h"
     19 #include "base/threading/sequenced_worker_pool.h"
     20 #include "base/time/time.h"
     21 #include "base/timer/timer.h"
     22 #include "base/values.h"
     23 #include "chrome/browser/chromeos/contacts/contact.pb.h"
     24 #include "content/public/browser/browser_thread.h"
     25 #include "google_apis/drive/gdata_contacts_requests.h"
     26 #include "google_apis/drive/gdata_errorcode.h"
     27 #include "google_apis/drive/request_sender.h"
     28 #include "google_apis/drive/time_util.h"
     29 
     30 using content::BrowserThread;
     31 
     32 namespace contacts {
     33 
     34 namespace {
     35 
     36 // Download outcomes reported via the "Contacts.FullUpdateResult" and
     37 // "Contacts.IncrementalUpdateResult" histograms.
     38 enum HistogramResult {
     39   HISTOGRAM_RESULT_SUCCESS = 0,
     40   HISTOGRAM_RESULT_GROUPS_DOWNLOAD_FAILURE = 1,
     41   HISTOGRAM_RESULT_GROUPS_PARSE_FAILURE = 2,
     42   HISTOGRAM_RESULT_MY_CONTACTS_GROUP_NOT_FOUND = 3,
     43   HISTOGRAM_RESULT_CONTACTS_DOWNLOAD_FAILURE = 4,
     44   HISTOGRAM_RESULT_CONTACTS_PARSE_FAILURE = 5,
     45   HISTOGRAM_RESULT_PHOTO_DOWNLOAD_FAILURE = 6,
     46   HISTOGRAM_RESULT_MAX_VALUE = 7,
     47 };
     48 
     49 // Maximum number of profile photos that we'll download per second.
     50 // At values above 10, Google starts returning 503 errors.
     51 const int kMaxPhotoDownloadsPerSecond = 10;
     52 
     53 // Give up after seeing more than this many transient errors while trying to
     54 // download a photo for a single contact.
     55 const int kMaxTransientPhotoDownloadErrorsPerContact = 2;
     56 
     57 // Hardcoded system group ID for the "My Contacts" group, per
     58 // https://developers.google.com/google-apps/contacts/v3/#contact_group_entry.
     59 const char kMyContactsSystemGroupId[] = "Contacts";
     60 
     61 // Top-level field in a contact groups feed containing the list of entries.
     62 const char kGroupEntryField[] = "feed.entry";
     63 
     64 // Field in group entries containing the system group ID (e.g. ID "Contacts"
     65 // for the "My Contacts" system group).  See
     66 // https://developers.google.com/google-apps/contacts/v3/#contact_group_entry
     67 // for more details.
     68 const char kSystemGroupIdField[] = "gContact$systemGroup.id";
     69 
     70 // Field in the top-level object containing the contacts feed.
     71 const char kFeedField[] = "feed";
     72 
     73 // Field in the contacts feed containing a list of category information, along
     74 // with fields within the dictionaries contained in the list and expected
     75 // values.
     76 const char kCategoryField[] = "category";
     77 const char kCategorySchemeField[] = "scheme";
     78 const char kCategorySchemeValue[] = "http://schemas.google.com/g/2005#kind";
     79 const char kCategoryTermField[] = "term";
     80 const char kCategoryTermValue[] =
     81     "http://schemas.google.com/contact/2008#contact";
     82 
     83 // Field in the contacts feed containing a list of contact entries.
     84 const char kEntryField[] = "entry";
     85 
     86 // Field in group and contact entries containing the item's ID.
     87 const char kIdField[] = "id.$t";
     88 
     89 // Top-level fields in contact entries.
     90 const char kDeletedField[] = "gd$deleted";
     91 const char kFullNameField[] = "gd$name.gd$fullName.$t";
     92 const char kGivenNameField[] = "gd$name.gd$givenName.$t";
     93 const char kAdditionalNameField[] = "gd$name.gd$additionalName.$t";
     94 const char kFamilyNameField[] = "gd$name.gd$familyName.$t";
     95 const char kNamePrefixField[] = "gd$name.gd$namePrefix.$t";
     96 const char kNameSuffixField[] = "gd$name.gd$nameSuffix.$t";
     97 const char kEmailField[] = "gd$email";
     98 const char kPhoneField[] = "gd$phoneNumber";
     99 const char kPostalAddressField[] = "gd$structuredPostalAddress";
    100 const char kInstantMessagingField[] = "gd$im";
    101 const char kLinkField[] = "link";
    102 const char kUpdatedField[] = "updated.$t";
    103 
    104 // Fields in entries in the |kEmailField| list.
    105 const char kEmailAddressField[] = "address";
    106 
    107 // Fields in entries in the |kPhoneField| list.
    108 const char kPhoneNumberField[] = "$t";
    109 
    110 // Fields in entries in the |kPostalAddressField| list.
    111 const char kPostalAddressFormattedField[] = "gd$formattedAddress.$t";
    112 
    113 // Fields in entries in the |kInstantMessagingField| list.
    114 const char kInstantMessagingAddressField[] = "address";
    115 const char kInstantMessagingProtocolField[] = "protocol";
    116 const char kInstantMessagingProtocolAimValue[] =
    117     "http://schemas.google.com/g/2005#AIM";
    118 const char kInstantMessagingProtocolMsnValue[] =
    119     "http://schemas.google.com/g/2005#MSN";
    120 const char kInstantMessagingProtocolYahooValue[] =
    121     "http://schemas.google.com/g/2005#YAHOO";
    122 const char kInstantMessagingProtocolSkypeValue[] =
    123     "http://schemas.google.com/g/2005#SKYPE";
    124 const char kInstantMessagingProtocolQqValue[] =
    125     "http://schemas.google.com/g/2005#QQ";
    126 const char kInstantMessagingProtocolGoogleTalkValue[] =
    127     "http://schemas.google.com/g/2005#GOOGLE_TALK";
    128 const char kInstantMessagingProtocolIcqValue[] =
    129     "http://schemas.google.com/g/2005#ICQ";
    130 const char kInstantMessagingProtocolJabberValue[] =
    131     "http://schemas.google.com/g/2005#JABBER";
    132 
    133 // Generic fields shared between address-like items (email, postal, etc.).
    134 const char kAddressPrimaryField[] = "primary";
    135 const char kAddressPrimaryTrueValue[] = "true";
    136 const char kAddressRelField[] = "rel";
    137 const char kAddressRelHomeValue[] = "http://schemas.google.com/g/2005#home";
    138 const char kAddressRelWorkValue[] = "http://schemas.google.com/g/2005#work";
    139 const char kAddressRelMobileValue[] = "http://schemas.google.com/g/2005#mobile";
    140 const char kAddressLabelField[] = "label";
    141 
    142 // Fields in entries in the |kLinkField| list.
    143 const char kLinkHrefField[] = "href";
    144 const char kLinkRelField[] = "rel";
    145 const char kLinkETagField[] = "gd$etag";
    146 const char kLinkRelPhotoValue[] =
    147     "http://schemas.google.com/contacts/2008/rel#photo";
    148 
    149 // Returns a string containing a pretty-printed JSON representation of |value|.
    150 std::string PrettyPrintValue(const base::Value& value) {
    151   std::string out;
    152   base::JSONWriter::WriteWithOptions(
    153       &value, base::JSONWriter::OPTIONS_PRETTY_PRINT, &out);
    154   return out;
    155 }
    156 
    157 // Assigns the value at |path| within |dict| to |out|, returning false if the
    158 // path wasn't present.  Unicode byte order marks are removed from the string.
    159 bool GetCleanedString(const DictionaryValue& dict,
    160                       const std::string& path,
    161                       std::string* out) {
    162   if (!dict.GetString(path, out))
    163     return false;
    164 
    165   // The Unicode byte order mark, U+FEFF, is useless in UTF-8 strings (which are
    166   // interpreted one byte at a time).
    167   ReplaceSubstringsAfterOffset(out, 0, "\xEF\xBB\xBF", "");
    168   return true;
    169 }
    170 
    171 // Returns whether an address is primary, given a dictionary representing a
    172 // single address.
    173 bool IsAddressPrimary(const DictionaryValue& address_dict) {
    174   std::string primary;
    175   address_dict.GetString(kAddressPrimaryField, &primary);
    176   return primary == kAddressPrimaryTrueValue;
    177 }
    178 
    179 // Initializes an AddressType message given a dictionary representing a single
    180 // address.
    181 void InitAddressType(const DictionaryValue& address_dict,
    182                      Contact_AddressType* type) {
    183   DCHECK(type);
    184   type->Clear();
    185 
    186   std::string rel;
    187   address_dict.GetString(kAddressRelField, &rel);
    188   if (rel == kAddressRelHomeValue)
    189     type->set_relation(Contact_AddressType_Relation_HOME);
    190   else if (rel == kAddressRelWorkValue)
    191     type->set_relation(Contact_AddressType_Relation_WORK);
    192   else if (rel == kAddressRelMobileValue)
    193     type->set_relation(Contact_AddressType_Relation_MOBILE);
    194   else
    195     type->set_relation(Contact_AddressType_Relation_OTHER);
    196 
    197   GetCleanedString(address_dict, kAddressLabelField, type->mutable_label());
    198 }
    199 
    200 // Maps the protocol from a dictionary representing a contact's IM address to a
    201 // contacts::Contact_InstantMessagingAddress_Protocol value.
    202 contacts::Contact_InstantMessagingAddress_Protocol
    203 GetInstantMessagingProtocol(const DictionaryValue& im_dict) {
    204   std::string protocol;
    205   im_dict.GetString(kInstantMessagingProtocolField, &protocol);
    206   if (protocol == kInstantMessagingProtocolAimValue)
    207     return contacts::Contact_InstantMessagingAddress_Protocol_AIM;
    208   else if (protocol == kInstantMessagingProtocolMsnValue)
    209     return contacts::Contact_InstantMessagingAddress_Protocol_MSN;
    210   else if (protocol == kInstantMessagingProtocolYahooValue)
    211     return contacts::Contact_InstantMessagingAddress_Protocol_YAHOO;
    212   else if (protocol == kInstantMessagingProtocolSkypeValue)
    213     return contacts::Contact_InstantMessagingAddress_Protocol_SKYPE;
    214   else if (protocol == kInstantMessagingProtocolQqValue)
    215     return contacts::Contact_InstantMessagingAddress_Protocol_QQ;
    216   else if (protocol == kInstantMessagingProtocolGoogleTalkValue)
    217     return contacts::Contact_InstantMessagingAddress_Protocol_GOOGLE_TALK;
    218   else if (protocol == kInstantMessagingProtocolIcqValue)
    219     return contacts::Contact_InstantMessagingAddress_Protocol_ICQ;
    220   else if (protocol == kInstantMessagingProtocolJabberValue)
    221     return contacts::Contact_InstantMessagingAddress_Protocol_JABBER;
    222   else
    223     return contacts::Contact_InstantMessagingAddress_Protocol_OTHER;
    224 }
    225 
    226 // Gets the photo URL from a contact's dictionary (within the "entry" list).
    227 // Returns an empty string if no photo was found.
    228 std::string GetPhotoUrl(const DictionaryValue& dict) {
    229   const ListValue* link_list = NULL;
    230   if (!dict.GetList(kLinkField, &link_list))
    231     return std::string();
    232 
    233   for (size_t i = 0; i < link_list->GetSize(); ++i) {
    234     const DictionaryValue* link_dict = NULL;
    235     if (!link_list->GetDictionary(i, &link_dict))
    236       continue;
    237 
    238     std::string rel;
    239     if (!link_dict->GetString(kLinkRelField, &rel))
    240       continue;
    241     if (rel != kLinkRelPhotoValue)
    242       continue;
    243 
    244     // From https://goo.gl/7T6Od: "If a contact does not have a photo, then the
    245     // photo link element has no gd:etag attribute."
    246     std::string etag;
    247     if (!link_dict->GetString(kLinkETagField, &etag))
    248       continue;
    249 
    250     std::string url;
    251     if (link_dict->GetString(kLinkHrefField, &url))
    252       return url;
    253   }
    254   return std::string();
    255 }
    256 
    257 // Fills a Contact's fields using an entry from a GData feed.
    258 bool FillContactFromDictionary(const base::DictionaryValue& dict,
    259                                contacts::Contact* contact) {
    260   DCHECK(contact);
    261   contact->Clear();
    262 
    263   if (!dict.GetString(kIdField, contact->mutable_contact_id()))
    264     return false;
    265 
    266   std::string updated;
    267   if (dict.GetString(kUpdatedField, &updated)) {
    268     base::Time update_time;
    269     if (!google_apis::util::GetTimeFromString(updated, &update_time)) {
    270       LOG(WARNING) << "Unable to parse time \"" << updated << "\"";
    271       return false;
    272     }
    273     contact->set_update_time(update_time.ToInternalValue());
    274   }
    275 
    276   const base::Value* deleted_value = NULL;
    277   contact->set_deleted(dict.Get(kDeletedField, &deleted_value));
    278   if (contact->deleted())
    279     return true;
    280 
    281   GetCleanedString(dict, kFullNameField, contact->mutable_full_name());
    282   GetCleanedString(dict, kGivenNameField, contact->mutable_given_name());
    283   GetCleanedString(
    284       dict, kAdditionalNameField, contact->mutable_additional_name());
    285   GetCleanedString(dict, kFamilyNameField, contact->mutable_family_name());
    286   GetCleanedString(dict, kNamePrefixField, contact->mutable_name_prefix());
    287   GetCleanedString(dict, kNameSuffixField, contact->mutable_name_suffix());
    288 
    289   const ListValue* email_list = NULL;
    290   if (dict.GetList(kEmailField, &email_list)) {
    291     for (size_t i = 0; i < email_list->GetSize(); ++i) {
    292       const DictionaryValue* email_dict = NULL;
    293       if (!email_list->GetDictionary(i, &email_dict))
    294         return false;
    295 
    296       contacts::Contact_EmailAddress* email = contact->add_email_addresses();
    297       if (!GetCleanedString(*email_dict,
    298                             kEmailAddressField,
    299                             email->mutable_address())) {
    300         return false;
    301       }
    302       email->set_primary(IsAddressPrimary(*email_dict));
    303       InitAddressType(*email_dict, email->mutable_type());
    304     }
    305   }
    306 
    307   const ListValue* phone_list = NULL;
    308   if (dict.GetList(kPhoneField, &phone_list)) {
    309     for (size_t i = 0; i < phone_list->GetSize(); ++i) {
    310       const DictionaryValue* phone_dict = NULL;
    311       if (!phone_list->GetDictionary(i, &phone_dict))
    312         return false;
    313 
    314       contacts::Contact_PhoneNumber* phone = contact->add_phone_numbers();
    315       if (!GetCleanedString(*phone_dict,
    316                             kPhoneNumberField,
    317                             phone->mutable_number())) {
    318         return false;
    319       }
    320       phone->set_primary(IsAddressPrimary(*phone_dict));
    321       InitAddressType(*phone_dict, phone->mutable_type());
    322     }
    323   }
    324 
    325   const ListValue* address_list = NULL;
    326   if (dict.GetList(kPostalAddressField, &address_list)) {
    327     for (size_t i = 0; i < address_list->GetSize(); ++i) {
    328       const DictionaryValue* address_dict = NULL;
    329       if (!address_list->GetDictionary(i, &address_dict))
    330         return false;
    331 
    332       contacts::Contact_PostalAddress* address =
    333           contact->add_postal_addresses();
    334       if (!GetCleanedString(*address_dict,
    335                             kPostalAddressFormattedField,
    336                             address->mutable_address())) {
    337         return false;
    338       }
    339       address->set_primary(IsAddressPrimary(*address_dict));
    340       InitAddressType(*address_dict, address->mutable_type());
    341     }
    342   }
    343 
    344   const ListValue* im_list = NULL;
    345   if (dict.GetList(kInstantMessagingField, &im_list)) {
    346     for (size_t i = 0; i < im_list->GetSize(); ++i) {
    347       const DictionaryValue* im_dict = NULL;
    348       if (!im_list->GetDictionary(i, &im_dict))
    349         return false;
    350 
    351       contacts::Contact_InstantMessagingAddress* im =
    352           contact->add_instant_messaging_addresses();
    353       if (!GetCleanedString(*im_dict,
    354                             kInstantMessagingAddressField,
    355                             im->mutable_address())) {
    356         return false;
    357       }
    358       im->set_primary(IsAddressPrimary(*im_dict));
    359       InitAddressType(*im_dict, im->mutable_type());
    360       im->set_protocol(GetInstantMessagingProtocol(*im_dict));
    361     }
    362   }
    363 
    364   return true;
    365 }
    366 
    367 // Structure into which we parse the contact groups feed using
    368 // JSONValueConverter.
    369 struct ContactGroups {
    370   struct ContactGroup {
    371     // Group ID, e.g.
    372     // "http://www.google.com/m8/feeds/groups/user%40gmail.com/base/6".
    373     std::string group_id;
    374 
    375     // System group ID (e.g. "Contacts" for the "My Contacts" system group) if
    376     // this is a system group, and empty otherwise.  See http://goo.gl/oWVnN
    377     // for more details.
    378     std::string system_group_id;
    379   };
    380 
    381   // Given a system group ID, returns the corresponding group ID or an empty
    382   // string if the requested system group wasn't present.
    383   std::string GetGroupIdForSystemGroup(const std::string& system_group_id) {
    384     for (size_t i = 0; i < groups.size(); ++i) {
    385       const ContactGroup& group = *groups[i];
    386       if (group.system_group_id == system_group_id)
    387         return group.group_id;
    388     }
    389     return std::string();
    390   }
    391 
    392   // Given |value| corresponding to a dictionary in a contact group feed's
    393   // "entry" list, fills |result| with information about the group.
    394   static bool GetContactGroup(const base::Value* value, ContactGroup* result) {
    395     DCHECK(value);
    396     DCHECK(result);
    397     const base::DictionaryValue* dict = NULL;
    398     if (!value->GetAsDictionary(&dict))
    399       return false;
    400 
    401     dict->GetString(kIdField, &result->group_id);
    402     dict->GetString(kSystemGroupIdField, &result->system_group_id);
    403     return true;
    404   }
    405 
    406   static void RegisterJSONConverter(
    407       base::JSONValueConverter<ContactGroups>* converter) {
    408     DCHECK(converter);
    409     converter->RegisterRepeatedCustomValue<ContactGroup>(
    410         kGroupEntryField, &ContactGroups::groups, &GetContactGroup);
    411   }
    412 
    413   ScopedVector<ContactGroup> groups;
    414 };
    415 
    416 }  // namespace
    417 
    418 // This class handles a single request to download all of a user's contacts.
    419 //
    420 // First, the feed containing the user's contact groups is downloaded via
    421 // GetContactGroupsRequest and examined to find the ID for the "My Contacts"
    422 // group (by default, the contacts API also returns suggested contacts).  The
    423 // group ID is cached in GDataContactsService so that this step can be skipped
    424 // by later DownloadContactRequests.
    425 //
    426 // Next, the contacts feed is downloaded via GetContactsRequest and parsed.
    427 // Individual contacts::Contact objects are created using the data from the
    428 // feed.
    429 //
    430 // Finally, GetContactPhotoRequests are created and used to start downloading
    431 // contacts' photos in parallel.  When all photos have been downloaded, the
    432 // contacts are passed to the passed-in callback.
    433 class GDataContactsService::DownloadContactsRequest {
    434  public:
    435   DownloadContactsRequest(
    436       GDataContactsService* service,
    437       google_apis::RequestSender* sender,
    438       SuccessCallback success_callback,
    439       FailureCallback failure_callback,
    440       const base::Time& min_update_time)
    441       : service_(service),
    442         sender_(sender),
    443         success_callback_(success_callback),
    444         failure_callback_(failure_callback),
    445         min_update_time_(min_update_time),
    446         contacts_(new ScopedVector<contacts::Contact>),
    447         my_contacts_group_id_(service->cached_my_contacts_group_id_),
    448         num_in_progress_photo_downloads_(0),
    449         photo_download_failed_(false),
    450         num_photo_download_404_errors_(0),
    451         total_photo_bytes_(0),
    452         weak_ptr_factory_(this) {
    453     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    454     DCHECK(service_);
    455     DCHECK(sender_);
    456   }
    457 
    458   ~DownloadContactsRequest() {
    459     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    460     service_ = NULL;
    461     sender_ = NULL;
    462   }
    463 
    464   const std::string my_contacts_group_id() const {
    465     return my_contacts_group_id_;
    466   }
    467 
    468   // Begins the contacts-downloading process.  If the ID for the "My Contacts"
    469   // group has previously been cached, then the contacts download is started.
    470   // Otherwise, the contact groups download is started.
    471   void Run() {
    472     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    473     download_start_time_ = base::TimeTicks::Now();
    474     if (!my_contacts_group_id_.empty()) {
    475       StartContactsDownload();
    476     } else {
    477       google_apis::GetContactGroupsRequest* operation =
    478           new google_apis::GetContactGroupsRequest(
    479               sender_,
    480               base::Bind(&DownloadContactsRequest::HandleGroupsFeedData,
    481                          weak_ptr_factory_.GetWeakPtr()));
    482       if (!service_->groups_feed_url_for_testing_.is_empty()) {
    483         operation->set_feed_url_for_testing(
    484             service_->groups_feed_url_for_testing_);
    485       }
    486       sender_->StartRequestWithRetry(operation);
    487     }
    488   }
    489 
    490  private:
    491   // Invokes the failure callback and notifies GDataContactsService that the
    492   // request is done.
    493   void ReportFailure(HistogramResult histogram_result) {
    494     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    495     SendHistograms(histogram_result);
    496     failure_callback_.Run();
    497     service_->OnRequestComplete(this);
    498   }
    499 
    500   // Reports UMA stats after the request has completed.
    501   void SendHistograms(HistogramResult result) {
    502     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    503     DCHECK_GE(result, 0);
    504     DCHECK_LT(result, HISTOGRAM_RESULT_MAX_VALUE);
    505 
    506     bool success = (result == HISTOGRAM_RESULT_SUCCESS);
    507     base::TimeDelta elapsed_time =
    508         base::TimeTicks::Now() - download_start_time_;
    509     int photo_error_percent = static_cast<int>(
    510         100.0 * transient_photo_download_errors_per_contact_.size() /
    511         contact_photo_urls_.size() + 0.5);
    512 
    513     if (min_update_time_.is_null()) {
    514       UMA_HISTOGRAM_ENUMERATION("Contacts.FullUpdateResult",
    515                                 result, HISTOGRAM_RESULT_MAX_VALUE);
    516       if (success) {
    517         UMA_HISTOGRAM_MEDIUM_TIMES("Contacts.FullUpdateDuration",
    518                                    elapsed_time);
    519         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdateContacts",
    520                                    contacts_->size());
    521         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdatePhotos",
    522                                    contact_photo_urls_.size());
    523         UMA_HISTOGRAM_MEMORY_KB("Contacts.FullUpdatePhotoBytes",
    524                                 total_photo_bytes_);
    525         UMA_HISTOGRAM_COUNTS_10000("Contacts.FullUpdatePhoto404Errors",
    526                                    num_photo_download_404_errors_);
    527         UMA_HISTOGRAM_PERCENTAGE("Contacts.FullUpdatePhotoErrorPercent",
    528                                  photo_error_percent);
    529       }
    530     } else {
    531       UMA_HISTOGRAM_ENUMERATION("Contacts.IncrementalUpdateResult",
    532                                 result, HISTOGRAM_RESULT_MAX_VALUE);
    533       if (success) {
    534         UMA_HISTOGRAM_MEDIUM_TIMES("Contacts.IncrementalUpdateDuration",
    535                                    elapsed_time);
    536         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdateContacts",
    537                                    contacts_->size());
    538         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdatePhotos",
    539                                    contact_photo_urls_.size());
    540         UMA_HISTOGRAM_MEMORY_KB("Contacts.IncrementalUpdatePhotoBytes",
    541                                 total_photo_bytes_);
    542         UMA_HISTOGRAM_COUNTS_10000("Contacts.IncrementalUpdatePhoto404Errors",
    543                                    num_photo_download_404_errors_);
    544         UMA_HISTOGRAM_PERCENTAGE("Contacts.IncrementalUpdatePhotoErrorPercent",
    545                                  photo_error_percent);
    546       }
    547     }
    548   }
    549 
    550   // Callback for GetContactGroupsRequest calls.  Starts downloading the
    551   // actual contacts after finding the "My Contacts" group ID.
    552   void HandleGroupsFeedData(google_apis::GDataErrorCode error,
    553                             scoped_ptr<base::Value> feed_data) {
    554     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    555     if (error != google_apis::HTTP_SUCCESS) {
    556       LOG(WARNING) << "Got error " << error << " while downloading groups";
    557       ReportFailure(HISTOGRAM_RESULT_GROUPS_DOWNLOAD_FAILURE);
    558       return;
    559     }
    560 
    561     VLOG(2) << "Got groups feed data:\n"
    562             << PrettyPrintValue(*(feed_data.get()));
    563     ContactGroups groups;
    564     base::JSONValueConverter<ContactGroups> converter;
    565     if (!converter.Convert(*feed_data, &groups)) {
    566       LOG(WARNING) << "Unable to parse groups feed";
    567       ReportFailure(HISTOGRAM_RESULT_GROUPS_PARSE_FAILURE);
    568       return;
    569     }
    570 
    571     my_contacts_group_id_ =
    572         groups.GetGroupIdForSystemGroup(kMyContactsSystemGroupId);
    573     if (!my_contacts_group_id_.empty()) {
    574       StartContactsDownload();
    575     } else {
    576       LOG(WARNING) << "Unable to find ID for \"My Contacts\" group";
    577       ReportFailure(HISTOGRAM_RESULT_MY_CONTACTS_GROUP_NOT_FOUND);
    578     }
    579   }
    580 
    581   // Starts a download of the contacts from the "My Contacts" group.
    582   void StartContactsDownload() {
    583     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    584     google_apis::GetContactsRequest* operation =
    585         new google_apis::GetContactsRequest(
    586             sender_,
    587             my_contacts_group_id_,
    588             min_update_time_,
    589             base::Bind(&DownloadContactsRequest::HandleContactsFeedData,
    590                        weak_ptr_factory_.GetWeakPtr()));
    591     if (!service_->contacts_feed_url_for_testing_.is_empty()) {
    592       operation->set_feed_url_for_testing(
    593           service_->contacts_feed_url_for_testing_);
    594     }
    595     sender_->StartRequestWithRetry(operation);
    596   }
    597 
    598   // Callback for GetContactsRequest calls.
    599   void HandleContactsFeedData(google_apis::GDataErrorCode error,
    600                               scoped_ptr<base::Value> feed_data) {
    601     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    602     if (error != google_apis::HTTP_SUCCESS) {
    603       LOG(WARNING) << "Got error " << error << " while downloading contacts";
    604       ReportFailure(HISTOGRAM_RESULT_CONTACTS_DOWNLOAD_FAILURE);
    605       return;
    606     }
    607 
    608     VLOG(2) << "Got contacts feed data:\n"
    609             << PrettyPrintValue(*(feed_data.get()));
    610     if (!ProcessContactsFeedData(*feed_data.get())) {
    611       LOG(WARNING) << "Unable to process contacts feed data";
    612       ReportFailure(HISTOGRAM_RESULT_CONTACTS_PARSE_FAILURE);
    613       return;
    614     }
    615 
    616     StartPhotoDownloads();
    617     photo_download_timer_.Start(
    618         FROM_HERE, service_->photo_download_timer_interval_,
    619         this, &DownloadContactsRequest::StartPhotoDownloads);
    620     CheckCompletion();
    621   }
    622 
    623   // Processes the raw contacts feed from |feed_data| and fills |contacts_|.
    624   // Returns true on success.
    625   bool ProcessContactsFeedData(const base::Value& feed_data) {
    626     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    627     const DictionaryValue* toplevel_dict = NULL;
    628     if (!feed_data.GetAsDictionary(&toplevel_dict)) {
    629       LOG(WARNING) << "Top-level object is not a dictionary";
    630       return false;
    631     }
    632 
    633     const DictionaryValue* feed_dict = NULL;
    634     if (!toplevel_dict->GetDictionary(kFeedField, &feed_dict)) {
    635       LOG(WARNING) << "Feed dictionary missing";
    636       return false;
    637     }
    638 
    639     // Check the category field to confirm that this is actually a contact feed.
    640     const ListValue* category_list = NULL;
    641     if (!feed_dict->GetList(kCategoryField, &category_list)) {
    642       LOG(WARNING) << "Category list missing";
    643       return false;
    644     }
    645     const DictionaryValue* category_dict = NULL;
    646     if (category_list->GetSize() != 1 ||
    647         !category_list->GetDictionary(0, &category_dict)) {
    648       LOG(WARNING) << "Unable to get dictionary from category list of size "
    649                    << category_list->GetSize();
    650       return false;
    651     }
    652     std::string category_scheme, category_term;
    653     if (!category_dict->GetString(kCategorySchemeField, &category_scheme) ||
    654         !category_dict->GetString(kCategoryTermField, &category_term) ||
    655         category_scheme != kCategorySchemeValue ||
    656         category_term != kCategoryTermValue) {
    657       LOG(WARNING) << "Unexpected category (scheme was \"" << category_scheme
    658                    << "\", term was \"" << category_term << "\")";
    659       return false;
    660     }
    661 
    662     // A missing entry list means no entries (maybe we're doing an incremental
    663     // update and nothing has changed).
    664     const ListValue* entry_list = NULL;
    665     if (!feed_dict->GetList(kEntryField, &entry_list))
    666       return true;
    667 
    668     contacts_needing_photo_downloads_.reserve(entry_list->GetSize());
    669 
    670     for (ListValue::const_iterator entry_it = entry_list->begin();
    671          entry_it != entry_list->end(); ++entry_it) {
    672       const size_t index = (entry_it - entry_list->begin());
    673       const DictionaryValue* contact_dict = NULL;
    674       if (!(*entry_it)->GetAsDictionary(&contact_dict)) {
    675         LOG(WARNING) << "Entry " << index << " isn't a dictionary";
    676         return false;
    677       }
    678 
    679       scoped_ptr<contacts::Contact> contact(new contacts::Contact);
    680       if (!FillContactFromDictionary(*contact_dict, contact.get())) {
    681         LOG(WARNING) << "Unable to fill entry " << index;
    682         return false;
    683       }
    684 
    685       VLOG(1) << "Got contact " << index << ":"
    686               << " id=" << contact->contact_id()
    687               << " full_name=\"" << contact->full_name() << "\""
    688               << " update_time=" << contact->update_time();
    689 
    690       std::string photo_url = GetPhotoUrl(*contact_dict);
    691       if (!photo_url.empty()) {
    692         if (!service_->rewrite_photo_url_callback_for_testing_.is_null()) {
    693           photo_url =
    694               service_->rewrite_photo_url_callback_for_testing_.Run(photo_url);
    695         }
    696         contact_photo_urls_[contact.get()] = photo_url;
    697         contacts_needing_photo_downloads_.push_back(contact.get());
    698       }
    699 
    700       contacts_->push_back(contact.release());
    701     }
    702 
    703     return true;
    704   }
    705 
    706   // If we're done downloading photos, invokes a callback and deletes |this|.
    707   // Otherwise, starts one or more downloads of URLs from
    708   // |contacts_needing_photo_downloads_|.
    709   void CheckCompletion() {
    710     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    711     if (contacts_needing_photo_downloads_.empty() &&
    712         num_in_progress_photo_downloads_ == 0) {
    713       VLOG(1) << "Done downloading photos; invoking callback";
    714       photo_download_timer_.Stop();
    715       if (photo_download_failed_) {
    716         ReportFailure(HISTOGRAM_RESULT_PHOTO_DOWNLOAD_FAILURE );
    717       } else {
    718         SendHistograms(HISTOGRAM_RESULT_SUCCESS);
    719         success_callback_.Run(contacts_.Pass());
    720         service_->OnRequestComplete(this);
    721       }
    722       return;
    723     }
    724   }
    725 
    726   // Starts photo downloads for contacts in |contacts_needing_photo_downloads_|.
    727   // Should be invoked only once per second.
    728   void StartPhotoDownloads() {
    729     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    730     while (!contacts_needing_photo_downloads_.empty() &&
    731            (num_in_progress_photo_downloads_ <
    732             service_->max_photo_downloads_per_second_)) {
    733       contacts::Contact* contact = contacts_needing_photo_downloads_.back();
    734       contacts_needing_photo_downloads_.pop_back();
    735       DCHECK(contact_photo_urls_.count(contact));
    736       std::string url = contact_photo_urls_[contact];
    737 
    738       VLOG(1) << "Starting download of photo " << url << " for "
    739               << contact->contact_id();
    740       sender_->StartRequestWithRetry(
    741           new google_apis::GetContactPhotoRequest(
    742               sender_,
    743               GURL(url),
    744               base::Bind(&DownloadContactsRequest::HandlePhotoData,
    745                          weak_ptr_factory_.GetWeakPtr(),
    746                          contact)));
    747       num_in_progress_photo_downloads_++;
    748     }
    749   }
    750 
    751   // Callback for GetContactPhotoRequest calls.  Updates the associated
    752   // Contact and checks for completion.
    753   void HandlePhotoData(contacts::Contact* contact,
    754                        google_apis::GDataErrorCode error,
    755                        scoped_ptr<std::string> download_data) {
    756     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    757     VLOG(1) << "Got photo data for " << contact->contact_id()
    758             << " (error=" << error << " size=" << download_data->size() << ")";
    759     num_in_progress_photo_downloads_--;
    760 
    761     if (error == google_apis::HTTP_INTERNAL_SERVER_ERROR ||
    762         error == google_apis::HTTP_SERVICE_UNAVAILABLE) {
    763       int num_errors = ++transient_photo_download_errors_per_contact_[contact];
    764       if (num_errors <= kMaxTransientPhotoDownloadErrorsPerContact) {
    765         LOG(WARNING) << "Got error " << error << " while downloading photo "
    766                      << "for " << contact->contact_id() << "; retrying";
    767         contacts_needing_photo_downloads_.push_back(contact);
    768         return;
    769       }
    770     }
    771 
    772     if (error == google_apis::HTTP_NOT_FOUND) {
    773       LOG(WARNING) << "Got error " << error << " while downloading photo "
    774                    << "for " << contact->contact_id() << "; skipping";
    775       num_photo_download_404_errors_++;
    776       CheckCompletion();
    777       return;
    778     }
    779 
    780     if (error != google_apis::HTTP_SUCCESS) {
    781       LOG(WARNING) << "Got error " << error << " while downloading photo "
    782                    << "for " << contact->contact_id() << "; giving up";
    783       photo_download_failed_ = true;
    784       // Make sure we don't start any more downloads.
    785       contacts_needing_photo_downloads_.clear();
    786       CheckCompletion();
    787       return;
    788     }
    789 
    790     total_photo_bytes_ += download_data->size();
    791     contact->set_raw_untrusted_photo(*download_data);
    792     CheckCompletion();
    793   }
    794 
    795   typedef std::map<contacts::Contact*, std::string> ContactPhotoUrls;
    796 
    797   GDataContactsService* service_;  // not owned
    798   google_apis::RequestSender* sender_;  // not owned
    799 
    800   SuccessCallback success_callback_;
    801   FailureCallback failure_callback_;
    802 
    803   base::Time min_update_time_;
    804 
    805   scoped_ptr<ScopedVector<contacts::Contact> > contacts_;
    806 
    807   // ID of the "My Contacts" contacts group.
    808   std::string my_contacts_group_id_;
    809 
    810   // Map from a contact to the URL at which its photo is located.
    811   // Contacts without photos do not appear in this map.
    812   ContactPhotoUrls contact_photo_urls_;
    813 
    814   // Invokes StartPhotoDownloads() once per second.
    815   base::RepeatingTimer<DownloadContactsRequest> photo_download_timer_;
    816 
    817   // Contacts that have photos that we still need to start downloading.
    818   // When we start a download, the contact is removed from this list.
    819   std::vector<contacts::Contact*> contacts_needing_photo_downloads_;
    820 
    821   // Number of in-progress photo downloads.
    822   int num_in_progress_photo_downloads_;
    823 
    824   // Map from a contact to the number of transient errors that we've encountered
    825   // while trying to download its photo.  Contacts for which no errors have been
    826   // encountered aren't represented in the map.
    827   std::map<contacts::Contact*, int>
    828       transient_photo_download_errors_per_contact_;
    829 
    830   // Did we encounter a fatal error while downloading a photo?
    831   bool photo_download_failed_;
    832 
    833   // How many photos did we skip due to 404 errors?
    834   int num_photo_download_404_errors_;
    835 
    836   // Total size of all photos that were downloaded.
    837   size_t total_photo_bytes_;
    838 
    839   // Time at which Run() was called.
    840   base::TimeTicks download_start_time_;
    841 
    842   // Note: This should remain the last member so it'll be destroyed and
    843   // invalidate its weak pointers before any other members are destroyed.
    844   base::WeakPtrFactory<DownloadContactsRequest> weak_ptr_factory_;
    845 
    846   DISALLOW_COPY_AND_ASSIGN(DownloadContactsRequest);
    847 };
    848 
    849 GDataContactsService::GDataContactsService(
    850     net::URLRequestContextGetter* url_request_context_getter,
    851     google_apis::AuthServiceInterface* auth_service)
    852     : max_photo_downloads_per_second_(kMaxPhotoDownloadsPerSecond),
    853       photo_download_timer_interval_(base::TimeDelta::FromSeconds(1)) {
    854   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    855   sender_.reset(new google_apis::RequestSender(
    856       auth_service,
    857       url_request_context_getter,
    858       BrowserThread::GetBlockingPool()->GetSequencedTaskRunner(
    859           BrowserThread::GetBlockingPool()->GetSequenceToken()).get(),
    860       "" /* custom_user_agent */));
    861 }
    862 
    863 GDataContactsService::~GDataContactsService() {
    864   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    865   STLDeleteContainerPointers(requests_.begin(), requests_.end());
    866   requests_.clear();
    867 }
    868 
    869 void GDataContactsService::DownloadContacts(SuccessCallback success_callback,
    870                                             FailureCallback failure_callback,
    871                                             const base::Time& min_update_time) {
    872   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    873   DownloadContactsRequest* request =
    874       new DownloadContactsRequest(this,
    875                                   sender_.get(),
    876                                   success_callback,
    877                                   failure_callback,
    878                                   min_update_time);
    879   VLOG(1) << "Starting contacts download with request " << request;
    880   requests_.insert(request);
    881   request->Run();
    882 }
    883 
    884 void GDataContactsService::OnRequestComplete(DownloadContactsRequest* request) {
    885   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    886   DCHECK(request);
    887   VLOG(1) << "Download request " << request << " complete";
    888   if (!request->my_contacts_group_id().empty())
    889     cached_my_contacts_group_id_ = request->my_contacts_group_id();
    890   requests_.erase(request);
    891   delete request;
    892 }
    893 
    894 }  // namespace contacts
    895