1 // Copyright (C) 2013 Google Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "retriever.h" 16 17 #include <libaddressinput/callback.h> 18 #include <libaddressinput/downloader.h> 19 #include <libaddressinput/storage.h> 20 #include <libaddressinput/util/basictypes.h> 21 #include <libaddressinput/util/scoped_ptr.h> 22 23 #include <cassert> 24 #include <cstddef> 25 #include <cstdlib> 26 #include <ctime> 27 #include <map> 28 #include <string> 29 30 #include "fallback_data_store.h" 31 #include "util/md5.h" 32 #include "util/stl_util.h" 33 #include "util/string_util.h" 34 35 namespace i18n { 36 namespace addressinput { 37 38 namespace { 39 40 // The number of seconds after which data is considered stale. The staleness 41 // threshold is 30 days: 42 // 30 days * 43 // 24 hours per day * 44 // 60 minutes per hour * 45 // 60 seconds per minute. 46 static const double kStaleDataAgeInSeconds = 30.0 * 24.0 * 60.0 * 60.0; 47 48 // The prefix for the timestamp line in the footer. 49 const char kTimestampPrefix[] = "timestamp="; 50 51 // The prefix for the checksum line in the footer. 52 const char kChecksumPrefix[] = "checksum="; 53 54 // The separator between lines of footer and data. 55 const char kSeparator = '\n'; 56 57 // Returns |data| with attached checksum and current timestamp. Format: 58 // 59 // <data> 60 // checksum=<checksum> 61 // timestamp=<timestamp> 62 // 63 // The timestamp is the time_t that was returned from time(NULL) function. The 64 // timestamp does not need to be portable because it is written and read only by 65 // Retriever. The value is somewhat human-readable: it is the number of seconds 66 // since the epoch. 67 // 68 // The checksum is the 32-character hexadecimal MD5 checksum of <data>. It is 69 // meant to protect from random file changes on disk. 70 void AppendTimestamp(std::string* data) { 71 std::string md5 = MD5String(*data); 72 73 data->push_back(kSeparator); 74 data->append(kChecksumPrefix); 75 data->append(md5); 76 77 data->push_back(kSeparator); 78 data->append(kTimestampPrefix); 79 data->append(TimeToString(time(NULL))); 80 } 81 82 // Places the footer value into |footer_value| parameter and the rest of the 83 // data into |data| parameter. Returns |true| if the footer format is valid. 84 bool ExtractFooter(scoped_ptr<std::string> data_and_footer, 85 const std::string& footer_prefix, 86 std::string* footer_value, 87 scoped_ptr<std::string>* data) { 88 assert(footer_value != NULL); 89 assert(data != NULL); 90 91 std::string::size_type separator_position = 92 data_and_footer->rfind(kSeparator); 93 if (separator_position == std::string::npos) { 94 return false; 95 } 96 97 std::string::size_type footer_start = separator_position + 1; 98 if (data_and_footer->compare(footer_start, 99 footer_prefix.length(), 100 footer_prefix) != 0) { 101 return false; 102 } 103 104 *footer_value = 105 data_and_footer->substr(footer_start + footer_prefix.length()); 106 *data = data_and_footer.Pass(); 107 (*data)->resize(separator_position); 108 return true; 109 } 110 111 // Strips out the timestamp and checksum from |data_and_footer|. Validates the 112 // checksum. Saves the footer-less data into |data|. Compares the parsed 113 // timestamp with current time and saves the difference into |age_in_seconds|. 114 // 115 // The parameters should not be NULL. 116 // 117 // Returns |true| if |data_and_footer| is correctly formatted and has the 118 // correct checksum. 119 bool VerifyAndExtractTimestamp(const std::string& data_and_footer, 120 scoped_ptr<std::string>* data, 121 double* age_in_seconds) { 122 assert(data != NULL); 123 assert(age_in_seconds != NULL); 124 125 std::string timestamp_string; 126 scoped_ptr<std::string> checksum_and_data; 127 if (!ExtractFooter(make_scoped_ptr(new std::string(data_and_footer)), 128 kTimestampPrefix, ×tamp_string, &checksum_and_data)) { 129 return false; 130 } 131 132 time_t timestamp = atol(timestamp_string.c_str()); 133 if (timestamp < 0) { 134 return false; 135 } 136 137 *age_in_seconds = difftime(time(NULL), timestamp); 138 if (*age_in_seconds < 0.0) { 139 return false; 140 } 141 142 std::string checksum; 143 if (!ExtractFooter(checksum_and_data.Pass(), 144 kChecksumPrefix, &checksum, data)) { 145 return false; 146 } 147 148 return checksum == MD5String(**data); 149 } 150 151 } // namespace 152 153 Retriever::Retriever(const std::string& validation_data_url, 154 scoped_ptr<Downloader> downloader, 155 scoped_ptr<Storage> storage) 156 : validation_data_url_(validation_data_url), 157 downloader_(downloader.Pass()), 158 storage_(storage.Pass()), 159 stale_data_() { 160 assert(validation_data_url_.length() > 0); 161 assert(validation_data_url_[validation_data_url_.length() - 1] == '/'); 162 assert(storage_ != NULL); 163 assert(downloader_ != NULL); 164 } 165 166 Retriever::~Retriever() { 167 STLDeleteValues(&requests_); 168 } 169 170 void Retriever::Retrieve(const std::string& key, 171 scoped_ptr<Callback> retrieved) { 172 std::map<std::string, Callback*>::iterator request_it = 173 requests_.find(key); 174 if (request_it != requests_.end()) { 175 // Abandon a previous request. 176 delete request_it->second; 177 requests_.erase(request_it); 178 } 179 180 requests_[key] = retrieved.release(); 181 storage_->Get(key, 182 BuildCallback(this, &Retriever::OnDataRetrievedFromStorage)); 183 } 184 185 void Retriever::OnDataRetrievedFromStorage(bool success, 186 const std::string& key, 187 const std::string& stored_data) { 188 scoped_ptr<std::string> unwrapped; 189 double age_in_seconds = 0.0; 190 if (success && 191 VerifyAndExtractTimestamp(stored_data, &unwrapped, &age_in_seconds)) { 192 if (age_in_seconds < kStaleDataAgeInSeconds) { 193 if (InvokeCallbackForKey(key, success, *unwrapped)) { 194 return; 195 } 196 } else { 197 stale_data_[key].swap(*unwrapped); 198 } 199 } 200 201 downloader_->Download(GetUrlForKey(key), 202 BuildScopedPtrCallback(this, &Retriever::OnDownloaded)); 203 } 204 205 void Retriever::OnDownloaded(bool success, 206 const std::string& url, 207 scoped_ptr<std::string> downloaded_data) { 208 const std::string& key = GetKeyForUrl(url); 209 std::map<std::string, std::string>::iterator stale_data_it = 210 stale_data_.find(key); 211 212 // This variable tracks whether the client "likes" the data we return. For 213 // example, it could be corrupt --- in this case, we won't place it in 214 // storage. 215 bool data_is_good = false; 216 if (success) { 217 data_is_good = InvokeCallbackForKey(key, success, *downloaded_data); 218 if (data_is_good) { 219 AppendTimestamp(downloaded_data.get()); 220 storage_->Put(key, downloaded_data.Pass()); 221 } 222 } else if (stale_data_it != stale_data_.end()) { 223 data_is_good = InvokeCallbackForKey(key, true, stale_data_it->second); 224 } 225 226 if (!success || !data_is_good) { 227 std::string fallback; 228 success = FallbackDataStore::Get(key, &fallback); 229 InvokeCallbackForKey(key, success, fallback); 230 } 231 232 if (stale_data_it != stale_data_.end()) { 233 stale_data_.erase(stale_data_it); 234 } 235 } 236 237 std::string Retriever::GetUrlForKey(const std::string& key) const { 238 return validation_data_url_ + key; 239 } 240 241 std::string Retriever::GetKeyForUrl(const std::string& url) const { 242 return 243 url.compare(0, validation_data_url_.length(), validation_data_url_) == 0 244 ? url.substr(validation_data_url_.length()) 245 : std::string(); 246 } 247 248 bool Retriever::IsValidationDataUrl(const std::string& url) const { 249 return 250 url.compare(0, validation_data_url_.length(), validation_data_url_) == 0; 251 } 252 253 bool Retriever::InvokeCallbackForKey(const std::string& key, 254 bool success, 255 const std::string& data) { 256 std::map<std::string, Callback*>::iterator iter = 257 requests_.find(key); 258 if (iter == requests_.end()) { 259 // An abandoned request. 260 return true; 261 } 262 263 scoped_ptr<Callback> callback(iter->second); 264 // If the data is no good, put the request back. 265 if (callback != NULL && !(*callback)(success, key, data)) { 266 requests_[key] = callback.release(); 267 return false; 268 } 269 270 requests_.erase(iter); 271 return true; 272 } 273 274 } // namespace addressinput 275 } // namespace i18n 276