1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_ 17 #define TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_ 18 19 #include <string> 20 #include <unordered_map> 21 #include <vector> 22 #include <curl/curl.h> 23 #include "tensorflow/core/lib/core/errors.h" 24 #include "tensorflow/core/lib/core/status.h" 25 #include "tensorflow/core/lib/core/stringpiece.h" 26 #include "tensorflow/core/platform/cloud/http_request.h" 27 #include "tensorflow/core/platform/env.h" 28 #include "tensorflow/core/platform/macros.h" 29 #include "tensorflow/core/platform/protobuf.h" 30 #include "tensorflow/core/platform/types.h" 31 32 namespace tensorflow { 33 34 class LibCurl; // libcurl interface as a class, for dependency injection. 35 36 /// \brief A basic HTTP client based on the libcurl library. 37 /// 38 /// The usage pattern for the class reflects the one of the libcurl library: 39 /// create a request object, set request parameters and call Send(). 40 /// 41 /// For example: 42 /// std::unique_ptr<HttpRequest> request(http_request_factory->Create()); 43 /// request->SetUri("http://www.google.com"); 44 /// request->SetResultsBuffer(out_buffer); 45 /// request->Send(); 46 class CurlHttpRequest : public HttpRequest { 47 public: 48 class Factory : public HttpRequest::Factory { 49 public: 50 virtual ~Factory() {} 51 virtual HttpRequest* Create() { return new CurlHttpRequest(); } 52 }; 53 54 CurlHttpRequest(); 55 explicit CurlHttpRequest(LibCurl* libcurl) 56 : CurlHttpRequest(libcurl, Env::Default()) {} 57 CurlHttpRequest(LibCurl* libcurl, Env* env); 58 ~CurlHttpRequest() override; 59 60 /// Sets the request URI. 61 void SetUri(const string& uri) override; 62 63 /// \brief Sets the Range header. 64 /// 65 /// Used for random seeks, for example "0-999" returns the first 1000 bytes 66 /// (note that the right border is included). 67 void SetRange(uint64 start, uint64 end) override; 68 69 /// Sets a request header. 70 void AddHeader(const string& name, const string& value) override; 71 72 void AddResolveOverride(const string& hostname, int64 port, 73 const string& ip_addr) override; 74 75 /// Sets the 'Authorization' header to the value of 'Bearer ' + auth_token. 76 void AddAuthBearerHeader(const string& auth_token) override; 77 78 /// Makes the request a DELETE request. 79 void SetDeleteRequest() override; 80 81 /// \brief Makes the request a PUT request. 82 /// 83 /// The request body will be taken from the specified file starting from 84 /// the given offset. 85 Status SetPutFromFile(const string& body_filepath, size_t offset) override; 86 87 /// Makes the request a PUT request with an empty body. 88 void SetPutEmptyBody() override; 89 90 /// \brief Makes the request a POST request. 91 /// 92 /// The request body will be taken from the specified buffer. 93 void SetPostFromBuffer(const char* buffer, size_t size) override; 94 95 /// Makes the request a POST request with an empty body. 96 void SetPostEmptyBody() override; 97 98 /// \brief Specifies the buffer for receiving the response body. 99 /// 100 /// Size of out_buffer after an access will be exactly the number of bytes 101 /// read. Existing content of the vector will be cleared. 102 void SetResultBuffer(std::vector<char>* out_buffer) override; 103 104 /// \brief Specifies the buffer for receiving the response body, when the 105 /// caller knows the maximum size of the response body. 106 /// 107 /// This method allows the caller to receive the response body without an 108 /// additional intermediate buffer allocation and copy. This method should 109 /// be called before calling Send(). After Send() has succeeded, the caller 110 /// should use the GetResultBufferDirectBytesTransferred() method in order 111 /// to learn how many bytes were transferred. 112 /// 113 /// Using this method is mutually exclusive with using SetResultBuffer(). 114 void SetResultBufferDirect(char* buffer, size_t size) override; 115 116 /// \brief Distinguish response type (direct vs. implicit). 117 bool IsDirectResponse() const; 118 119 /// \brief Returns the number of bytes (of the response body) that were 120 /// transferred, when using the SetResultBufferDirect() method. The returned 121 /// value will always be less than or equal to the 'size' parameter that 122 /// was passed to SetResultBufferDirect(). If the actual HTTP response body 123 /// was greater than 'size' bytes, then this transfer method will only copy 124 /// the first 'size' bytes, and the rest will be ignored. 125 size_t GetResultBufferDirectBytesTransferred() override; 126 127 /// \brief Returns the response headers of a completed request. 128 /// 129 /// If the header is not found, returns an empty string. 130 string GetResponseHeader(const string& name) const override; 131 132 /// Returns the response code of a completed request. 133 uint64 GetResponseCode() const override; 134 135 /// \brief Sends the formed request. 136 /// 137 /// If the result buffer was defined, the response will be written there. 138 /// The object is not designed to be re-used after Send() is executed. 139 Status Send() override; 140 141 // Url encodes str and returns a new string. 142 string EscapeString(const string& str) override; 143 144 void SetTimeouts(uint32 connection, uint32 inactivity, uint32 total) override; 145 146 private: 147 /// A write callback in the form which can be accepted by libcurl. 148 static size_t WriteCallback(const void* ptr, size_t size, size_t nmemb, 149 void* userdata); 150 151 /// Processes response body content received when using SetResultBufferDirect. 152 static size_t WriteCallbackDirect(const void* ptr, size_t size, size_t nmemb, 153 void* userdata); 154 /// A read callback in the form which can be accepted by libcurl. 155 static size_t ReadCallback(void* ptr, size_t size, size_t nmemb, 156 FILE* userdata); 157 /// A header callback in the form which can be accepted by libcurl. 158 static size_t HeaderCallback(const void* ptr, size_t size, size_t nmemb, 159 void* this_object); 160 /// A progress meter callback in the form which can be accepted by libcurl. 161 static int ProgressCallback(void* this_object, curl_off_t dltotal, 162 curl_off_t dlnow, curl_off_t ultotal, 163 curl_off_t ulnow); 164 void CheckMethodNotSet() const; 165 void CheckNotSent() const; 166 StringPiece GetResponse() const; 167 168 LibCurl* libcurl_; 169 Env* env_; 170 171 FILE* put_body_ = nullptr; 172 173 StringPiece post_body_buffer_; 174 size_t post_body_read_ = 0; 175 176 std::vector<char>* response_buffer_ = nullptr; 177 178 struct DirectResponseState { 179 char* buffer_; 180 size_t buffer_size_; 181 size_t bytes_transferred_; 182 }; 183 DirectResponseState direct_response_ = {}; 184 185 CURL* curl_ = nullptr; 186 curl_slist* curl_headers_ = nullptr; 187 curl_slist* resolve_list_ = nullptr; 188 189 std::vector<char> default_response_buffer_; 190 191 std::unordered_map<string, string> response_headers_; 192 uint64 response_code_ = 0; 193 194 // The timestamp of the last activity related to the request execution, in 195 // seconds since epoch. 196 uint64 last_progress_timestamp_ = 0; 197 // The last progress in terms of bytes transmitted. 198 curl_off_t last_progress_bytes_ = 0; 199 200 // The maximum period of request inactivity. 201 uint32 inactivity_timeout_secs_ = 60; // 1 minute 202 203 // Timeout for the connection phase. 204 uint32 connect_timeout_secs_ = 120; // 2 minutes 205 206 // Tiemout for the whole request. Set only to prevent hanging indefinitely. 207 uint32 request_timeout_secs_ = 3600; // 1 hour 208 209 // Members to enforce the usage flow. 210 bool is_uri_set_ = false; 211 bool is_method_set_ = false; 212 bool is_sent_ = false; 213 214 // Store the URI to help disambiguate requests when errors occur. 215 string uri_; 216 217 // Limit the size of a http response that is copied into an error message. 218 const size_t response_to_error_limit_ = 500; 219 220 TF_DISALLOW_COPY_AND_ASSIGN(CurlHttpRequest); 221 }; 222 223 /// \brief A proxy to the libcurl C interface as a dependency injection measure. 224 /// 225 /// This class is meant as a very thin wrapper for the libcurl C library. 226 class LibCurl { 227 public: 228 virtual ~LibCurl() {} 229 230 virtual CURL* curl_easy_init() = 0; 231 virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option, 232 uint64 param) = 0; 233 virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option, 234 const char* param) = 0; 235 virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option, 236 void* param) = 0; 237 virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option, 238 size_t (*param)(void*, size_t, size_t, 239 FILE*)) = 0; 240 virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option, 241 size_t (*param)(const void*, size_t, size_t, 242 void*)) = 0; 243 virtual CURLcode curl_easy_setopt( 244 CURL* curl, CURLoption option, 245 int (*param)(void* clientp, curl_off_t dltotal, curl_off_t dlnow, 246 curl_off_t ultotal, curl_off_t ulnow)) = 0; 247 virtual CURLcode curl_easy_perform(CURL* curl) = 0; 248 virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info, 249 uint64* value) = 0; 250 virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info, 251 double* value) = 0; 252 virtual void curl_easy_cleanup(CURL* curl) = 0; 253 virtual curl_slist* curl_slist_append(curl_slist* list, const char* str) = 0; 254 virtual void curl_slist_free_all(curl_slist* list) = 0; 255 virtual char* curl_easy_escape(CURL* curl, const char* str, int length) = 0; 256 virtual void curl_free(void* p) = 0; 257 258 virtual const char* curl_easy_strerror(CURLcode errornum) = 0; 259 }; 260 261 } // namespace tensorflow 262 263 #endif // TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_ 264
