Home | History | Annotate | Download | only in socket 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/socket/client_socket_pool_base.h"
      6 
      7 #include "base/compiler_specific.h"
      8 #include "base/format_macros.h"
      9 #include "base/message_loop.h"
     10 #include "base/metrics/stats_counters.h"
     11 #include "base/stl_util-inl.h"
     12 #include "base/string_util.h"
     13 #include "base/time.h"
     14 #include "base/values.h"
     15 #include "net/base/net_log.h"
     16 #include "net/base/net_errors.h"
     17 #include "net/socket/client_socket_handle.h"
     18 
     19 using base::TimeDelta;
     20 
     21 namespace {
     22 
     23 // The timeout value, in seconds, used to clean up idle sockets that can't be
     24 // reused.
     25 //
     26 // Note: It's important to close idle sockets that have received data as soon
     27 // as possible because the received data may cause BSOD on Windows XP under
     28 // some conditions.  See http://crbug.com/4606.
     29 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
     30 
     31 // Indicate whether or not we should establish a new transport layer connection
     32 // after a certain timeout has passed without receiving an ACK.
     33 bool g_connect_backup_jobs_enabled = true;
     34 
     35 }  // namespace
     36 
     37 namespace net {
     38 
     39 ConnectJob::ConnectJob(const std::string& group_name,
     40                        base::TimeDelta timeout_duration,
     41                        Delegate* delegate,
     42                        const BoundNetLog& net_log)
     43     : group_name_(group_name),
     44       timeout_duration_(timeout_duration),
     45       delegate_(delegate),
     46       net_log_(net_log),
     47       idle_(true),
     48       preconnect_state_(NOT_PRECONNECT) {
     49   DCHECK(!group_name.empty());
     50   DCHECK(delegate);
     51   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB, NULL);
     52 }
     53 
     54 ConnectJob::~ConnectJob() {
     55   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB, NULL);
     56 }
     57 
     58 void ConnectJob::Initialize(bool is_preconnect) {
     59   if (is_preconnect)
     60     preconnect_state_ = UNUSED_PRECONNECT;
     61   else
     62     preconnect_state_ = NOT_PRECONNECT;
     63 }
     64 
     65 int ConnectJob::Connect() {
     66   if (timeout_duration_ != base::TimeDelta())
     67     timer_.Start(timeout_duration_, this, &ConnectJob::OnTimeout);
     68 
     69   idle_ = false;
     70 
     71   LogConnectStart();
     72 
     73   int rv = ConnectInternal();
     74 
     75   if (rv != ERR_IO_PENDING) {
     76     LogConnectCompletion(rv);
     77     delegate_ = NULL;
     78   }
     79 
     80   return rv;
     81 }
     82 
     83 void ConnectJob::UseForNormalRequest() {
     84   DCHECK_EQ(UNUSED_PRECONNECT, preconnect_state_);
     85   preconnect_state_ = USED_PRECONNECT;
     86 }
     87 
     88 void ConnectJob::set_socket(ClientSocket* socket) {
     89   if (socket) {
     90     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET, make_scoped_refptr(
     91           new NetLogSourceParameter("source_dependency",
     92                                     socket->NetLog().source())));
     93   }
     94   socket_.reset(socket);
     95 }
     96 
     97 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
     98   // The delegate will delete |this|.
     99   Delegate *delegate = delegate_;
    100   delegate_ = NULL;
    101 
    102   LogConnectCompletion(rv);
    103   delegate->OnConnectJobComplete(rv, this);
    104 }
    105 
    106 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
    107   timer_.Stop();
    108   timer_.Start(remaining_time, this, &ConnectJob::OnTimeout);
    109 }
    110 
    111 void ConnectJob::LogConnectStart() {
    112   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT,
    113       make_scoped_refptr(new NetLogStringParameter("group_name", group_name_)));
    114 }
    115 
    116 void ConnectJob::LogConnectCompletion(int net_error) {
    117   net_log().EndEventWithNetErrorCode(
    118       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
    119 }
    120 
    121 void ConnectJob::OnTimeout() {
    122   // Make sure the socket is NULL before calling into |delegate|.
    123   set_socket(NULL);
    124 
    125   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT, NULL);
    126 
    127   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
    128 }
    129 
    130 namespace internal {
    131 
    132 ClientSocketPoolBaseHelper::Request::Request(
    133     ClientSocketHandle* handle,
    134     CompletionCallback* callback,
    135     RequestPriority priority,
    136     bool ignore_limits,
    137     Flags flags,
    138     const BoundNetLog& net_log
    139 #ifdef ANDROID
    140     , bool valid_uid, uid_t calling_uid
    141 #endif
    142     )
    143     : handle_(handle),
    144       callback_(callback),
    145       priority_(priority),
    146       ignore_limits_(ignore_limits),
    147       flags_(flags),
    148       net_log_(net_log)
    149 #ifdef ANDROID
    150       , valid_uid_(valid_uid), calling_uid_(calling_uid)
    151 #endif
    152       {}
    153 
    154 ClientSocketPoolBaseHelper::Request::~Request() {}
    155 
    156 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
    157     int max_sockets,
    158     int max_sockets_per_group,
    159     base::TimeDelta unused_idle_socket_timeout,
    160     base::TimeDelta used_idle_socket_timeout,
    161     ConnectJobFactory* connect_job_factory)
    162     : idle_socket_count_(0),
    163       connecting_socket_count_(0),
    164       handed_out_socket_count_(0),
    165       max_sockets_(max_sockets),
    166       max_sockets_per_group_(max_sockets_per_group),
    167       unused_idle_socket_timeout_(unused_idle_socket_timeout),
    168       used_idle_socket_timeout_(used_idle_socket_timeout),
    169       connect_job_factory_(connect_job_factory),
    170       connect_backup_jobs_enabled_(false),
    171       pool_generation_number_(0),
    172       method_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)) {
    173   DCHECK_LE(0, max_sockets_per_group);
    174   DCHECK_LE(max_sockets_per_group, max_sockets);
    175 
    176   NetworkChangeNotifier::AddIPAddressObserver(this);
    177 }
    178 
    179 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
    180   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
    181   // remaining active sockets or pending requests.  They should have all been
    182   // cleaned up prior to |this| being destroyed.
    183   Flush();
    184   DCHECK(group_map_.empty());
    185   DCHECK(pending_callback_map_.empty());
    186   DCHECK_EQ(0, connecting_socket_count_);
    187 
    188   NetworkChangeNotifier::RemoveIPAddressObserver(this);
    189 }
    190 
    191 // InsertRequestIntoQueue inserts the request into the queue based on
    192 // priority.  Highest priorities are closest to the front.  Older requests are
    193 // prioritized over requests of equal priority.
    194 //
    195 // static
    196 void ClientSocketPoolBaseHelper::InsertRequestIntoQueue(
    197     const Request* r, RequestQueue* pending_requests) {
    198   RequestQueue::iterator it = pending_requests->begin();
    199   while (it != pending_requests->end() && r->priority() >= (*it)->priority())
    200     ++it;
    201   pending_requests->insert(it, r);
    202 }
    203 
    204 // static
    205 const ClientSocketPoolBaseHelper::Request*
    206 ClientSocketPoolBaseHelper::RemoveRequestFromQueue(
    207     const RequestQueue::iterator& it, Group* group) {
    208   const Request* req = *it;
    209   group->mutable_pending_requests()->erase(it);
    210   // If there are no more requests, we kill the backup timer.
    211   if (group->pending_requests().empty())
    212     group->CleanupBackupJob();
    213   return req;
    214 }
    215 
    216 int ClientSocketPoolBaseHelper::RequestSocket(
    217     const std::string& group_name,
    218     const Request* request) {
    219   CHECK(request->callback());
    220   CHECK(request->handle());
    221 
    222   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL, NULL);
    223   Group* group = GetOrCreateGroup(group_name);
    224 
    225   int rv = RequestSocketInternal(group_name, request);
    226   if (rv != ERR_IO_PENDING) {
    227     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
    228     CHECK(!request->handle()->is_initialized());
    229     delete request;
    230   } else {
    231     InsertRequestIntoQueue(request, group->mutable_pending_requests());
    232   }
    233   return rv;
    234 }
    235 
    236 void ClientSocketPoolBaseHelper::RequestSockets(
    237     const std::string& group_name,
    238     const Request& request,
    239     int num_sockets) {
    240   DCHECK(!request.callback());
    241   DCHECK(!request.handle());
    242 
    243   if (num_sockets > max_sockets_per_group_) {
    244     num_sockets = max_sockets_per_group_;
    245   }
    246 
    247   request.net_log().BeginEvent(
    248       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
    249       make_scoped_refptr(new NetLogIntegerParameter(
    250           "num_sockets", num_sockets)));
    251 
    252   Group* group = GetOrCreateGroup(group_name);
    253 
    254   // RequestSocketsInternal() may delete the group.
    255   bool deleted_group = false;
    256 
    257   int rv = OK;
    258   for (int num_iterations_left = num_sockets;
    259        group->NumActiveSocketSlots() < num_sockets &&
    260        num_iterations_left > 0 ; num_iterations_left--) {
    261     rv = RequestSocketInternal(group_name, &request);
    262     if (rv < 0 && rv != ERR_IO_PENDING) {
    263       // We're encountering a synchronous error.  Give up.
    264       if (!ContainsKey(group_map_, group_name))
    265         deleted_group = true;
    266       break;
    267     }
    268     if (!ContainsKey(group_map_, group_name)) {
    269       // Unexpected.  The group should only be getting deleted on synchronous
    270       // error.
    271       NOTREACHED();
    272       deleted_group = true;
    273       break;
    274     }
    275   }
    276 
    277   if (!deleted_group && group->IsEmpty())
    278     RemoveGroup(group_name);
    279 
    280   if (rv == ERR_IO_PENDING)
    281     rv = OK;
    282   request.net_log().EndEventWithNetErrorCode(
    283       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
    284 }
    285 
    286 int ClientSocketPoolBaseHelper::RequestSocketInternal(
    287     const std::string& group_name,
    288     const Request* request) {
    289   DCHECK_GE(request->priority(), 0);
    290   ClientSocketHandle* const handle = request->handle();
    291   const bool preconnecting = !handle;
    292   Group* group = GetOrCreateGroup(group_name);
    293 
    294   if (!(request->flags() & NO_IDLE_SOCKETS)) {
    295     // Try to reuse a socket.
    296     if (AssignIdleSocketToGroup(request, group))
    297       return OK;
    298   }
    299 
    300   if (!preconnecting && group->TryToUsePreconnectConnectJob())
    301     return ERR_IO_PENDING;
    302 
    303   // Can we make another active socket now?
    304   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
    305       !request->ignore_limits()) {
    306     request->net_log().AddEvent(
    307         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP, NULL);
    308     return ERR_IO_PENDING;
    309   }
    310 
    311   if (ReachedMaxSocketsLimit() && !request->ignore_limits()) {
    312     if (idle_socket_count() > 0) {
    313       bool closed = CloseOneIdleSocketExceptInGroup(group);
    314       if (preconnecting && !closed)
    315         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
    316     } else {
    317       // We could check if we really have a stalled group here, but it requires
    318       // a scan of all groups, so just flip a flag here, and do the check later.
    319       request->net_log().AddEvent(
    320           NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS, NULL);
    321       return ERR_IO_PENDING;
    322     }
    323   }
    324 
    325   // We couldn't find a socket to reuse, so allocate and connect a new one.
    326   scoped_ptr<ConnectJob> connect_job(
    327       connect_job_factory_->NewConnectJob(group_name, *request, this));
    328 
    329   connect_job->Initialize(preconnecting);
    330   int rv = connect_job->Connect();
    331   if (rv == OK) {
    332     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
    333     if (!preconnecting) {
    334       HandOutSocket(connect_job->ReleaseSocket(), false /* not reused */,
    335                     handle, base::TimeDelta(), group, request->net_log());
    336     } else {
    337       AddIdleSocket(connect_job->ReleaseSocket(), group);
    338     }
    339   } else if (rv == ERR_IO_PENDING) {
    340     // If we don't have any sockets in this group, set a timer for potentially
    341     // creating a new one.  If the SYN is lost, this backup socket may complete
    342     // before the slow socket, improving end user latency.
    343     if (connect_backup_jobs_enabled_ &&
    344         group->IsEmpty() && !group->HasBackupJob() &&
    345         handle) {
    346       group->StartBackupSocketTimer(group_name, this);
    347     }
    348 
    349     connecting_socket_count_++;
    350 
    351     group->AddJob(connect_job.release());
    352   } else {
    353     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
    354     ClientSocket* error_socket = NULL;
    355     if (!preconnecting) {
    356       DCHECK(handle);
    357       connect_job->GetAdditionalErrorState(handle);
    358       error_socket = connect_job->ReleaseSocket();
    359     }
    360     if (error_socket) {
    361       HandOutSocket(error_socket, false /* not reused */, handle,
    362                     base::TimeDelta(), group, request->net_log());
    363     } else if (group->IsEmpty()) {
    364       RemoveGroup(group_name);
    365     }
    366   }
    367 
    368   return rv;
    369 }
    370 
    371 bool ClientSocketPoolBaseHelper::AssignIdleSocketToGroup(
    372     const Request* request, Group* group) {
    373   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
    374   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
    375 
    376   // Iterate through the idle sockets forwards (oldest to newest)
    377   //   * Delete any disconnected ones.
    378   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
    379   //   the |idle_socket_it| will be set to the newest used idle socket.
    380   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
    381        it != idle_sockets->end();) {
    382     if (!it->socket->IsConnectedAndIdle()) {
    383       DecrementIdleCount();
    384       delete it->socket;
    385       it = idle_sockets->erase(it);
    386       continue;
    387     }
    388 
    389     if (it->socket->WasEverUsed()) {
    390       // We found one we can reuse!
    391       idle_socket_it = it;
    392     }
    393 
    394     ++it;
    395   }
    396 
    397   // If we haven't found an idle socket, that means there are no used idle
    398   // sockets.  Pick the oldest (first) idle socket (FIFO).
    399 
    400   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
    401     idle_socket_it = idle_sockets->begin();
    402 
    403   if (idle_socket_it != idle_sockets->end()) {
    404     DecrementIdleCount();
    405     base::TimeDelta idle_time =
    406         base::TimeTicks::Now() - idle_socket_it->start_time;
    407     IdleSocket idle_socket = *idle_socket_it;
    408     idle_sockets->erase(idle_socket_it);
    409     HandOutSocket(
    410         idle_socket.socket,
    411         idle_socket.socket->WasEverUsed(),
    412         request->handle(),
    413         idle_time,
    414         group,
    415         request->net_log());
    416     return true;
    417   }
    418 
    419   return false;
    420 }
    421 
    422 // static
    423 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
    424     const NetLog::Source& connect_job_source, const Request* request) {
    425   request->net_log().AddEvent(
    426       NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
    427       make_scoped_refptr(new NetLogSourceParameter(
    428           "source_dependency", connect_job_source)));
    429 }
    430 
    431 void ClientSocketPoolBaseHelper::CancelRequest(
    432     const std::string& group_name, ClientSocketHandle* handle) {
    433   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
    434   if (callback_it != pending_callback_map_.end()) {
    435     int result = callback_it->second.result;
    436     pending_callback_map_.erase(callback_it);
    437     ClientSocket* socket = handle->release_socket();
    438     if (socket) {
    439       if (result != OK)
    440         socket->Disconnect();
    441       ReleaseSocket(handle->group_name(), socket, handle->id());
    442     }
    443     return;
    444   }
    445 
    446   CHECK(ContainsKey(group_map_, group_name));
    447 
    448   Group* group = GetOrCreateGroup(group_name);
    449 
    450   // Search pending_requests for matching handle.
    451   RequestQueue::iterator it = group->mutable_pending_requests()->begin();
    452   for (; it != group->pending_requests().end(); ++it) {
    453     if ((*it)->handle() == handle) {
    454       scoped_ptr<const Request> req(RemoveRequestFromQueue(it, group));
    455       req->net_log().AddEvent(NetLog::TYPE_CANCELLED, NULL);
    456       req->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL, NULL);
    457 
    458       // We let the job run, unless we're at the socket limit.
    459       if (group->jobs().size() && ReachedMaxSocketsLimit()) {
    460         RemoveConnectJob(*group->jobs().begin(), group);
    461         CheckForStalledSocketGroups();
    462       }
    463       break;
    464     }
    465   }
    466 }
    467 
    468 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
    469   return ContainsKey(group_map_, group_name);
    470 }
    471 
    472 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
    473   CleanupIdleSockets(true);
    474   DCHECK_EQ(0, idle_socket_count_);
    475 }
    476 
    477 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
    478     const std::string& group_name) const {
    479   GroupMap::const_iterator i = group_map_.find(group_name);
    480   CHECK(i != group_map_.end());
    481 
    482   return i->second->idle_sockets().size();
    483 }
    484 
    485 LoadState ClientSocketPoolBaseHelper::GetLoadState(
    486     const std::string& group_name,
    487     const ClientSocketHandle* handle) const {
    488   if (ContainsKey(pending_callback_map_, handle))
    489     return LOAD_STATE_CONNECTING;
    490 
    491   if (!ContainsKey(group_map_, group_name)) {
    492     NOTREACHED() << "ClientSocketPool does not contain group: " << group_name
    493                  << " for handle: " << handle;
    494     return LOAD_STATE_IDLE;
    495   }
    496 
    497   // Can't use operator[] since it is non-const.
    498   const Group& group = *group_map_.find(group_name)->second;
    499 
    500   // Search pending_requests for matching handle.
    501   RequestQueue::const_iterator it = group.pending_requests().begin();
    502   for (size_t i = 0; it != group.pending_requests().end(); ++it, ++i) {
    503     if ((*it)->handle() == handle) {
    504       if (i < group.jobs().size()) {
    505         LoadState max_state = LOAD_STATE_IDLE;
    506         for (ConnectJobSet::const_iterator job_it = group.jobs().begin();
    507              job_it != group.jobs().end(); ++job_it) {
    508           max_state = std::max(max_state, (*job_it)->GetLoadState());
    509         }
    510         return max_state;
    511       } else {
    512         // TODO(wtc): Add a state for being on the wait list.
    513         // See http://www.crbug.com/5077.
    514         return LOAD_STATE_IDLE;
    515       }
    516     }
    517   }
    518 
    519   NOTREACHED();
    520   return LOAD_STATE_IDLE;
    521 }
    522 
    523 DictionaryValue* ClientSocketPoolBaseHelper::GetInfoAsValue(
    524     const std::string& name, const std::string& type) const {
    525   DictionaryValue* dict = new DictionaryValue();
    526   dict->SetString("name", name);
    527   dict->SetString("type", type);
    528   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
    529   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
    530   dict->SetInteger("idle_socket_count", idle_socket_count_);
    531   dict->SetInteger("max_socket_count", max_sockets_);
    532   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
    533   dict->SetInteger("pool_generation_number", pool_generation_number_);
    534 
    535   if (group_map_.empty())
    536     return dict;
    537 
    538   DictionaryValue* all_groups_dict = new DictionaryValue();
    539   for (GroupMap::const_iterator it = group_map_.begin();
    540        it != group_map_.end(); it++) {
    541     const Group* group = it->second;
    542     DictionaryValue* group_dict = new DictionaryValue();
    543 
    544     group_dict->SetInteger("pending_request_count",
    545                            group->pending_requests().size());
    546     if (!group->pending_requests().empty()) {
    547       group_dict->SetInteger("top_pending_priority",
    548                              group->TopPendingPriority());
    549     }
    550 
    551     group_dict->SetInteger("active_socket_count", group->active_socket_count());
    552 
    553     ListValue* idle_socket_list = new ListValue();
    554     std::list<IdleSocket>::const_iterator idle_socket;
    555     for (idle_socket = group->idle_sockets().begin();
    556          idle_socket != group->idle_sockets().end();
    557          idle_socket++) {
    558       int source_id = idle_socket->socket->NetLog().source().id;
    559       idle_socket_list->Append(Value::CreateIntegerValue(source_id));
    560     }
    561     group_dict->Set("idle_sockets", idle_socket_list);
    562 
    563     ListValue* connect_jobs_list = new ListValue();
    564     std::set<ConnectJob*>::const_iterator job = group->jobs().begin();
    565     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
    566       int source_id = (*job)->net_log().source().id;
    567       connect_jobs_list->Append(Value::CreateIntegerValue(source_id));
    568     }
    569     group_dict->Set("connect_jobs", connect_jobs_list);
    570 
    571     group_dict->SetBoolean("is_stalled",
    572                            group->IsStalled(max_sockets_per_group_));
    573     group_dict->SetBoolean("has_backup_job", group->HasBackupJob());
    574 
    575     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
    576   }
    577   dict->Set("groups", all_groups_dict);
    578   return dict;
    579 }
    580 
    581 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
    582     base::TimeTicks now,
    583     base::TimeDelta timeout) const {
    584   bool timed_out = (now - start_time) >= timeout;
    585   if (timed_out)
    586     return true;
    587   if (socket->WasEverUsed())
    588     return !socket->IsConnectedAndIdle();
    589   return !socket->IsConnected();
    590 }
    591 
    592 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
    593   if (idle_socket_count_ == 0)
    594     return;
    595 
    596   // Current time value. Retrieving it once at the function start rather than
    597   // inside the inner loop, since it shouldn't change by any meaningful amount.
    598   base::TimeTicks now = base::TimeTicks::Now();
    599 
    600   GroupMap::iterator i = group_map_.begin();
    601   while (i != group_map_.end()) {
    602     Group* group = i->second;
    603 
    604     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
    605     while (j != group->idle_sockets().end()) {
    606       base::TimeDelta timeout =
    607           j->socket->WasEverUsed() ?
    608           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
    609       if (force || j->ShouldCleanup(now, timeout)) {
    610         delete j->socket;
    611         j = group->mutable_idle_sockets()->erase(j);
    612         DecrementIdleCount();
    613       } else {
    614         ++j;
    615       }
    616     }
    617 
    618     // Delete group if no longer needed.
    619     if (group->IsEmpty()) {
    620       RemoveGroup(i++);
    621     } else {
    622       ++i;
    623     }
    624   }
    625 }
    626 
    627 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
    628     const std::string& group_name) {
    629   GroupMap::iterator it = group_map_.find(group_name);
    630   if (it != group_map_.end())
    631     return it->second;
    632   Group* group = new Group;
    633   group_map_[group_name] = group;
    634   return group;
    635 }
    636 
    637 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
    638   GroupMap::iterator it = group_map_.find(group_name);
    639   CHECK(it != group_map_.end());
    640 
    641   RemoveGroup(it);
    642 }
    643 
    644 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
    645   delete it->second;
    646   group_map_.erase(it);
    647 }
    648 
    649 // static
    650 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
    651   return g_connect_backup_jobs_enabled;
    652 }
    653 
    654 // static
    655 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
    656   bool old_value = g_connect_backup_jobs_enabled;
    657   g_connect_backup_jobs_enabled = enabled;
    658   return old_value;
    659 }
    660 
    661 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
    662   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
    663 }
    664 
    665 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
    666   if (++idle_socket_count_ == 1)
    667     timer_.Start(TimeDelta::FromSeconds(kCleanupInterval), this,
    668                  &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
    669 }
    670 
    671 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
    672   if (--idle_socket_count_ == 0)
    673     timer_.Stop();
    674 }
    675 
    676 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
    677                                                ClientSocket* socket,
    678                                                int id) {
    679   GroupMap::iterator i = group_map_.find(group_name);
    680   CHECK(i != group_map_.end());
    681 
    682   Group* group = i->second;
    683 
    684   CHECK_GT(handed_out_socket_count_, 0);
    685   handed_out_socket_count_--;
    686 
    687   CHECK_GT(group->active_socket_count(), 0);
    688   group->DecrementActiveSocketCount();
    689 
    690   const bool can_reuse = socket->IsConnectedAndIdle() &&
    691       id == pool_generation_number_;
    692   if (can_reuse) {
    693     // Add it to the idle list.
    694     AddIdleSocket(socket, group);
    695     OnAvailableSocketSlot(group_name, group);
    696   } else {
    697     delete socket;
    698   }
    699 
    700   CheckForStalledSocketGroups();
    701 }
    702 
    703 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
    704   // If we have idle sockets, see if we can give one to the top-stalled group.
    705   std::string top_group_name;
    706   Group* top_group = NULL;
    707   if (!FindTopStalledGroup(&top_group, &top_group_name))
    708     return;
    709 
    710   if (ReachedMaxSocketsLimit()) {
    711     if (idle_socket_count() > 0) {
    712       CloseOneIdleSocket();
    713     } else {
    714       // We can't activate more sockets since we're already at our global
    715       // limit.
    716       return;
    717     }
    718   }
    719 
    720   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
    721   //        its limit, may be left with other stalled groups that could be
    722   //        woken.  This isn't optimal, but there is no starvation, so to avoid
    723   //        the looping we leave it at this.
    724   OnAvailableSocketSlot(top_group_name, top_group);
    725 }
    726 
    727 // Search for the highest priority pending request, amongst the groups that
    728 // are not at the |max_sockets_per_group_| limit. Note: for requests with
    729 // the same priority, the winner is based on group hash ordering (and not
    730 // insertion order).
    731 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(Group** group,
    732                                                      std::string* group_name) {
    733   Group* top_group = NULL;
    734   const std::string* top_group_name = NULL;
    735   bool has_stalled_group = false;
    736   for (GroupMap::iterator i = group_map_.begin();
    737        i != group_map_.end(); ++i) {
    738     Group* curr_group = i->second;
    739     const RequestQueue& queue = curr_group->pending_requests();
    740     if (queue.empty())
    741       continue;
    742     if (curr_group->IsStalled(max_sockets_per_group_)) {
    743       has_stalled_group = true;
    744       bool has_higher_priority = !top_group ||
    745           curr_group->TopPendingPriority() < top_group->TopPendingPriority();
    746       if (has_higher_priority) {
    747         top_group = curr_group;
    748         top_group_name = &i->first;
    749       }
    750     }
    751   }
    752 
    753   if (top_group) {
    754     *group = top_group;
    755     *group_name = *top_group_name;
    756   }
    757   return has_stalled_group;
    758 }
    759 
    760 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
    761     int result, ConnectJob* job) {
    762   DCHECK_NE(ERR_IO_PENDING, result);
    763   const std::string group_name = job->group_name();
    764   GroupMap::iterator group_it = group_map_.find(group_name);
    765   CHECK(group_it != group_map_.end());
    766   Group* group = group_it->second;
    767 
    768   scoped_ptr<ClientSocket> socket(job->ReleaseSocket());
    769 
    770   BoundNetLog job_log = job->net_log();
    771 
    772   if (result == OK) {
    773     DCHECK(socket.get());
    774     RemoveConnectJob(job, group);
    775     if (!group->pending_requests().empty()) {
    776       scoped_ptr<const Request> r(RemoveRequestFromQueue(
    777           group->mutable_pending_requests()->begin(), group));
    778       LogBoundConnectJobToRequest(job_log.source(), r.get());
    779       HandOutSocket(
    780           socket.release(), false /* unused socket */, r->handle(),
    781           base::TimeDelta(), group, r->net_log());
    782       r->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL, NULL);
    783       InvokeUserCallbackLater(r->handle(), r->callback(), result);
    784     } else {
    785       AddIdleSocket(socket.release(), group);
    786       OnAvailableSocketSlot(group_name, group);
    787       CheckForStalledSocketGroups();
    788     }
    789   } else {
    790     // If we got a socket, it must contain error information so pass that
    791     // up so that the caller can retrieve it.
    792     bool handed_out_socket = false;
    793     if (!group->pending_requests().empty()) {
    794       scoped_ptr<const Request> r(RemoveRequestFromQueue(
    795           group->mutable_pending_requests()->begin(), group));
    796       LogBoundConnectJobToRequest(job_log.source(), r.get());
    797       job->GetAdditionalErrorState(r->handle());
    798       RemoveConnectJob(job, group);
    799       if (socket.get()) {
    800         handed_out_socket = true;
    801         HandOutSocket(socket.release(), false /* unused socket */, r->handle(),
    802                       base::TimeDelta(), group, r->net_log());
    803       }
    804       r->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL,
    805                                             result);
    806       InvokeUserCallbackLater(r->handle(), r->callback(), result);
    807     } else {
    808       RemoveConnectJob(job, group);
    809     }
    810     if (!handed_out_socket) {
    811       OnAvailableSocketSlot(group_name, group);
    812       CheckForStalledSocketGroups();
    813     }
    814   }
    815 }
    816 
    817 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
    818   Flush();
    819 }
    820 
    821 void ClientSocketPoolBaseHelper::Flush() {
    822   pool_generation_number_++;
    823   CancelAllConnectJobs();
    824   CloseIdleSockets();
    825   AbortAllRequests();
    826 }
    827 
    828 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
    829                                                   Group* group) {
    830   CHECK_GT(connecting_socket_count_, 0);
    831   connecting_socket_count_--;
    832 
    833   DCHECK(group);
    834   DCHECK(ContainsKey(group->jobs(), job));
    835   group->RemoveJob(job);
    836 
    837   // If we've got no more jobs for this group, then we no longer need a
    838   // backup job either.
    839   if (group->jobs().empty())
    840     group->CleanupBackupJob();
    841 
    842   DCHECK(job);
    843   delete job;
    844 }
    845 
    846 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
    847     const std::string& group_name, Group* group) {
    848   DCHECK(ContainsKey(group_map_, group_name));
    849   if (group->IsEmpty())
    850     RemoveGroup(group_name);
    851   else if (!group->pending_requests().empty())
    852     ProcessPendingRequest(group_name, group);
    853 }
    854 
    855 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
    856     const std::string& group_name, Group* group) {
    857   int rv = RequestSocketInternal(group_name,
    858                                  *group->pending_requests().begin());
    859   if (rv != ERR_IO_PENDING) {
    860     scoped_ptr<const Request> request(RemoveRequestFromQueue(
    861           group->mutable_pending_requests()->begin(), group));
    862     if (group->IsEmpty())
    863       RemoveGroup(group_name);
    864 
    865     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
    866     InvokeUserCallbackLater(
    867         request->handle(), request->callback(), rv);
    868   }
    869 }
    870 
    871 void ClientSocketPoolBaseHelper::HandOutSocket(
    872     ClientSocket* socket,
    873     bool reused,
    874     ClientSocketHandle* handle,
    875     base::TimeDelta idle_time,
    876     Group* group,
    877     const BoundNetLog& net_log) {
    878   DCHECK(socket);
    879   handle->set_socket(socket);
    880   handle->set_is_reused(reused);
    881   handle->set_idle_time(idle_time);
    882   handle->set_pool_id(pool_generation_number_);
    883 
    884   if (reused) {
    885     net_log.AddEvent(
    886         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
    887         make_scoped_refptr(new NetLogIntegerParameter(
    888             "idle_ms", static_cast<int>(idle_time.InMilliseconds()))));
    889   }
    890 
    891   net_log.AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
    892                    make_scoped_refptr(new NetLogSourceParameter(
    893                        "source_dependency", socket->NetLog().source())));
    894 
    895   handed_out_socket_count_++;
    896   group->IncrementActiveSocketCount();
    897 }
    898 
    899 void ClientSocketPoolBaseHelper::AddIdleSocket(
    900     ClientSocket* socket, Group* group) {
    901   DCHECK(socket);
    902   IdleSocket idle_socket;
    903   idle_socket.socket = socket;
    904   idle_socket.start_time = base::TimeTicks::Now();
    905 
    906   group->mutable_idle_sockets()->push_back(idle_socket);
    907   IncrementIdleCount();
    908 }
    909 
    910 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
    911   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
    912     Group* group = i->second;
    913     connecting_socket_count_ -= group->jobs().size();
    914     group->RemoveAllJobs();
    915 
    916     // Delete group if no longer needed.
    917     if (group->IsEmpty()) {
    918       // RemoveGroup() will call .erase() which will invalidate the iterator,
    919       // but i will already have been incremented to a valid iterator before
    920       // RemoveGroup() is called.
    921       RemoveGroup(i++);
    922     } else {
    923       ++i;
    924     }
    925   }
    926   DCHECK_EQ(0, connecting_socket_count_);
    927 }
    928 
    929 void ClientSocketPoolBaseHelper::AbortAllRequests() {
    930   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
    931     Group* group = i->second;
    932 
    933     RequestQueue pending_requests;
    934     pending_requests.swap(*group->mutable_pending_requests());
    935     for (RequestQueue::iterator it2 = pending_requests.begin();
    936          it2 != pending_requests.end(); ++it2) {
    937       scoped_ptr<const Request> request(*it2);
    938       InvokeUserCallbackLater(
    939           request->handle(), request->callback(), ERR_ABORTED);
    940     }
    941 
    942     // Delete group if no longer needed.
    943     if (group->IsEmpty()) {
    944       // RemoveGroup() will call .erase() which will invalidate the iterator,
    945       // but i will already have been incremented to a valid iterator before
    946       // RemoveGroup() is called.
    947       RemoveGroup(i++);
    948     } else {
    949       ++i;
    950     }
    951   }
    952 }
    953 
    954 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
    955   // Each connecting socket will eventually connect and be handed out.
    956   int total = handed_out_socket_count_ + connecting_socket_count_ +
    957       idle_socket_count();
    958   // There can be more sockets than the limit since some requests can ignore
    959   // the limit
    960   if (total < max_sockets_)
    961     return false;
    962   return true;
    963 }
    964 
    965 void ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
    966   CloseOneIdleSocketExceptInGroup(NULL);
    967 }
    968 
    969 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
    970     const Group* exception_group) {
    971   CHECK_GT(idle_socket_count(), 0);
    972 
    973   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
    974     Group* group = i->second;
    975     if (exception_group == group)
    976       continue;
    977     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
    978 
    979     if (!idle_sockets->empty()) {
    980       delete idle_sockets->front().socket;
    981       idle_sockets->pop_front();
    982       DecrementIdleCount();
    983       if (group->IsEmpty())
    984         RemoveGroup(i);
    985 
    986       return true;
    987     }
    988   }
    989 
    990   if (!exception_group)
    991     LOG(DFATAL) << "No idle socket found to close!.";
    992 
    993   return false;
    994 }
    995 
    996 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
    997     ClientSocketHandle* handle, CompletionCallback* callback, int rv) {
    998   CHECK(!ContainsKey(pending_callback_map_, handle));
    999   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
   1000   MessageLoop::current()->PostTask(
   1001       FROM_HERE,
   1002       method_factory_.NewRunnableMethod(
   1003           &ClientSocketPoolBaseHelper::InvokeUserCallback,
   1004           handle));
   1005 }
   1006 
   1007 void ClientSocketPoolBaseHelper::InvokeUserCallback(
   1008     ClientSocketHandle* handle) {
   1009   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
   1010 
   1011   // Exit if the request has already been cancelled.
   1012   if (it == pending_callback_map_.end())
   1013     return;
   1014 
   1015   CHECK(!handle->is_initialized());
   1016   CompletionCallback* callback = it->second.callback;
   1017   int result = it->second.result;
   1018   pending_callback_map_.erase(it);
   1019   callback->Run(result);
   1020 }
   1021 
   1022 ClientSocketPoolBaseHelper::Group::Group()
   1023     : active_socket_count_(0),
   1024       ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {}
   1025 
   1026 ClientSocketPoolBaseHelper::Group::~Group() {
   1027   CleanupBackupJob();
   1028 }
   1029 
   1030 void ClientSocketPoolBaseHelper::Group::StartBackupSocketTimer(
   1031     const std::string& group_name,
   1032     ClientSocketPoolBaseHelper* pool) {
   1033   // Only allow one timer pending to create a backup socket.
   1034   if (!method_factory_.empty())
   1035     return;
   1036 
   1037   MessageLoop::current()->PostDelayedTask(
   1038       FROM_HERE,
   1039       method_factory_.NewRunnableMethod(
   1040           &Group::OnBackupSocketTimerFired, group_name, pool),
   1041       pool->ConnectRetryIntervalMs());
   1042 }
   1043 
   1044 bool ClientSocketPoolBaseHelper::Group::TryToUsePreconnectConnectJob() {
   1045   for (std::set<ConnectJob*>::iterator it = jobs_.begin();
   1046        it != jobs_.end(); ++it) {
   1047     ConnectJob* job = *it;
   1048     if (job->is_unused_preconnect()) {
   1049       job->UseForNormalRequest();
   1050       return true;
   1051     }
   1052   }
   1053   return false;
   1054 }
   1055 
   1056 void ClientSocketPoolBaseHelper::Group::OnBackupSocketTimerFired(
   1057     std::string group_name,
   1058     ClientSocketPoolBaseHelper* pool) {
   1059   // If there are no more jobs pending, there is no work to do.
   1060   // If we've done our cleanups correctly, this should not happen.
   1061   if (jobs_.empty()) {
   1062     NOTREACHED();
   1063     return;
   1064   }
   1065 
   1066   // If our backup job is waiting on DNS, or if we can't create any sockets
   1067   // right now due to limits, just reset the timer.
   1068   if (pool->ReachedMaxSocketsLimit() ||
   1069       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
   1070       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
   1071     StartBackupSocketTimer(group_name, pool);
   1072     return;
   1073   }
   1074 
   1075   if (pending_requests_.empty()) {
   1076     LOG(DFATAL) << "No pending request for backup job.";
   1077     return;
   1078   }
   1079 
   1080   ConnectJob* backup_job = pool->connect_job_factory_->NewConnectJob(
   1081       group_name, **pending_requests_.begin(), pool);
   1082   backup_job->net_log().AddEvent(NetLog::TYPE_SOCKET_BACKUP_CREATED, NULL);
   1083   SIMPLE_STATS_COUNTER("socket.backup_created");
   1084   int rv = backup_job->Connect();
   1085   pool->connecting_socket_count_++;
   1086   AddJob(backup_job);
   1087   if (rv != ERR_IO_PENDING)
   1088     pool->OnConnectJobComplete(rv, backup_job);
   1089 }
   1090 
   1091 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
   1092   // Delete active jobs.
   1093   STLDeleteElements(&jobs_);
   1094 
   1095   // Cancel pending backup job.
   1096   method_factory_.RevokeAll();
   1097 }
   1098 
   1099 }  // namespace internal
   1100 
   1101 }  // namespace net
   1102