Add a PerHostRequestQueue object.

Adds a std::map for hostname (or urls) --> PerHostRequestQueue objects. The latter keeps track of the number of added curl easy requests and decides if a new request should be throttled or not, as well as provides the queue to queue throttled requests. At the moment CurlConcurrentConnectionsPerHost is set to 16, because things really don't work without LL supporting connection reuse if we limit it to 2. CurlConcurrentConnectionsPerHost is also set to non-persistent so that we can easily change it in the future (once we decide on it's final value it can be set to persistent).
2012-11-07 15:41:50 +01:00
parent cb52e82a60
commit 68cbf31c8b
11 changed files with 414 additions and 54 deletions
--- a/indra/llmessage/CMakeLists.txt
+++ b/indra/llmessage/CMakeLists.txt
@@ -24,6 +24,7 @@ include_directories(
 set(llmessage_SOURCE_FILES
    aicurl.cpp
    aicurleasyrequeststatemachine.cpp
+    aicurlperhost.cpp
    aicurlthread.cpp
    aihttpheaders.cpp
    aihttptimeoutpolicy.cpp
@@ -113,6 +114,7 @@ set(llmessage_HEADER_FILES
    aicurl.h
    aicurleasyrequeststatemachine.h
    aicurlprivate.h
+    aicurlperhost.h
    aicurlthread.h
    aihttpheaders.h
    aihttptimeoutpolicy.h
--- a/indra/llmessage/aicurl.cpp
+++ b/indra/llmessage/aicurl.cpp
@@ -58,6 +58,7 @@
 #include "aihttpheaders.h"
 #include "aihttptimeoutpolicy.h"
 #include "aicurleasyrequeststatemachine.h"
+#include "aicurlperhost.h"

 //==================================================================================
 // Debug Settings
@@ -880,6 +881,10 @@ CurlEasyRequest::~CurlEasyRequest()
  // be available anymore.
  send_handle_events_to(NULL);
  revokeCallbacks();
+  if (mPerHostPtr)
+  {
+	 PerHostRequestQueue::release(mPerHostPtr);
+  }
  // This wasn't freed yet if the request never finished.
  curl_slist_free_all(mHeaders);
 }
@@ -1078,6 +1083,7 @@ void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolic
 #endif
  setopt(CURLOPT_HTTPHEADER, mHeaders);
  setoptString(CURLOPT_URL, url);
+  llassert(!mPerHostPtr);
  mLowercaseHostname = extract_canonical_hostname(url);
  mTimeoutPolicy = &policy;
  state_machine->setTotalDelayTimeout(policy.getTotalDelay());
@@ -1195,6 +1201,24 @@ void CurlEasyRequest::print_diagnostics(CURLcode code)
  }
 }

+PerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void)
+{
+  if (!mPerHostPtr)
+  {
+	// mPerHostPtr is really just a speed-up cache.
+	// The reason we can cache it is because mLowercaseHostname is only set
+	// in finalizeRequest which may only be called once: it never changes.
+	mPerHostPtr = PerHostRequestQueue::instance(mLowercaseHostname);
+  }
+  return mPerHostPtr;
+}
+
+bool CurlEasyRequest::removeFromPerHostQueue(AICurlEasyRequest const& easy_request) const
+{
+  // Note that easy_request (must) represent(s) this object; it's just passed for convenience.
+  return mPerHostPtr && PerHostRequestQueue_wat(*mPerHostPtr)->cancel(easy_request);
+}
+
 //-----------------------------------------------------------------------------
 // BufferedCurlEasyRequest

--- a/indra/llmessage/aicurl.h
+++ b/indra/llmessage/aicurl.h
@@ -148,7 +148,8 @@ struct Stats {
 // Global functions.

 // Called to handle changes in Debug Settings.
-bool handleCurlConcurrentConnections(LLSD const& newvalue);
+bool handleCurlMaxTotalConcurrentConnections(LLSD const& newvalue);
+bool handleCurlConcurrentConnectionsPerHost(LLSD const& newvalue);
 bool handleNoVerifySSLCert(LLSD const& newvalue);

 // Called once at start of application (from newview/llappviewer.cpp by main thread (before threads are created)),
@@ -156,7 +157,7 @@ bool handleNoVerifySSLCert(LLSD const& newvalue);
 void initCurl(void);

 // Called once at start of application (from LLAppViewer::initThreads), starts AICurlThread.
-void startCurlThread(U32 CurlConcurrentConnections, bool NoVerifySSLCert);
+void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerHost, bool NoVerifySSLCert);

 // Called once at end of application (from newview/llappviewer.cpp by main thread),
 // with purpose to stop curl threads, free curl resources and deinitialize curl.
--- a/indra/llmessage/aicurlperhost.cpp
+++ b/indra/llmessage/aicurlperhost.cpp
@@ -0,0 +1,171 @@
+/**
+ * @file aiperhost.cpp
+ * @brief Implementation of PerHostRequestQueue
+ *
+ * Copyright (c) 2012, Aleric Inglewood.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution.
+ *
+ * CHANGELOG
+ *   and additional copyright holders.
+ *
+ *   04/11/2012
+ *   Initial version, written by Aleric Inglewood @ SL
+ */
+
+#include "sys.h"
+#include "aicurlperhost.h"
+#include "aicurlthread.h"
+
+#undef AICurlPrivate
+
+namespace AICurlPrivate {
+
+PerHostRequestQueue::threadsafe_instance_map_type PerHostRequestQueue::sInstanceMap;
+U32 curl_concurrent_connections_per_host;
+
+//static
+PerHostRequestQueuePtr PerHostRequestQueue::instance(std::string const& hostname)
+{
+  llassert(!hostname.empty());
+  instance_map_wat instance_map_w(sInstanceMap);
+  PerHostRequestQueue::iterator iter = instance_map_w->find(hostname);
+  if (iter == instance_map_w->end())
+  {
+	iter = instance_map_w->insert(instance_map_type::value_type(hostname, new RefCountedThreadSafePerHostRequestQueue)).first;
+  }
+  // Note: the creation of PerHostRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()).
+  return iter->second;
+}
+
+//static
+void PerHostRequestQueue::release(PerHostRequestQueuePtr& instance)
+{
+  if (instance->lastone())
+  {
+	instance_map_wat instance_map_w(sInstanceMap);
+	// It is possible that 'lastone' is not up to date anymore.
+	// Therefore, recheck the condition now that we have locked sInstanceMap.
+	if (!instance->lastone())
+	{
+	  // Some other thread added this host in the meantime.
+	  return;
+	}
+	// The reference in the map is the last one; that means there can't be any curl easy requests queued for this host.
+	llassert(PerHostRequestQueue_wat(*instance)->mQueuedRequests.empty());
+	// Find the host and erase it from the map.
+	iterator const end = instance_map_w->end();
+	for(iterator iter = instance_map_w->begin(); iter != end; ++iter)
+	{
+	  if (instance == iter->second)
+	  {
+		instance_map_w->erase(iter);
+		instance.reset();
+		return;
+	  }
+	}
+	// We should always find the host.
+	llassert(false);
+  }
+  instance.reset();
+}
+
+bool PerHostRequestQueue::throttled() const
+{
+  llassert(mAdded <= curl_concurrent_connections_per_host);
+  return mAdded == curl_concurrent_connections_per_host;
+}
+
+void PerHostRequestQueue::added_to_multi_handle(void)
+{
+  llassert(mAdded < curl_concurrent_connections_per_host);
+  ++mAdded;
+}
+
+void PerHostRequestQueue::removed_from_multi_handle(void)
+{
+  --mAdded;
+  llassert(mAdded >= 0);
+}
+
+void PerHostRequestQueue::queue(AICurlEasyRequest const& easy_request)
+{
+  mQueuedRequests.push_back(easy_request);
+}
+
+bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request)
+{
+  std::deque<AICurlEasyRequest>::iterator const end = mQueuedRequests.end();
+  std::deque<AICurlEasyRequest>::iterator cur = std::find(mQueuedRequests.begin(), end, easy_request);
+
+  if (cur == end)
+	return false;		// Not found.
+
+  // We can't use erase because that uses assignment to move elements, which is
+  // private because it isn't thread-safe for AICurlEasyRequest. Therefore, move
+  // the element that we found to the back with swap (could just swap with the
+  // end immediately, but I don't want to break the order in which requests where
+  // added). Swap is also not thread-safe, but OK here because it only touches the
+  // AICurlEasyRequest objects in the deque, and the deque is protected by the
+  // lock on the PerHostRequestQueue object.
+  std::deque<AICurlEasyRequest>::iterator prev = cur;
+  while (++cur != end)
+  {
+	prev->swap(*cur);				// This is safe,
+	prev = cur;
+  }
+  mQueuedRequests.pop_back();		// if this is safe.
+  return true;
+}
+
+void PerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle)
+{
+  if (!mQueuedRequests.empty())
+  {
+	multi_handle->add_easy_request(mQueuedRequests.front());
+	mQueuedRequests.pop_front();
+  }
+}
+
+//static
+void PerHostRequestQueue::purge(void)
+{
+  instance_map_wat instance_map_w(sInstanceMap);
+  for (iterator host = instance_map_w->begin(); host != instance_map_w->end(); ++host)
+  {
+	Dout(dc::curl, "Purging queue of host \"" << host->first << "\".");
+	PerHostRequestQueue_wat(*host->second)->mQueuedRequests.clear();
+  }
+}
+
+// Friend functions of RefCountedThreadSafePerHostRequestQueue
+
+void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* per_host)
+{
+  per_host->mReferenceCount++;
+}
+
+void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* per_host)
+{
+  if (--per_host->mReferenceCount == 0)
+  {
+    delete per_host;
+  }
+}
+
+} // namespace AICurlPrivate
--- a/indra/llmessage/aicurlperhost.h
+++ b/indra/llmessage/aicurlperhost.h
@@ -0,0 +1,131 @@
+/**
+ * @file aicurlperhost.h
+ * @brief Definition of class PerHostRequestQueue
+ *
+ * Copyright (c) 2012, Aleric Inglewood.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution.
+ *
+ * CHANGELOG
+ *   and additional copyright holders.
+ *
+ *   04/11/2012
+ *   Initial version, written by Aleric Inglewood @ SL
+ */
+
+#ifndef AICURLPERHOST_H
+#define AICURLPERHOST_H
+
+#include "llerror.h"		// llassert
+#include <string>
+#include <deque>
+#include <map>
+#include <boost/intrusive_ptr.hpp>
+#include "aithreadsafe.h"
+
+class AICurlEasyRequest;
+
+namespace AICurlPrivate {
+namespace curlthread { class MultiHandle; }
+
+class PerHostRequestQueue;
+class RefCountedThreadSafePerHostRequestQueue;
+
+// PerHostRequestQueue objects are created by the curl thread and destructed by the main thread.
+// We need locking.
+typedef AIThreadSafeSimpleDC<PerHostRequestQueue> threadsafe_PerHostRequestQueue;
+typedef AIAccessConst<PerHostRequestQueue> PerHostRequestQueue_crat;
+typedef AIAccess<PerHostRequestQueue> PerHostRequestQueue_rat;
+typedef AIAccess<PerHostRequestQueue> PerHostRequestQueue_wat;
+
+// We can't put threadsafe_PerHostRequestQueue in a std::map because you can't copy a mutex.
+// Therefore, use an intrusive pointer for the threadsafe type.
+typedef boost::intrusive_ptr<RefCountedThreadSafePerHostRequestQueue> PerHostRequestQueuePtr;
+
+//-----------------------------------------------------------------------------
+// PerHostRequestQueue
+
+// This class provides a static interface to create and maintain instances
+// of PerHostRequestQueue objects, so that at any moment there is at most
+// one instance per hostname. Those instances then are used to queue curl
+// requests when the maximum number of connections for that host already
+// have been reached.
+class PerHostRequestQueue {
+  private:
+	typedef std::map<std::string, PerHostRequestQueuePtr> instance_map_type;
+	typedef AIThreadSafeSimpleDC<instance_map_type> threadsafe_instance_map_type;
+	typedef AIAccess<instance_map_type> instance_map_rat;
+	typedef AIAccess<instance_map_type> instance_map_wat;
+
+	static threadsafe_instance_map_type sInstanceMap;				// Map of PerHostRequestQueue instances with the hostname as key.
+
+	friend threadsafe_PerHostRequestQueue;
+	PerHostRequestQueue(void) : mAdded(0) { }
+
+  public:
+	typedef instance_map_type::iterator iterator;
+	typedef instance_map_type::const_iterator const_iterator;
+
+	// Return (possibly create) a unique instance for the given hostname.
+	static PerHostRequestQueuePtr instance(std::string const& hostname);
+
+	// Release instance (object will be deleted if this was the last instance).
+	static void release(PerHostRequestQueuePtr& instance);
+
+	// Remove everything. Called upon viewer exit.
+	static void purge(void);
+
+  private:
+	typedef std::deque<AICurlEasyRequest> queued_request_type;
+
+	int mAdded;									// Number of active easy handles with this host.
+	queued_request_type mQueuedRequests;		// Waiting (throttled) requests.
+
+  public:
+	void added_to_multi_handle(void);					// Called when an easy handle for this host has been added to the multi handle.
+	void removed_from_multi_handle(void);				// Called when an easy handle for this host is removed again from the multi handle.
+	bool throttled(void) const;							// Returns true if the maximum number of allowed requests for this host have been added to the multi handle.
+
+	void queue(AICurlEasyRequest const& easy_request);	// Add easy_request to the queue.
+	bool cancel(AICurlEasyRequest const& easy_request);	// Remove easy_request from the queue (if it's there).
+
+    void add_queued_to(curlthread::MultiHandle* mh);	// Add queued easy handle (if any) to the multi handle. The request is removed from the queue,
+														// followed by either a call to added_to_multi_handle() or to queue() to add it back.
+  private:
+	// Disallow copying.
+	PerHostRequestQueue(PerHostRequestQueue const&) { }
+};
+
+class RefCountedThreadSafePerHostRequestQueue : public threadsafe_PerHostRequestQueue {
+  public:
+	RefCountedThreadSafePerHostRequestQueue(void) : mReferenceCount(0) { }
+	bool lastone(void) const { llassert(mReferenceCount >= 2); return mReferenceCount == 2; }
+
+  private:
+	// Used by PerHostRequestQueuePtr. Object is deleted when reference count reaches zero.
+	LLAtomicU32 mReferenceCount;
+
+	friend void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* p);
+	friend void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* p);
+};
+
+extern U32 curl_concurrent_connections_per_host;
+
+} // namespace AICurlPrivate
+
+#endif // AICURLPERHOST_H
--- a/indra/llmessage/aicurlprivate.h
+++ b/indra/llmessage/aicurlprivate.h
@@ -34,9 +34,11 @@
 #include <sstream>
 #include "llatomic.h"
 #include "llrefcount.h"
+#include "aicurlperhost.h"

 class AIHTTPHeaders;
 class AIHTTPTimeoutPolicy;
+class AICurlEasyRequest;
 class AICurlEasyRequestStateMachine;

 namespace AICurlPrivate {
@@ -361,6 +363,7 @@ class CurlEasyRequest : public CurlEasyHandle {

 	AIHTTPTimeoutPolicy const* mTimeoutPolicy;
 	std::string mLowercaseHostname;				// Lowercase hostname (canonicalized) extracted from the url.
+	PerHostRequestQueuePtr mPerHostPtr;			// Pointer to the corresponding PerHostRequestQueue.
 	LLPointer<curlthread::HTTPTimeout> mTimeout;// Timeout administration object associated with last created CurlSocketInfo.
 	bool mTimeoutIsOrphan;						// Set to true when mTimeout is not (yet) associated with a CurlSocketInfo.
 #if defined(CWDEBUG) || defined(DEBUG_CURLIO)
@@ -402,6 +405,11 @@ class CurlEasyRequest : public CurlEasyHandle {
 	inline ThreadSafeBufferedCurlEasyRequest* get_lockobj(void);
 	inline ThreadSafeBufferedCurlEasyRequest const* get_lockobj(void) const;

+	// PerHost API.
+	PerHostRequestQueuePtr getPerHostPtr(void);						// (Optionally create and) return a pointer to the unique
+																	// PerHostRequestQueue corresponding to mLowercaseHostname.
+	bool removeFromPerHostQueue(AICurlEasyRequest const&) const;	// Remove this request from the per-host queue, if queued at all.
+																	// Returns true if it was queued.
  protected:
 	// Pass events to parent.
 	/*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w);
--- a/indra/llmessage/aicurlthread.cpp
+++ b/indra/llmessage/aicurlthread.cpp
@@ -31,6 +31,7 @@
 #include "linden_common.h"
 #include "aicurlthread.h"
 #include "aihttptimeoutpolicy.h"
+#include "aicurlperhost.h"
 #include "lltimer.h"		// ms_sleep, get_clock_count
 #include "llhttpstatuscodes.h"
 #include "llbuffer.h"
@@ -1413,6 +1414,8 @@ void AICurlThread::run(void)
 	  }
 	  multi_handle_w->check_msg_queue();
 	}
+	// Clear the queued requests.
+	PerHostRequestQueue::purge();
  }
  AICurlMultiHandle::destroyInstance();
 }
@@ -1531,27 +1534,35 @@ CURLMsg const* MultiHandle::info_read(int* msgs_in_queue) const
  return ret;
 }

-static U32 curl_concurrent_connections = 8;						// Initialized on start up by startCurlThread().
+static U32 curl_max_total_concurrent_connections = 32;						// Initialized on start up by startCurlThread().

 void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request)
 {
-  if (mAddedEasyRequests.size() < curl_concurrent_connections)	// Not throttled?
+  bool throttled = true;		// Default.
+  PerHostRequestQueuePtr per_host;
  {
-	CURLMcode ret;
+	AICurlEasyRequest_wat curl_easy_request_w(*easy_request);
+	per_host = curl_easy_request_w->getPerHostPtr();
+	PerHostRequestQueue_wat per_host_w(*per_host);
+	if (mAddedEasyRequests.size() < curl_max_total_concurrent_connections && !per_host_w->throttled())
 	{
-	  AICurlEasyRequest_wat curl_easy_request_w(*easy_request);
 	  curl_easy_request_w->set_timeout_opts();
-	  ret = curl_easy_request_w->add_handle_to_multi(curl_easy_request_w, mMultiHandle);
-	}
-	if (ret == CURLM_OK)
-	{
-	  std::pair<addedEasyRequests_type::iterator, bool> res = mAddedEasyRequests.insert(easy_request);
-	  llassert(res.second);							// May not have been added before.
-	  Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get_ptr().get() << "; now processing " << mAddedEasyRequests.size() << " easy handles.");
-	  return;
+	  if (curl_easy_request_w->add_handle_to_multi(curl_easy_request_w, mMultiHandle) == CURLM_OK)
+	  {
+		per_host_w->added_to_multi_handle();	// (About to be) added to mAddedEasyRequests.
+		throttled = false;						// Fall through...
+	  }
 	}
+  } // Release the lock on easy_request.
+  if (!throttled)
+  {												// ... to here.
+	std::pair<addedEasyRequests_type::iterator, bool> res = mAddedEasyRequests.insert(easy_request);
+	llassert(res.second);						// May not have been added before.
+	Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get_ptr().get() << "; now processing " << mAddedEasyRequests.size() << " easy handles.");
+	return;
  }
-  mQueuedRequests.push_back(easy_request);
+  // The request could not be added, we have to queue it.
+  PerHostRequestQueue_wat(*per_host)->queue(easy_request);
 #ifdef SHOW_ASSERT
  // Not active yet, but it's no longer an error if next we try to remove the request.
  AICurlEasyRequest_wat(*easy_request)->mRemovedPerCommand = false;
@@ -1560,31 +1571,22 @@ void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request)

 CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request, bool as_per_command)
 {
+  AICurlEasyRequest_wat easy_request_w(*easy_request);
  addedEasyRequests_type::iterator iter = mAddedEasyRequests.find(easy_request);
  if (iter == mAddedEasyRequests.end())
  {
 	// The request could be queued.
-	std::deque<AICurlEasyRequest>::iterator const end = mQueuedRequests.end();
-	std::deque<AICurlEasyRequest>::iterator cur = std::find(mQueuedRequests.begin(), end, easy_request);
-	if (cur != end)
-	{
-	  // We can't use erase because that uses assignment to move elements, which is private because it isn't thread-safe for AICurlEasyRequest.
-	  // Therefore, move the element that we found to the back with swap (could just swap with the end immediately,
-	  // but I don't want to break the order in which requests where added). Swap is also not thread-safe, but OK here
-	  // because it only touches the AICurlEasyRequest objects in the deque, and the deque is protected by the
-	  // lock on MultiHandle.
-	  std::deque<AICurlEasyRequest>::iterator prev = cur;
-	  while (++cur != end)
-	  {
-		prev->swap(*cur);
-		prev = cur;
-	  }
 #ifdef SHOW_ASSERT
-	  // Now a second remove command would be an error again.
-	  AICurlEasyRequest_wat(**prev)->mRemovedPerCommand = true;
+	bool removed =
 #endif
-	  mQueuedRequests.pop_back();
+	easy_request_w->removeFromPerHostQueue(easy_request);
+#ifdef SHOW_ASSERT
+	if (removed)
+	{
+	  // Now a second remove command would be an error again.
+	  AICurlEasyRequest_wat(*easy_request)->mRemovedPerCommand = true;
 	}
+#endif
 	return (CURLMcode)-2;				// Was already removed before, or never added (queued).
  }
  return remove_easy_request(iter, as_per_command);
@@ -1593,9 +1595,12 @@ CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request
 CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator const& iter, bool as_per_command)
 {
  CURLMcode res;
+  PerHostRequestQueuePtr per_host;
  {
 	AICurlEasyRequest_wat curl_easy_request_w(**iter);
 	res = curl_easy_request_w->remove_handle_from_multi(curl_easy_request_w, mMultiHandle);
+	per_host = curl_easy_request_w->getPerHostPtr();
+	PerHostRequestQueue_wat(*per_host)->removed_from_multi_handle();		// (About to be) removed from mAddedEasyRequests.
 #ifdef SHOW_ASSERT
 	curl_easy_request_w->mRemovedPerCommand = as_per_command;
 #endif
@@ -1607,12 +1612,7 @@ CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator cons
  Dout(dc::curl, "MultiHandle::remove_easy_request: Removed AICurlEasyRequest " << (void*)lockobj << "; now processing " << mAddedEasyRequests.size() << " easy handles.");

  // Attempt to add a queued request, if any.
-  if (!mQueuedRequests.empty())
-  {
-	add_easy_request(mQueuedRequests.front());
-	mQueuedRequests.pop_front();
-  }
-
+  PerHostRequestQueue_wat(*per_host)->add_queued_to(this);
  return res;
 }

@@ -2455,23 +2455,37 @@ void AICurlEasyRequest::removeRequest(void)

 namespace AICurlInterface {

-void startCurlThread(U32 CurlConcurrentConnections, bool NoVerifySSLCert)
+void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerHost, bool NoVerifySSLCert)
 {
+  using namespace AICurlPrivate;
  using namespace AICurlPrivate::curlthread;

  llassert(is_main_thread());
-  curl_concurrent_connections = CurlConcurrentConnections;	// Debug Setting.
-  gNoVerifySSLCert = NoVerifySSLCert;						// Debug Setting.
+
+  // Cache Debug Settings.
+  curl_max_total_concurrent_connections = CurlMaxTotalConcurrentConnections;
+  curl_concurrent_connections_per_host = CurlConcurrentConnectionsPerHost;
+  gNoVerifySSLCert = NoVerifySSLCert;
+
  AICurlThread::sInstance = new AICurlThread;
  AICurlThread::sInstance->start();
 }

-bool handleCurlConcurrentConnections(LLSD const& newvalue)
+bool handleCurlMaxTotalConcurrentConnections(LLSD const& newvalue)
 {
  using namespace AICurlPrivate::curlthread;

-  curl_concurrent_connections = newvalue.asInteger();
-  llinfos << "CurlConcurrentConnections set to " << curl_concurrent_connections << llendl;
+  curl_max_total_concurrent_connections = newvalue.asInteger();
+  llinfos << "CurlMaxTotalConcurrentConnections set to " << curl_max_total_concurrent_connections << llendl;
+  return true;
+}
+
+bool handleCurlConcurrentConnectionsPerHost(LLSD const& newvalue)
+{
+  using namespace AICurlPrivate;
+
+  curl_concurrent_connections_per_host = newvalue.asInteger();
+  llinfos << "CurlConcurrentConnectionsPerHost set to " << curl_concurrent_connections_per_host << llendl;
  return true;
 }

--- a/indra/llmessage/aicurlthread.h
+++ b/indra/llmessage/aicurlthread.h
@@ -33,7 +33,6 @@

 #include "aicurl.h"
 #include <vector>
-#include <deque>

 #undef AICurlPrivate

@@ -103,10 +102,6 @@ class MultiHandle : public CurlMultiHandle

 	PollSet* mReadPollSet;
 	PollSet* mWritePollSet;
-
-  private:
-	// Temporary throttling hack.
-	std::deque<AICurlEasyRequest> mQueuedRequests;	// Waiting (throttled) requests.
 };

 } // namespace curlthread
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -4205,15 +4205,26 @@
      <key>Value</key>
      <integer>0</integer>
    </map>
-    <key>CurlConcurrentConnections</key>
+    <key>CurlMaxTotalConcurrentConnections</key>
    <map>
      <key>Comment</key>
-      <string>Maximum number of simultaneous curl connections</string>
+      <string>Maximum total number of simultaneous curl connections</string>
      <key>Persist</key>
      <integer>1</integer>
      <key>Type</key>
      <string>U32</string>
      <key>Value</key>
+      <integer>64</integer>
+    </map>
+    <key>CurlConcurrentConnectionsPerHost</key>
+    <map>
+      <key>Comment</key>
+      <string>Maximum number of simultaneous curl connections per host</string>
+      <key>Persist</key>
+      <integer>0</integer>
+      <key>Type</key>
+      <string>U32</string>
+      <key>Value</key>
      <integer>16</integer>
    </map>
    <key>CurlMaximumNumberOfHandles</key>
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -1836,7 +1836,9 @@ bool LLAppViewer::initThreads()
 		LLWatchdog::getInstance()->init(watchdog_killer_callback);
 	}

-	AICurlInterface::startCurlThread(gSavedSettings.getU32("CurlConcurrentConnections"), gSavedSettings.getBOOL("NoVerifySSLCert"));
+	AICurlInterface::startCurlThread(gSavedSettings.getU32("CurlMaxTotalConcurrentConnections"),
+		                             gSavedSettings.getU32("CurlConcurrentConnectionsPerHost"),
+		                             gSavedSettings.getBOOL("NoVerifySSLCert"));

 	LLImage::initClass();
 	
--- a/indra/newview/llviewercontrol.cpp
+++ b/indra/newview/llviewercontrol.cpp
@@ -796,7 +796,8 @@ void settings_setup_listeners()
 	gSavedSettings.getControl("AscentAvatarYModifier")->getSignal()->connect(boost::bind(&handleAscentAvatarModifier, _2));
 	gSavedSettings.getControl("AscentAvatarZModifier")->getSignal()->connect(boost::bind(&handleAscentAvatarModifier, _2));

-	gSavedSettings.getControl("CurlConcurrentConnections")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlConcurrentConnections, _2));
+	gSavedSettings.getControl("CurlMaxTotalConcurrentConnections")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlMaxTotalConcurrentConnections, _2));
+	gSavedSettings.getControl("CurlConcurrentConnectionsPerHost")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlConcurrentConnectionsPerHost, _2));
 	gSavedSettings.getControl("NoVerifySSLCert")->getSignal()->connect(boost::bind(&AICurlInterface::handleNoVerifySSLCert, _2));

 	gSavedSettings.getControl("CurlTimeoutDNSLookup")->getValidateSignal()->connect(boost::bind(&validateCurlTimeoutDNSLookup, _2));