Merge remote-tracking branch 'aleric/master'

indra/llmessage/CMakeLists.txt

@@ -22,6 +22,7 @@ include_directories(
     )
 
 set(llmessage_SOURCE_FILES
+    aiaverage.cpp
     aicurl.cpp
     aicurleasyrequeststatemachine.cpp
     aicurlperservice.cpp
@@ -109,6 +110,7 @@ set(llmessage_SOURCE_FILES
 set(llmessage_HEADER_FILES
     CMakeLists.txt
 
+    aiaverage.h
     aicurl.h
     aicurleasyrequeststatemachine.h
     aicurlperservice.h

indra/llmessage/aiaverage.cpp

@@ -0,0 +1,81 @@
+/**
+ * @file aiaverage.cpp
+ * @brief Implementation of AIAverage
+ *
+ * Copyright (c) 2013, Aleric Inglewood.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution.
+ *
+ * CHANGELOG
+ *   and additional copyright holders.
+ *
+ *   11/04/2013
+ *   Initial version, written by Aleric Inglewood @ SL
+ */
+
+#include "sys.h"
+#include "aiaverage.h"
+#include "llerror.h"	// llassert
+
+void AIAverage::cleanup(U64 clock_tick)
+{
+  // This expression can fail because the curl thread caches the time in
+  // sTime_10ms for the duration of an entire loop. Therefore, the time can
+  // go into the next 40ms and a texture fetch worker thread might call
+  // cleanup() with that time, setting mCurrentClock to a value (one)
+  // larger than sTime_10ms / 4. Next, the curl thread can continue to call
+  // this function with the smaller value; in that case just add the new
+  // data to the current bucket.
+  //
+  // Or, this is just the one-time initialization that happens the first
+  // time this is called. In that case initialize just mCurrentClock:
+  // the rest is already initialized upon construction.
+  if (LL_LIKELY(clock_tick > mCurrentClock))
+  {
+	// Advance to the next bucket.
+	++mCurrentBucket;
+	mCurrentBucket %= mNrOfBuckets;
+	// Initialize the new bucket.
+	mData[mCurrentBucket].time = clock_tick;
+	// Clean up old buckets.
+	U64 old_time = clock_tick - mNrOfBuckets;
+	if (LL_UNLIKELY(mTail == mCurrentBucket) ||		// Extremely unlikely: only happens when data was added EVERY clock tick for the past mNrOfBuckets clock ticks.
+		mData[mTail].time <= old_time)
+	{
+	  do
+	  {
+		mSum -= mData[mTail].sum;
+		mN -= mData[mTail].n;
+		mData[mTail].sum = 0;
+		mData[mTail].n = 0;
+		++mTail;
+		if (LL_UNLIKELY(mTail == mNrOfBuckets))
+		{
+		  mTail = 0;
+		}
+	  }
+	  while (mData[mTail].time <= old_time);
+	}
+	// This was set to zero when mTail passed this point (likely not this call, but a few calls ago).
+	llassert(mData[mCurrentBucket].sum == 0 &&
+	         mData[mCurrentBucket].n == 0);
+  }
+  mCurrentClock = clock_tick;
+  return;
+}
+

indra/llmessage/aiaverage.h

@@ -0,0 +1,108 @@
+/**
+ * @file aiaverage.h
+ * @brief Definition of class AIAverage
+ *
+ * Copyright (c) 2013, Aleric Inglewood.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * There are special exceptions to the terms and conditions of the GPL as
+ * it is applied to this Source Code. View the full text of the exception
+ * in the file doc/FLOSS-exception.txt in this software distribution.
+ *
+ * CHANGELOG
+ *   and additional copyright holders.
+ *
+ *   11/04/2013
+ *   Initial version, written by Aleric Inglewood @ SL
+ */
+
+#ifndef AIAVERAGE_H
+#define AIAVERAGE_H
+
+#include "llpreprocessor.h"
+#include "stdtypes.h"	// U32, U64
+#include "llthread.h"	// LLMutex
+#include <cstddef>      // size_t
+#include <vector>
+
+class AIAverage {
+  private:
+	struct Data {
+	  U32 sum;						// Accumulated sum of the 'n' passed to operator()(size_t n, U64 clock_tick) with clock_tick == time.
+	  U32 n;						// The number of calls to operator().
+	  U64 time;						// The clock_tick as passed to operator()(size_t n, U64 clock_tick) that sum corresponds to.
+	};
+
+	U64 mCurrentClock;				// The current (last) time that operator() was called with, or -1 when not initialized.
+	int mTail;						// The oldest bucket with still valid data.
+	int mCurrentBucket;				// The bucket that corresponds to mCurrentClock.
+	size_t mSum;					// The sum of all the 'n' passed to operator()(size_t n, U64 clock_tick) for all passed mNrOfBuckets time units.
+	U32 mN;							// The number of calls to operator().
+	int const mNrOfBuckets;			// Size of mData.
+	std::vector<Data> mData;		// The buckets.
+	LLMutex mLock;					// Mutex for all of the above data.
+
+  public:
+	AIAverage(int number_of_buckets) : mCurrentClock(~(U64)0), mTail(0), mCurrentBucket(0), mSum(0), mN(0), mNrOfBuckets(number_of_buckets), mData(number_of_buckets)
+	{
+	  // Fill mData with all zeroes (much faster than adding a constructor to Data).
+	  std::memset(&*mData.begin(), 0, number_of_buckets * sizeof(Data));
+	}
+	size_t addData(U32 n, U64 clock_tick)
+	{
+	  DoutEntering(dc::curl, "AIAverage::addData(" << n << ", " << clock_tick << ")");
+	  mLock.lock();
+	  if (LL_UNLIKELY(clock_tick != mCurrentClock))
+	  {
+		cleanup(clock_tick);
+	  }
+	  mSum += n;
+	  mN += 1;
+	  mData[mCurrentBucket].sum += n;
+	  mData[mCurrentBucket].n += 1;
+	  size_t sum = mSum;
+	  mLock.unlock();
+	  Dout(dc::curl, "Current sum: " << sum << ", average: " << (sum / mN));
+	  return sum;
+	}
+	size_t truncateData(U64 clock_tick)
+	{
+	  mLock.lock();
+	  if (clock_tick != mCurrentClock)
+	  {
+		cleanup(clock_tick);
+	  }
+	  size_t sum = mSum;
+	  mLock.unlock();
+	  return sum;
+	}
+	double getAverage(double avg_no_data)
+	{
+	  mLock.lock();
+	  double avg = mSum;
+	  llassert(mN != 0 || mSum == 0);
+	  if (LL_UNLIKELY(mN == 0))
+		avg = avg_no_data;
+	  else
+		avg /= mN;
+	  mLock.unlock();
+	  return avg;
+	}
+
+  private:
+	void cleanup(U64 clock_tick);
+};
+
+#endif // AIAVERAGE

indra/llmessage/aicurl.cpp

@@ -1267,8 +1267,9 @@ static int const HTTP_REDIRECTS_DEFAULT = 10;
 LLChannelDescriptors const BufferedCurlEasyRequest::sChannels;
 LLMutex BufferedCurlEasyRequest::sResponderCallbackMutex;
 bool BufferedCurlEasyRequest::sShuttingDown = false;
+AIAverage BufferedCurlEasyRequest::sHTTPBandwidth(25);
 
-BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mResponseTransferedBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER)
+BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mTotalRawBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER)
 {
   AICurlInterface::Stats::BufferedCurlEasyRequest_count++;
 }
@@ -1332,7 +1333,7 @@ void BufferedCurlEasyRequest::resetState(void)
   mOutput.reset();
   mInput.reset();
   mRequestTransferedBytes = 0;
-  mResponseTransferedBytes = 0;
+  mTotalRawBytes = 0;
   mBufferEventsTarget = NULL;
   mStatus = HTTP_INTERNAL_ERROR_OTHER;
 }

indra/llmessage/aicurlperservice.h

@@ -45,6 +45,7 @@
 #include <map>
 #include <boost/intrusive_ptr.hpp>
 #include "aithreadsafe.h"
+#include "aiaverage.h"
 
 class AICurlEasyRequest;
 class AIPerServiceRequestQueue;
@@ -89,7 +90,7 @@ class AIPerServiceRequestQueue {
 	static threadsafe_instance_map_type sInstanceMap;				// Map of AIPerServiceRequestQueue instances with the hostname as key.
 
 	friend class AIThreadSafeSimpleDC<AIPerServiceRequestQueue>;		//threadsafe_PerServiceRequestQueue
-	AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { }
+	AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false), mHTTPBandwidth(25) { }	// 25 = 1000 ms / 40 ms.
 
   public:
 	typedef instance_map_type::iterator iterator;
@@ -124,6 +125,8 @@ class AIPerServiceRequestQueue {
 	static bool sQueueFull;						// Set to true when sTotalQueued is still larger than zero after popping any queue.
 	static bool sRequestStarvation;				// Set to true when any queue was about to be popped when sTotalQueued was already zero.
 
+	AIAverage mHTTPBandwidth;				// Keeps track on number of bytes received for this service in the past second.
+
   public:
 	void added_to_command_queue(void) { ++mQueuedCommands; }
 	void removed_from_command_queue(void) { --mQueuedCommands; llassert(mQueuedCommands >= 0); }
@@ -141,14 +144,17 @@ class AIPerServiceRequestQueue {
 	S32 pipelined_requests(void) const { return mQueuedCommands + mQueuedRequests.size() + mAdded; }
 	static S32 total_queued_size(void) { return sTotalQueued; }
 
+	AIAverage& bandwidth(void) { return mHTTPBandwidth; }
+	AIAverage const& bandwidth(void) const { return mHTTPBandwidth; }
+
 	// Returns true if curl can handle another request for this host.
 	// Should return false if the maximum allowed HTTP bandwidth is reached, or when
 	// the latency between request and actual delivery becomes too large.
-	static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth);
+	static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling);
 
   private:
 	// Disallow copying.
-	AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) { }
+	AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) : mHTTPBandwidth(0) { }
 };
 
 namespace AICurlPrivate {

indra/llmessage/aicurlprivate.h

@@ -396,6 +396,9 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
     // Post-initialization, set the parent to pass the events to.
     void send_buffer_events_to(AIBufferedCurlEasyRequestEvents* target) { mBufferEventsTarget = target; }
 
+	// Called whenever new body data was (might be) received. Keeps track of the used HTTP bandwidth.
+	void update_body_bandwidth(void);
+
   protected:
 	// Events from this class.
 	/*virtual*/ void received_HTTP_header(void);
@@ -411,13 +414,14 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
 	U32 mStatus;										// HTTP status, decoded from the first header line.
 	std::string mReason;								// The "reason" from the same header line.
 	U32 mRequestTransferedBytes;
-	U32 mResponseTransferedBytes;
+	size_t mTotalRawBytes;								// Raw body data (still, possibly, compressed) received from the server so far.
 	AIBufferedCurlEasyRequestEvents* mBufferEventsTarget;
 
   public:
 	static LLChannelDescriptors const sChannels;		// Channel object for mInput (channel out()) and mOutput (channel in()).
 	static LLMutex sResponderCallbackMutex;				// Locked while calling back any overridden ResponderBase::finished and/or accessing sShuttingDown.
 	static bool sShuttingDown;							// If true, no additional calls to ResponderBase::finished will be made anymore.
+	static AIAverage sHTTPBandwidth;					// HTTP bandwidth usage of all services combined.
 
   private:
 	// This class may only be created by constructing a ThreadSafeBufferedCurlEasyRequest.

indra/llmessage/aicurlthread.cpp

@@ -33,6 +33,7 @@
 #include "aihttptimeoutpolicy.h"
 #include "aihttptimeout.h"
 #include "aicurlperservice.h"
+#include "aiaverage.h"
 #include "lltimer.h"		// ms_sleep, get_clock_count
 #include "llhttpstatuscodes.h"
 #include "llbuffer.h"
@@ -1829,6 +1830,8 @@ void MultiHandle::check_msg_queue(void)
 void MultiHandle::finish_easy_request(AICurlEasyRequest const& easy_request, CURLcode result)
 {
   AICurlEasyRequest_wat curl_easy_request_w(*easy_request);
+  // Final body bandwidth update.
+  curl_easy_request_w->update_body_bandwidth();
   // Store the result in the easy handle.
   curl_easy_request_w->storeResult(result);
 #ifdef CWDEBUG
@@ -2023,10 +2026,10 @@ void BufferedCurlEasyRequest::processOutput(void)
 	if (responseCode == HTTP_INTERNAL_ERROR_LOW_SPEED)
 	{
 		// Rewrite error to something understandable.
-		responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %u bytes). "
+		responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %lu bytes). "
 			"To change these values, go to Advanced --> Debug Settings and change CurlTimeoutLowSpeedLimit and CurlTimeoutLowSpeedTime respectively.",
 			mResponder->getURL().c_str(), mResponder->getHTTPTimeoutPolicy().getLowSpeedLimit(), mResponder->getHTTPTimeoutPolicy().getLowSpeedTime(),
-			mResponder->getName(), mResponseTransferedBytes);
+			mResponder->getName(), mTotalRawBytes);
 	}
 	setopt(CURLOPT_FRESH_CONNECT, TRUE);
   }
@@ -2093,7 +2096,9 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_
   // BufferedCurlEasyRequest::setBodyLimit is never called, so buffer_w->mBodyLimit is infinite.
   //S32 bytes = llmin(size * nmemb, buffer_w->mBodyLimit); buffer_w->mBodyLimit -= bytes;
   self_w->getOutput()->append(sChannels.in(), (U8 const*)data, bytes);
-  self_w->mResponseTransferedBytes += bytes;						// Accumulate data received from the server.
+  // Update HTTP bandwith.
+  self_w->update_body_bandwidth();
+  // Update timeout administration.
   if (self_w->httptimeout()->data_received(bytes))					// Update timeout administration.
   {
 	// Transfer timed out. Return 0 which will abort with error CURLE_WRITE_ERROR.
@@ -2102,6 +2107,25 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_
   return bytes;
 }
 
+void BufferedCurlEasyRequest::update_body_bandwidth(void)
+{
+  double size_download;	// Total amount of raw bytes received so far (ie. still compressed, 'bytes' is uncompressed).
+  getinfo(CURLINFO_SIZE_DOWNLOAD, &size_download);
+  size_t total_raw_bytes = size_download;
+  size_t raw_bytes = total_raw_bytes - mTotalRawBytes;
+  mTotalRawBytes = total_raw_bytes;
+  // Note that in some cases (like HTTP_PARTIAL_CONTENT), the output of CURLINFO_SIZE_DOWNLOAD lags
+  // behind and will return 0 the first time, and the value of the previous chunk the next time.
+  // The last call from MultiHandle::finish_easy_request recorrects this, in that case.
+  if (raw_bytes > 0)
+  {
+	U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2;
+	AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*getPerServicePtr())->bandwidth());
+	http_bandwidth.addData(raw_bytes, sTime_40ms);
+	sHTTPBandwidth.addData(raw_bytes, sTime_40ms);
+  }
+}
+
 //static
 size_t BufferedCurlEasyRequest::curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data)
 {
@@ -2189,6 +2213,11 @@ size_t BufferedCurlEasyRequest::curlHeaderCallback(char* data, size_t size, size
 	  self_w->httptimeout()->being_redirected();
 	}
   }
+  // Update HTTP bandwidth.
+  U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2;
+  AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*self_w->getPerServicePtr())->bandwidth());
+  http_bandwidth.addData(header_len, sTime_40ms);
+  sHTTPBandwidth.addData(header_len, sTime_40ms);
   // Update timeout administration. This must be done after the status is already known.
   if (self_w->httptimeout()->data_received(header_len/*,*/ ASSERT_ONLY_COMMA(self_w->upload_error_status())))
   {
@@ -2533,6 +2562,14 @@ U32 getNumHTTPAdded(void)
   return AICurlPrivate::curlthread::MultiHandle::total_added_size();
 }
 
+size_t getHTTPBandwidth(void)
+{
+  using namespace AICurlPrivate;
+
+  U64 const sTime_40ms = get_clock_count() * curlthread::HTTPTimeout::sClockWidth_40ms;
+  return BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms);
+}
+
 } // namespace AICurlInterface
 
 // Return true if we want at least one more HTTP request for this host.
@@ -2560,7 +2597,7 @@ U32 getNumHTTPAdded(void)
 // running requests (in MultiHandle::mAddedEasyRequests)).
 //
 //static
-bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth)
+bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling)
 {
   using namespace AICurlPrivate;
   using namespace AICurlPrivate::curlthread;
@@ -2587,17 +2624,21 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue
 
   // Check if it's ok to get a new request for this particular host and update the per-host threshold.
 
+  AIAverage* http_bandwidth_ptr;
+
   // Atomic read max_pipelined_requests_per_service for the below calculations.
   S32 const max_pipelined_requests_per_service_cache = max_pipelined_requests_per_service;
   {
-	PerServiceRequestQueue_rat per_service_r(*per_service);
+	PerServiceRequestQueue_wat per_service_w(*per_service);
-	S32 const pipelined_requests_per_service = per_service_r->pipelined_requests();
+	S32 const pipelined_requests_per_service = per_service_w->pipelined_requests();
 	reject = pipelined_requests_per_service >= max_pipelined_requests_per_service_cache;
 	equal = pipelined_requests_per_service == max_pipelined_requests_per_service_cache;
-	increment_threshold = per_service_r->mRequestStarvation;
+	increment_threshold = per_service_w->mRequestStarvation;
-	decrement_threshold = per_service_r->mQueueFull && !per_service_r->mQueueEmpty;
+	decrement_threshold = per_service_w->mQueueFull && !per_service_w->mQueueEmpty;
 	// Reset flags.
-	per_service_r->mQueueFull = per_service_r->mQueueEmpty = per_service_r->mRequestStarvation = false;
+	per_service_w->mQueueFull = per_service_w->mQueueEmpty = per_service_w->mRequestStarvation = false;
+	// Grab per service bandwidth object.
+	http_bandwidth_ptr = &per_service_w->bandwidth();
   }
   if (decrement_threshold)
   {
@@ -2621,10 +2662,69 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue
 	return false;
   }
   
-  //AIFIXME: better bandwidth check here.
+  if (!no_bandwidth_throttling)
-  if (too_much_bandwidth)
   {
-	return false;	// wait
+	// Throttle on bandwidth usage.
+
+	static size_t throttle_fraction = 1024;	// A value between 0 and 1024: each service is throttled when it uses more than max_bandwidth * (throttle_fraction/1024) bandwidth.
+	static AIAverage fraction(25);			// Average over 25 * 40ms = 1 second.
+	static U64 last_sTime_40ms = 0;
+
+	// Truncate the sums to the last second, and get their value.
+	U64 const sTime_40ms = get_clock_count() * HTTPTimeout::sClockWidth_40ms;							// Time in 40ms units.
+	size_t const max_bandwidth = 125.f * max_kbps;														// Convert kbps to bytes per second.
+	size_t const total_bandwidth = BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms);	// Bytes received in the past second.
+	size_t const service_bandwidth = http_bandwidth_ptr->truncateData(sTime_40ms);						// Idem for just this service.
+	if (sTime_40ms > last_sTime_40ms)
+	{
+	  // Only add throttle_fraction once every 40 ms at most.
+	  // It's ok to ignore other values in the same 40 ms because the value only changes on the scale of 1 second.
+	  fraction.addData(throttle_fraction, sTime_40ms);
+	  last_sTime_40ms = sTime_40ms;
+	}
+	double fraction_avg = fraction.getAverage(1024.0);													// throttle_fraction averaged over the past second, or 1024 if there is no data.
+
+	// Adjust throttle_fraction based on total bandwidth usage.
+	if (total_bandwidth == 0)
+	  throttle_fraction = 1024;
+	else
+	{
+	  // This is the main formula. It can be made plausible by assuming
+	  // an equilibrium where total_bandwidth == max_bandwidth and
+	  // thus throttle_fraction == fraction_avg for more than a second.
+	  //
+	  // Then, more bandwidth is being used (for example because another
+	  // service starts downloading). Assuming that all services that use
+	  // a significant portion of the bandwidth, the new service included,
+	  // must be throttled (all using the same bandwidth; note that the
+	  // new service is immediately throttled at the same value), then
+	  // the limit should be reduced linear with the fraction:
+	  // max_bandwidth / total_bandwidth.
+	  //
+	  // For example, let max_bandwidth be 1. Let there be two throttled
+	  // services, each using 0.5 (fraction_avg = 1024/2). Lets the new
+	  // service use what it can: also 0.5 - then without reduction the
+	  // total_bandwidth would become 1.5, and throttle_fraction would
+	  // become (1024/2) * 1/1.5 = 1024/3: from 2 to 3 services.
+	  //
+	  // In reality, total_bandwidth would rise linear from 1.0 to 1.5 in
+	  // one second if the throttle fraction wasn't changed. However it is
+	  // changed here. The end result is that any change more or less
+	  // linear fades away in one second.
+	  throttle_fraction = fraction_avg * max_bandwidth / total_bandwidth;
+	}
+	if (throttle_fraction > 1024)
+	  throttle_fraction = 1024;
+	if (total_bandwidth > max_bandwidth)
+	{
+	  throttle_fraction *= 0.95;
+	}
+
+	// Throttle this service if it uses too much bandwidth.
+	if (service_bandwidth > (max_bandwidth * throttle_fraction / 1024))
+	{
+	  return false;	// wait
+	}
   }
 
   // Check if it's ok to get a new request based on the total number of requests and increment the threshold if appropriate.

indra/llmessage/aihttptimeout.cpp

@@ -97,8 +97,9 @@ namespace curlthread {
 // HTTPTimeout
 
 //static
-F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency();	// Time between two clock ticks, in 10ms units.
+F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency();		// Time between two clock ticks, in 10ms units.
-U64 HTTPTimeout::sTime_10ms;												// Time in 10ms units, set once per select() exit.
+F64 const HTTPTimeout::sClockWidth_40ms = HTTPTimeout::sClockWidth_10ms * 0.25;	// Time between two clock ticks, in 40ms units.
+U64 HTTPTimeout::sTime_10ms;													// Time in 10ms units, set once per select() exit.
 
 // CURL-THREAD
 // This is called when body data was sent to the server socket.

indra/llmessage/aihttptimeout.h

@@ -89,6 +89,7 @@ class HTTPTimeout : public LLRefCount {
 	U64 mStalled;								// The time (sTime_10ms) at which this transaction is considered to be stalling if nothing is transfered anymore.
   public:
 	static F64 const sClockWidth_10ms;			// Time between two clock ticks in 10 ms units.
+	static F64 const sClockWidth_40ms;			// Time between two clock ticks in 40 ms units.
 	static U64 sTime_10ms;						// Time since the epoch in 10 ms units.
 #if defined(CWDEBUG) || defined(DEBUG_CURLIO)
 	ThreadSafeBufferedCurlEasyRequest* mLockObj;

indra/llmessage/message.cpp

@@ -50,7 +50,6 @@
 #include "lldarray.h"
 #include "lldir.h"
 #include "llerror.h"
-#include "llerrorlegacy.h"
 #include "llfasttimer.h"
 #include "llhttpclient.h"
 #include "llhttpnodeadapter.h"

indra/newview/app_settings/settings.xml

@@ -4921,7 +4921,18 @@ This should be as low as possible, but too low may break functionality</string>
       <key>Value</key>
       <integer>-1</integer>
     </map>
-    <key>DebugStatModeTexture</key>
+    <key>DebugStatModeHTTPTexture</key>
+    <map>
+      <key>Comment</key>
+      <string>Mode of stat in Statistics floater</string>
+      <key>Persist</key>
+      <integer>1</integer>
+      <key>Type</key>
+      <string>S32</string>
+      <key>Value</key>
+      <integer>-1</integer>
+    </map>
+    <key>DebugStatModeUDPTexture</key>
     <map>
       <key>Comment</key>
       <string>Mode of stat in Statistics floater</string>

indra/newview/hippogridmanager.cpp

@@ -374,8 +374,11 @@ void HippoGridInfo::onXmlCharacterData(void* userData, const XML_Char* s, int le
 	{
 		case XML_GRIDNICK:
 		{
-			if (self->mGridNick == "") self->mGridNick.assign(s, len);
+			if (self->mGridNick == "")
-			self->mGridNick = sanitizeGridNick(self->mGridNick);
+			{
+			  self->mGridNick.assign(s, len);
+			  self->mGridNick = sanitizeGridNick(self->mGridNick);
+			}
 			break;
 		}
 
@@ -407,7 +410,15 @@ void HippoGridInfo::onXmlCharacterData(void* userData, const XML_Char* s, int le
 			break;
 		}
 
-		case XML_GRIDNAME: self->mGridName.assign(s, len); break;
+		case XML_GRIDNAME:
+		{
+		  if (self->mGridName == "")
+		  {
+			self->mGridName.assign(s, len);
+		  }
+		  break;
+		}
+
 		case XML_LOGINPAGE: self->mLoginPage.assign(s, len); break;
 		case XML_WEBSITE: self->mWebSite.assign(s, len); break;
 		case XML_SUPPORT: self->mSupportUrl.assign(s, len); break;
@@ -678,13 +689,6 @@ HippoGridInfo* HippoGridManager::getGrid(const std::string& grid) const
 	}
 }
 
-
-HippoGridInfo* HippoGridManager::getConnectedGrid() const
-{
-	return (mConnectedGrid)? mConnectedGrid: getCurrentGrid();
-}
-
-
 HippoGridInfo* HippoGridManager::getCurrentGrid() const
 {
 	HippoGridInfo* grid = getGrid(mCurrentGrid);

indra/newview/hippogridmanager.h

@@ -157,7 +157,8 @@ public:
 	void discardAndReload();
 
 	HippoGridInfo* getGrid(const std::string& grid) const;
-	HippoGridInfo* getConnectedGrid()               const;
+	HippoGridInfo* getConnectedGrid()               const { return mConnectedGrid ? mConnectedGrid : getCurrentGrid(); }
+
 	HippoGridInfo* getCurrentGrid()                 const;
 	const std::string& getDefaultGridNick()         const;
 	const std::string& getCurrentGridNick()         const;

indra/newview/hippopanelgrids.cpp

@@ -133,7 +133,7 @@ BOOL HippoPanelGridsImpl::postBuild()
 	requires<LLButton>("btn_add");
 	requires<LLButton>("btn_copy");
 	requires<LLButton>("btn_default");
-	//requires<LLButton>("btn_gridinfo");
+	requires<LLButton>("btn_gridinfo");
 	requires<LLButton>("btn_help_render_compat");
 	if (!checkRequirements()) return false;
 	
@@ -146,7 +146,7 @@ BOOL HippoPanelGridsImpl::postBuild()
 	childSetAction("btn_add", onClickAdd, this);
 	childSetAction("btn_copy", onClickCopy, this);
 	childSetAction("btn_default", onClickDefault, this);
-	//childSetAction("btn_gridinfo", onClickGridInfo, this);
+	childSetAction("btn_gridinfo", onClickGridInfo, this);
 	childSetAction("btn_help_render_compat", onClickHelpRenderCompat, this);
 	childSetAction("btn_advanced", onClickAdvanced, this);
 	

indra/newview/llfloaterstats.cpp

@@ -238,25 +238,52 @@ void LLFloaterStats::buildStats()
 	// Network statistics
 	LLStatView *net_statviewp = stat_viewp->addStatView("network stat view", "Network", "OpenDebugStatNet", rect);
 
-	stat_barp = net_statviewp->addStat("Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn");
+	stat_barp = net_statviewp->addStat("UDP Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn");
 	stat_barp->setUnitLabel("/sec");
 
-	stat_barp = net_statviewp->addStat("Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut");
+	stat_barp = net_statviewp->addStat("UDP Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut");
 	stat_barp->setUnitLabel("/sec");
 
+	stat_barp = net_statviewp->addStat("HTTP Textures", &(LLViewerStats::getInstance()->mHTTPTextureKBitStat), "DebugStatModeHTTPTexture");
+	stat_barp->setUnitLabel(" kbps");
+	stat_barp->mMinBar = 0.f;
+	stat_barp->mMaxBar = gSavedSettings.getF32("HTTPThrottleBandwidth") * 2;	// Times two because we'll have over shoots.
+	stat_barp->mTickSpacing = 1.f;
+	while (stat_barp->mTickSpacing < stat_barp->mMaxBar / 8)
+	  stat_barp->mTickSpacing *= 2.f;
+	stat_barp->mLabelSpacing = 2 * stat_barp->mTickSpacing;
+	stat_barp->mPerSec = FALSE;
+	stat_barp->mDisplayMean = FALSE;
+
+	stat_barp = net_statviewp->addStat("UDP Textures", &(LLViewerStats::getInstance()->mUDPTextureKBitStat), "DebugStatModeUDPTexture");
+	stat_barp->setUnitLabel(" kbps");
+	stat_barp->mMinBar = 0.f;
+	stat_barp->mMaxBar = 1024.f;
+	stat_barp->mTickSpacing = 128.f;
+	stat_barp->mLabelSpacing = 256.f;
+
 	stat_barp = net_statviewp->addStat("Objects", &(LLViewerStats::getInstance()->mObjectKBitStat), "DebugStatModeObjects");
 	stat_barp->setUnitLabel(" kbps");
+	stat_barp->mMinBar = 0.f;
+	stat_barp->mMaxBar = 1024.f;
+	stat_barp->mTickSpacing = 128.f;
+	stat_barp->mLabelSpacing = 256.f;
 
-	stat_barp = net_statviewp->addStat("Texture", &(LLViewerStats::getInstance()->mTextureKBitStat), "DebugStatModeTexture");
+	stat_barp = net_statviewp->addStat("Assets (UDP)", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset");
 	stat_barp->setUnitLabel(" kbps");
+	stat_barp->mMinBar = 0.f;
+	stat_barp->mMaxBar = 1024.f;
+	stat_barp->mTickSpacing = 128.f;
+	stat_barp->mLabelSpacing = 256.f;
 
-	stat_barp = net_statviewp->addStat("Asset", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset");
+	stat_barp = net_statviewp->addStat("Layers (UDP)", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers");
 	stat_barp->setUnitLabel(" kbps");
+	stat_barp->mMinBar = 0.f;
+	stat_barp->mMaxBar = 1024.f;
+	stat_barp->mTickSpacing = 128.f;
+	stat_barp->mLabelSpacing = 256.f;
 
-	stat_barp = net_statviewp->addStat("Layers", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers");
+	stat_barp = net_statviewp->addStat("Actual In (UDP)", &(LLViewerStats::getInstance()->mActualInKBitStat),
-	stat_barp->setUnitLabel(" kbps");
-
-	stat_barp = net_statviewp->addStat("Actual In", &(LLViewerStats::getInstance()->mActualInKBitStat),
 									   "DebugStatModeActualIn", TRUE, FALSE);
 	stat_barp->setUnitLabel(" kbps");
 	stat_barp->mMinBar = 0.f;
@@ -264,7 +291,7 @@ void LLFloaterStats::buildStats()
 	stat_barp->mTickSpacing = 128.f;
 	stat_barp->mLabelSpacing = 256.f;
 
-	stat_barp = net_statviewp->addStat("Actual Out", &(LLViewerStats::getInstance()->mActualOutKBitStat),
+	stat_barp = net_statviewp->addStat("Actual Out (UDP)", &(LLViewerStats::getInstance()->mActualOutKBitStat),
 									   "DebugStatModeActualOut", TRUE, FALSE);
 	stat_barp->setUnitLabel(" kbps");
 	stat_barp->mMinBar = 0.f;

indra/newview/lltexturefetch.cpp

@@ -36,7 +36,7 @@
 
 #include "lltexturefetch.h"
 
-#include "llcurl.h"
+#include "aicurl.h"
 #include "lldir.h"
 #include "llhttpclient.h"
 #include "llhttpstatuscodes.h"
@@ -1273,7 +1273,8 @@ bool LLTextureFetchWorker::doWork(S32 param)
 
 			// Let AICurl decide if we can process more HTTP requests at the moment or not.
 			static const LLCachedControl<F32> throttle_bandwidth("HTTPThrottleBandwidth", 2000);
-			if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, mFetcher->getTextureBandwidth() > throttle_bandwidth))
+			bool const no_bandwidth_throttling = gHippoGridManager->getConnectedGrid()->isAvination();
+			if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, throttle_bandwidth, no_bandwidth_throttling))
 			{
 				return false ; //wait.
 			}
@@ -1290,16 +1291,8 @@ bool LLTextureFetchWorker::doWork(S32 param)
 				mLoaded = FALSE;
 				mGetStatus = 0;
 				mGetReason.clear();
-				// Note: comparing mFetcher->getTextureBandwidth() with throttle_bandwidth is a bit like
-				// comparing apples and oranges, but it's only debug output. The first is the averaged
-				// bandwidth used for the body of successfully downloaded textures, averaged over roughtly
-				// 10 seconds, in kbits/s. The latter is the limit of the actual http curl downloads,
-				// including header and failures for anything (not just textures), averaged over the last
-				// second, also in kbits/s.
-				static const LLCachedControl<F32> throttle_bandwidth("HTTPThrottleBandwidth", 2000);
 				LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset
 									 << " Bytes: " << mRequestedSize
-									 << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << throttle_bandwidth
 									 << LL_ENDL;
 				setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
 				mState = WAIT_HTTP_REQ;	
@@ -2041,7 +2034,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image
 	  mBadPacketCount(0),
 	  mTextureCache(cache),
 	  mImageDecodeThread(imagedecodethread),
-	  mTextureBandwidth(0),
 	  mHTTPTextureBits(0),
 	  mTotalHTTPRequests(0),
 	  mQAMode(qa_mode),

indra/newview/lltexturefetch.h

@@ -79,9 +79,6 @@ public:
 	bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data);
 	bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data);
 
-	void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; }
-	F32 getTextureBandwidth() { return mTextureBandwidth; }
-	
 	// Debug
 	BOOL isFromLocalCache(const LLUUID& id);
 	S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p,
@@ -164,7 +161,6 @@ private:
 	queue_t mHTTPTextureQueue;
 	typedef std::map<LLHost,std::set<LLUUID> > cancel_queue_t;
 	cancel_queue_t mCancelQueue;
-	F32 mTextureBandwidth;
 	LLTextureInfo mTextureInfo;
 
 	U32 mHTTPTextureBits;

indra/newview/lltextureview.cpp

@@ -70,6 +70,7 @@ namespace AICurlInterface {
   U32 getNumHTTPQueued(void);
   U32 getNumHTTPAdded(void);
   U32 getNumHTTPRunning(void);
+  size_t getHTTPBandwidth(void);
 } // namespace AICurlInterface
 
 ////////////////////////////////////////////////////////////////////////////
@@ -620,17 +621,15 @@ void LLGLTexMemBar::draw()
 									 text_color, LLFontGL::LEFT, LLFontGL::TOP);
 
 	left += LLFontGL::getFontMonospace()->getWidth(text);
-	// This bandwidth is averaged over roughly 10 seconds (in kbps) and therefore pretty inaccurate.
+	// This bandwidth is averaged over 1 seconds (in kbps).
-	// Also, it only takes into account actual texture data (not headers etc). But all it is used for
+	F32 bandwidth = AICurlInterface::getHTTPBandwidth() / 125.f;		// Convert from bytes/s to kbps.
-	// is for the color of some text in the texture console, so I guess it doesn't matter.
-	F32 bandwidth = LLAppViewer::getTextureFetch()->getTextureBandwidth();
 	// This is the maximum bandwidth allowed for curl transactions (of any type and averaged per second),
 	// that is actually used to limit the number of HTTP texture requests (and only those).
 	// Comparing that with 'bandwidth' is a bit like comparing apples and oranges, but again... who really cares.
-	F32 max_bandwidth = gSavedSettings.getF32("HTTPThrottleBandwidth");
+	static const LLCachedControl<F32> max_bandwidth("HTTPThrottleBandwidth", 2000);
 	color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth*.75f ? LLColor4::yellow : text_color;
 	color[VALPHA] = text_color[VALPHA];
-	text = llformat("BW:%.0f/%.0f",bandwidth, max_bandwidth);
+	text = llformat("BW:%.0f/%.0f", bandwidth, max_bandwidth.get());
 	LLFontGL::getFontMonospace()->renderUTF8(text, 0, left, v_offset + line_height*2,
 											 color, LLFontGL::LEFT, LLFontGL::TOP);
 	

indra/newview/llviewerstats.cpp

@@ -67,6 +67,10 @@
 class AIHTTPTimeoutPolicy;
 extern AIHTTPTimeoutPolicy viewerStatsResponder_timeout;
 
+namespace AICurlInterface {
+size_t getHTTPBandwidth(void);
+}
+
 class StatAttributes
 {
 public:
@@ -210,7 +214,8 @@ LLViewerStats::LLViewerStats() :
 	mLayersKBitStat("layerskbitstat"),
 	mObjectKBitStat("objectkbitstat"),
 	mAssetKBitStat("assetkbitstat"),
-	mTextureKBitStat("texturekbitstat"),
+	mHTTPTextureKBitStat("httptexturekbitstat"),
+	mUDPTextureKBitStat("udptexturekbitstat"),
 	mMallocStat("mallocstat"),
 	mVFSPendingOperations("vfspendingoperations"),
 	mObjectsDrawnStat("objectsdrawnstat"),
@@ -300,7 +305,8 @@ void LLViewerStats::resetStats()
 	stats.mKBitStat.reset();
 	stats.mLayersKBitStat.reset();
 	stats.mObjectKBitStat.reset();
-	stats.mTextureKBitStat.reset();
+	stats.mHTTPTextureKBitStat.reset();
+	stats.mUDPTextureKBitStat.reset();
 	stats.mVFSPendingOperations.reset();
 	stats.mAssetKBitStat.reset();
 	stats.mPacketsInStat.reset();
@@ -673,13 +679,13 @@ void update_statistics()
 
 	// Only update texture stats periodically so that they are less noisy
 	{
-		static const F32 texture_stats_freq = 10.f;
+		static const F32 texture_stats_freq = 0.25f;
 		static LLFrameTimer texture_stats_timer;
 		if (texture_stats_timer.getElapsedTimeF32() >= texture_stats_freq)
 		{
-			stats.mTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f);
+			stats.mHTTPTextureKBitStat.addValue(AICurlInterface::getHTTPBandwidth()/125.f);
+			stats.mUDPTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f);
 			stats.mTexturePacketsStat.addValue(LLViewerTextureList::sTexturePackets);
-			LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerTextureList::sTextureBits/1024.f/texture_stats_timer.getElapsedTimeF32());
 			gTotalTextureBytes += LLViewerTextureList::sTextureBits / 8;
 			LLViewerTextureList::sTextureBits = 0;
 			LLViewerTextureList::sTexturePackets = 0;

indra/newview/llviewerstats.h

@@ -43,7 +43,8 @@ public:
 			mLayersKBitStat,
 			mObjectKBitStat,
 			mAssetKBitStat,
-			mTextureKBitStat,
+			mHTTPTextureKBitStat,
+			mUDPTextureKBitStat,
 			mVFSPendingOperations,
 			mObjectsDrawnStat,
 			mObjectsCulledStat,

indra/newview/llviewertexturelist.cpp

@@ -687,8 +687,6 @@ void LLViewerTextureList::updateImages(F32 max_time)
 	}
 	cleared = FALSE;
 
-	LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerStats::getInstance()->mTextureKBitStat.getMeanPerSec());
-
 	S32 global_raw_memory;
 	{
 		global_raw_memory = *AIAccess<S32>(LLImageRaw::sGlobalRawMemory);

indra/newview/skins/default/xui/en-us/panel_preferences_grids.xml

@@ -30,17 +30,17 @@
   <!-- Buttons -->
   <button label="Delete" label_selected="Delete" enabled="true" name="btn_delete"
 		height="18" width="75" left="120" bottom_delta="-27"
-		halign="center"
+		halign="center" tool_type="Delete the selected grid from the list"
 		follows="left|top" scale_image="true"
 	    font="SansSerifSmall" mouse_opaque="true" />
-  <button label="Add" label_selected="Add" enabled="true" name="btn_add"
+  <button label="Create" label_selected="Add" enabled="true" name="btn_add"
 		height="18" width="75" left_delta="78" bottom_delta="0"
-		halign="center"
+		halign="center" tool_tip="Add a new grid to the list"
 		follows="left|top" scale_image="true"
 	    font="SansSerifSmall" mouse_opaque="true" />
   <button label="Copy" label_selected="Copy" enabled="true" name="btn_copy"
 		height="18" width="75" left_delta="78" bottom_delta="0"
-		halign="center"
+		halign="center" tool_tip="Create a new entry using the selected entry as template"
 		follows="left|top" scale_image="true"
 	    font="SansSerifSmall" mouse_opaque="true" />
   <!-- Advanced -->
@@ -65,11 +65,11 @@
 		font="SansSerifSmall"
 		follows="left|top|right" border_visible="false" mouse_opaque="false"
 		drop_shadow_visible="true" border_drop_shadow_visible="false" />
-  <!--<button label="Get Grid Info" label_selected="Get Grid Info" enabled="true" name="btn_gridinfo"
+  <button label="Refresh Grid URLs" enabled="true" name="btn_gridinfo"
 		height="18" width="100" left="120" bottom_delta="-22"
-		halign="center"
+		halign="center" tool_tip="Retrieve all grid info from the given Login URI (overwriting the current URLs, see Advanced)"
 		follows="left|top" scale_image="true"
-	    font="SansSerifSmall" mouse_opaque="true" />-->
+	    font="SansSerifSmall" mouse_opaque="true" />
   <!-- Platform -->
   <text type="string" length="1" enabled="true" name="platform_label"
 		height="10" width="100" left="12" bottom_delta="-32"