From 45e6b7975f82f90345e35f87e6292b706362db4b Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 24 Apr 2013 22:04:21 +0200 Subject: [PATCH] First version of HTTP bandwidth throttling. Adds throttling based on on average bandwidth usage per HTTP service. Since only HTTP textures are using this, they are still starved by other services like inventory and mesh dowloads. Also, it will be needed to move the maximum number of connections per service the to the PerService class, and dynamically tune them: reducing the number of connections is the first thing to do when using too much bandwidth. I also added a graph for HTTP texture bandwidth to the stats floater. For some reason the average bandwidth (over 1 second) look almost like scattered noise... weird for something that is averaged... --- indra/llmessage/CMakeLists.txt | 2 + indra/llmessage/aiaverage.cpp | 81 ++++++++++++++++ indra/llmessage/aiaverage.h | 108 +++++++++++++++++++++ indra/llmessage/aicurl.cpp | 5 +- indra/llmessage/aicurlperservice.h | 12 ++- indra/llmessage/aicurlprivate.h | 6 +- indra/llmessage/aicurlthread.cpp | 124 +++++++++++++++++++++--- indra/llmessage/aihttptimeout.cpp | 5 +- indra/llmessage/aihttptimeout.h | 1 + indra/llmessage/message.cpp | 1 - indra/newview/app_settings/settings.xml | 13 ++- indra/newview/hippogridmanager.cpp | 7 -- indra/newview/hippogridmanager.h | 3 +- indra/newview/llfloaterstats.cpp | 45 +++++++-- indra/newview/lltexturefetch.cpp | 14 +-- indra/newview/lltexturefetch.h | 4 - indra/newview/lltextureview.cpp | 11 +-- indra/newview/llviewerstats.cpp | 16 ++- indra/newview/llviewerstats.h | 3 +- indra/newview/llviewertexturelist.cpp | 2 - 20 files changed, 395 insertions(+), 68 deletions(-) create mode 100644 indra/llmessage/aiaverage.cpp create mode 100644 indra/llmessage/aiaverage.h diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index eb45729b9..53e890040 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -22,6 +22,7 @@ include_directories( ) set(llmessage_SOURCE_FILES + aiaverage.cpp aicurl.cpp aicurleasyrequeststatemachine.cpp aicurlperservice.cpp @@ -109,6 +110,7 @@ set(llmessage_SOURCE_FILES set(llmessage_HEADER_FILES CMakeLists.txt + aiaverage.h aicurl.h aicurleasyrequeststatemachine.h aicurlperservice.h diff --git a/indra/llmessage/aiaverage.cpp b/indra/llmessage/aiaverage.cpp new file mode 100644 index 000000000..10aafd852 --- /dev/null +++ b/indra/llmessage/aiaverage.cpp @@ -0,0 +1,81 @@ +/** + * @file aiaverage.cpp + * @brief Implementation of AIAverage + * + * Copyright (c) 2013, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 11/04/2013 + * Initial version, written by Aleric Inglewood @ SL + */ + +#include "sys.h" +#include "aiaverage.h" +#include "llerror.h" // llassert + +void AIAverage::cleanup(U64 clock_tick) +{ + // This expression can fail because the curl thread caches the time in + // sTime_10ms for the duration of an entire loop. Therefore, the time can + // go into the next 40ms and a texture fetch worker thread might call + // cleanup() with that time, setting mCurrentClock to a value (one) + // larger than sTime_10ms / 4. Next, the curl thread can continue to call + // this function with the smaller value; in that case just add the new + // data to the current bucket. + // + // Or, this is just the one-time initialization that happens the first + // time this is called. In that case initialize just mCurrentClock: + // the rest is already initialized upon construction. + if (LL_LIKELY(clock_tick > mCurrentClock)) + { + // Advance to the next bucket. + ++mCurrentBucket; + mCurrentBucket %= mNrOfBuckets; + // Initialize the new bucket. + mData[mCurrentBucket].time = clock_tick; + // Clean up old buckets. + U64 old_time = clock_tick - mNrOfBuckets; + if (LL_UNLIKELY(mTail == mCurrentBucket) || // Extremely unlikely: only happens when data was added EVERY clock tick for the past mNrOfBuckets clock ticks. + mData[mTail].time <= old_time) + { + do + { + mSum -= mData[mTail].sum; + mN -= mData[mTail].n; + mData[mTail].sum = 0; + mData[mTail].n = 0; + ++mTail; + if (LL_UNLIKELY(mTail == mNrOfBuckets)) + { + mTail = 0; + } + } + while (mData[mTail].time <= old_time); + } + // This was set to zero when mTail passed this point (likely not this call, but a few calls ago). + llassert(mData[mCurrentBucket].sum == 0 && + mData[mCurrentBucket].n == 0); + } + mCurrentClock = clock_tick; + return; +} + diff --git a/indra/llmessage/aiaverage.h b/indra/llmessage/aiaverage.h new file mode 100644 index 000000000..f8f344df6 --- /dev/null +++ b/indra/llmessage/aiaverage.h @@ -0,0 +1,108 @@ +/** + * @file aiaverage.h + * @brief Definition of class AIAverage + * + * Copyright (c) 2013, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 11/04/2013 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AIAVERAGE_H +#define AIAVERAGE_H + +#include "llpreprocessor.h" +#include "stdtypes.h" // U32, U64 +#include "llthread.h" // LLMutex +#include // size_t +#include + +class AIAverage { + private: + struct Data { + U32 sum; // Accumulated sum of the 'n' passed to operator()(size_t n, U64 clock_tick) with clock_tick == time. + U32 n; // The number of calls to operator(). + U64 time; // The clock_tick as passed to operator()(size_t n, U64 clock_tick) that sum corresponds to. + }; + + U64 mCurrentClock; // The current (last) time that operator() was called with, or -1 when not initialized. + int mTail; // The oldest bucket with still valid data. + int mCurrentBucket; // The bucket that corresponds to mCurrentClock. + size_t mSum; // The sum of all the 'n' passed to operator()(size_t n, U64 clock_tick) for all passed mNrOfBuckets time units. + U32 mN; // The number of calls to operator(). + int const mNrOfBuckets; // Size of mData. + std::vector mData; // The buckets. + LLMutex mLock; // Mutex for all of the above data. + + public: + AIAverage(int number_of_buckets) : mCurrentClock(~(U64)0), mTail(0), mCurrentBucket(0), mSum(0), mN(0), mNrOfBuckets(number_of_buckets), mData(number_of_buckets) + { + // Fill mData with all zeroes (much faster than adding a constructor to Data). + std::memset(&*mData.begin(), 0, number_of_buckets * sizeof(Data)); + } + size_t addData(U32 n, U64 clock_tick) + { + DoutEntering(dc::curl, "AIAverage::addData(" << n << ", " << clock_tick << ")"); + mLock.lock(); + if (LL_UNLIKELY(clock_tick != mCurrentClock)) + { + cleanup(clock_tick); + } + mSum += n; + mN += 1; + mData[mCurrentBucket].sum += n; + mData[mCurrentBucket].n += 1; + size_t sum = mSum; + mLock.unlock(); + Dout(dc::curl, "Current sum: " << sum << ", average: " << (sum / mN)); + return sum; + } + size_t truncateData(U64 clock_tick) + { + mLock.lock(); + if (clock_tick != mCurrentClock) + { + cleanup(clock_tick); + } + size_t sum = mSum; + mLock.unlock(); + return sum; + } + double getAverage(double avg_no_data) + { + mLock.lock(); + double avg = mSum; + llassert(mN != 0 || mSum == 0); + if (LL_UNLIKELY(mN == 0)) + avg = avg_no_data; + else + avg /= mN; + mLock.unlock(); + return avg; + } + + private: + void cleanup(U64 clock_tick); +}; + +#endif // AIAVERAGE diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 651e0e9eb..93153951a 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -1267,8 +1267,9 @@ static int const HTTP_REDIRECTS_DEFAULT = 10; LLChannelDescriptors const BufferedCurlEasyRequest::sChannels; LLMutex BufferedCurlEasyRequest::sResponderCallbackMutex; bool BufferedCurlEasyRequest::sShuttingDown = false; +AIAverage BufferedCurlEasyRequest::sHTTPBandwidth(25); -BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mResponseTransferedBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER) +BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mTotalRawBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER) { AICurlInterface::Stats::BufferedCurlEasyRequest_count++; } @@ -1332,7 +1333,7 @@ void BufferedCurlEasyRequest::resetState(void) mOutput.reset(); mInput.reset(); mRequestTransferedBytes = 0; - mResponseTransferedBytes = 0; + mTotalRawBytes = 0; mBufferEventsTarget = NULL; mStatus = HTTP_INTERNAL_ERROR_OTHER; } diff --git a/indra/llmessage/aicurlperservice.h b/indra/llmessage/aicurlperservice.h index cf4d8ba71..8cd310f40 100644 --- a/indra/llmessage/aicurlperservice.h +++ b/indra/llmessage/aicurlperservice.h @@ -45,6 +45,7 @@ #include #include #include "aithreadsafe.h" +#include "aiaverage.h" class AICurlEasyRequest; class AIPerServiceRequestQueue; @@ -89,7 +90,7 @@ class AIPerServiceRequestQueue { static threadsafe_instance_map_type sInstanceMap; // Map of AIPerServiceRequestQueue instances with the hostname as key. friend class AIThreadSafeSimpleDC; //threadsafe_PerServiceRequestQueue - AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { } + AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false), mHTTPBandwidth(25) { } // 25 = 1000 ms / 40 ms. public: typedef instance_map_type::iterator iterator; @@ -124,6 +125,8 @@ class AIPerServiceRequestQueue { static bool sQueueFull; // Set to true when sTotalQueued is still larger than zero after popping any queue. static bool sRequestStarvation; // Set to true when any queue was about to be popped when sTotalQueued was already zero. + AIAverage mHTTPBandwidth; // Keeps track on number of bytes received for this service in the past second. + public: void added_to_command_queue(void) { ++mQueuedCommands; } void removed_from_command_queue(void) { --mQueuedCommands; llassert(mQueuedCommands >= 0); } @@ -141,14 +144,17 @@ class AIPerServiceRequestQueue { S32 pipelined_requests(void) const { return mQueuedCommands + mQueuedRequests.size() + mAdded; } static S32 total_queued_size(void) { return sTotalQueued; } + AIAverage& bandwidth(void) { return mHTTPBandwidth; } + AIAverage const& bandwidth(void) const { return mHTTPBandwidth; } + // Returns true if curl can handle another request for this host. // Should return false if the maximum allowed HTTP bandwidth is reached, or when // the latency between request and actual delivery becomes too large. - static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth); + static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling); private: // Disallow copying. - AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) { } + AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) : mHTTPBandwidth(0) { } }; namespace AICurlPrivate { diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index d0fa202ca..5558e31a4 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -396,6 +396,9 @@ class BufferedCurlEasyRequest : public CurlEasyRequest { // Post-initialization, set the parent to pass the events to. void send_buffer_events_to(AIBufferedCurlEasyRequestEvents* target) { mBufferEventsTarget = target; } + // Called whenever new body data was (might be) received. Keeps track of the used HTTP bandwidth. + void update_body_bandwidth(void); + protected: // Events from this class. /*virtual*/ void received_HTTP_header(void); @@ -411,13 +414,14 @@ class BufferedCurlEasyRequest : public CurlEasyRequest { U32 mStatus; // HTTP status, decoded from the first header line. std::string mReason; // The "reason" from the same header line. U32 mRequestTransferedBytes; - U32 mResponseTransferedBytes; + size_t mTotalRawBytes; // Raw body data (still, possibly, compressed) received from the server so far. AIBufferedCurlEasyRequestEvents* mBufferEventsTarget; public: static LLChannelDescriptors const sChannels; // Channel object for mInput (channel out()) and mOutput (channel in()). static LLMutex sResponderCallbackMutex; // Locked while calling back any overridden ResponderBase::finished and/or accessing sShuttingDown. static bool sShuttingDown; // If true, no additional calls to ResponderBase::finished will be made anymore. + static AIAverage sHTTPBandwidth; // HTTP bandwidth usage of all services combined. private: // This class may only be created by constructing a ThreadSafeBufferedCurlEasyRequest. diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index a586ece81..a7b0e4642 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -33,6 +33,7 @@ #include "aihttptimeoutpolicy.h" #include "aihttptimeout.h" #include "aicurlperservice.h" +#include "aiaverage.h" #include "lltimer.h" // ms_sleep, get_clock_count #include "llhttpstatuscodes.h" #include "llbuffer.h" @@ -1829,6 +1830,8 @@ void MultiHandle::check_msg_queue(void) void MultiHandle::finish_easy_request(AICurlEasyRequest const& easy_request, CURLcode result) { AICurlEasyRequest_wat curl_easy_request_w(*easy_request); + // Final body bandwidth update. + curl_easy_request_w->update_body_bandwidth(); // Store the result in the easy handle. curl_easy_request_w->storeResult(result); #ifdef CWDEBUG @@ -2023,10 +2026,10 @@ void BufferedCurlEasyRequest::processOutput(void) if (responseCode == HTTP_INTERNAL_ERROR_LOW_SPEED) { // Rewrite error to something understandable. - responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %u bytes). " + responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %lu bytes). " "To change these values, go to Advanced --> Debug Settings and change CurlTimeoutLowSpeedLimit and CurlTimeoutLowSpeedTime respectively.", mResponder->getURL().c_str(), mResponder->getHTTPTimeoutPolicy().getLowSpeedLimit(), mResponder->getHTTPTimeoutPolicy().getLowSpeedTime(), - mResponder->getName(), mResponseTransferedBytes); + mResponder->getName(), mTotalRawBytes); } setopt(CURLOPT_FRESH_CONNECT, TRUE); } @@ -2093,7 +2096,9 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_ // BufferedCurlEasyRequest::setBodyLimit is never called, so buffer_w->mBodyLimit is infinite. //S32 bytes = llmin(size * nmemb, buffer_w->mBodyLimit); buffer_w->mBodyLimit -= bytes; self_w->getOutput()->append(sChannels.in(), (U8 const*)data, bytes); - self_w->mResponseTransferedBytes += bytes; // Accumulate data received from the server. + // Update HTTP bandwith. + self_w->update_body_bandwidth(); + // Update timeout administration. if (self_w->httptimeout()->data_received(bytes)) // Update timeout administration. { // Transfer timed out. Return 0 which will abort with error CURLE_WRITE_ERROR. @@ -2102,6 +2107,25 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_ return bytes; } +void BufferedCurlEasyRequest::update_body_bandwidth(void) +{ + double size_download; // Total amount of raw bytes received so far (ie. still compressed, 'bytes' is uncompressed). + getinfo(CURLINFO_SIZE_DOWNLOAD, &size_download); + size_t total_raw_bytes = size_download; + size_t raw_bytes = total_raw_bytes - mTotalRawBytes; + mTotalRawBytes = total_raw_bytes; + // Note that in some cases (like HTTP_PARTIAL_CONTENT), the output of CURLINFO_SIZE_DOWNLOAD lags + // behind and will return 0 the first time, and the value of the previous chunk the next time. + // The last call from MultiHandle::finish_easy_request recorrects this, in that case. + if (raw_bytes > 0) + { + U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2; + AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*getPerServicePtr())->bandwidth()); + http_bandwidth.addData(raw_bytes, sTime_40ms); + sHTTPBandwidth.addData(raw_bytes, sTime_40ms); + } +} + //static size_t BufferedCurlEasyRequest::curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) { @@ -2189,6 +2213,11 @@ size_t BufferedCurlEasyRequest::curlHeaderCallback(char* data, size_t size, size self_w->httptimeout()->being_redirected(); } } + // Update HTTP bandwidth. + U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2; + AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*self_w->getPerServicePtr())->bandwidth()); + http_bandwidth.addData(header_len, sTime_40ms); + sHTTPBandwidth.addData(header_len, sTime_40ms); // Update timeout administration. This must be done after the status is already known. if (self_w->httptimeout()->data_received(header_len/*,*/ ASSERT_ONLY_COMMA(self_w->upload_error_status()))) { @@ -2533,6 +2562,14 @@ U32 getNumHTTPAdded(void) return AICurlPrivate::curlthread::MultiHandle::total_added_size(); } +size_t getHTTPBandwidth(void) +{ + using namespace AICurlPrivate; + + U64 const sTime_40ms = get_clock_count() * curlthread::HTTPTimeout::sClockWidth_40ms; + return BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms); +} + } // namespace AICurlInterface // Return true if we want at least one more HTTP request for this host. @@ -2560,7 +2597,7 @@ U32 getNumHTTPAdded(void) // running requests (in MultiHandle::mAddedEasyRequests)). // //static -bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth) +bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling) { using namespace AICurlPrivate; using namespace AICurlPrivate::curlthread; @@ -2587,17 +2624,21 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue // Check if it's ok to get a new request for this particular host and update the per-host threshold. + AIAverage* http_bandwidth_ptr; + // Atomic read max_pipelined_requests_per_service for the below calculations. S32 const max_pipelined_requests_per_service_cache = max_pipelined_requests_per_service; { - PerServiceRequestQueue_rat per_service_r(*per_service); - S32 const pipelined_requests_per_service = per_service_r->pipelined_requests(); + PerServiceRequestQueue_wat per_service_w(*per_service); + S32 const pipelined_requests_per_service = per_service_w->pipelined_requests(); reject = pipelined_requests_per_service >= max_pipelined_requests_per_service_cache; equal = pipelined_requests_per_service == max_pipelined_requests_per_service_cache; - increment_threshold = per_service_r->mRequestStarvation; - decrement_threshold = per_service_r->mQueueFull && !per_service_r->mQueueEmpty; + increment_threshold = per_service_w->mRequestStarvation; + decrement_threshold = per_service_w->mQueueFull && !per_service_w->mQueueEmpty; // Reset flags. - per_service_r->mQueueFull = per_service_r->mQueueEmpty = per_service_r->mRequestStarvation = false; + per_service_w->mQueueFull = per_service_w->mQueueEmpty = per_service_w->mRequestStarvation = false; + // Grab per service bandwidth object. + http_bandwidth_ptr = &per_service_w->bandwidth(); } if (decrement_threshold) { @@ -2621,10 +2662,69 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue return false; } - //AIFIXME: better bandwidth check here. - if (too_much_bandwidth) + if (!no_bandwidth_throttling) { - return false; // wait + // Throttle on bandwidth usage. + + static size_t throttle_fraction = 1024; // A value between 0 and 1024: each service is throttled when it uses more than max_bandwidth * (throttle_fraction/1024) bandwidth. + static AIAverage fraction(25); // Average over 25 * 40ms = 1 second. + static U64 last_sTime_40ms = 0; + + // Truncate the sums to the last second, and get their value. + U64 const sTime_40ms = get_clock_count() * HTTPTimeout::sClockWidth_40ms; // Time in 40ms units. + size_t const max_bandwidth = 125.f * max_kbps; // Convert kbps to bytes per second. + size_t const total_bandwidth = BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms); // Bytes received in the past second. + size_t const service_bandwidth = http_bandwidth_ptr->truncateData(sTime_40ms); // Idem for just this service. + if (sTime_40ms > last_sTime_40ms) + { + // Only add throttle_fraction once every 40 ms at most. + // It's ok to ignore other values in the same 40 ms because the value only changes on the scale of 1 second. + fraction.addData(throttle_fraction, sTime_40ms); + last_sTime_40ms = sTime_40ms; + } + double fraction_avg = fraction.getAverage(1024.0); // throttle_fraction averaged over the past second, or 1024 if there is no data. + + // Adjust throttle_fraction based on total bandwidth usage. + if (total_bandwidth == 0) + throttle_fraction = 1024; + else + { + // This is the main formula. It can be made plausible by assuming + // an equilibrium where total_bandwidth == max_bandwidth and + // thus throttle_fraction == fraction_avg for more than a second. + // + // Then, more bandwidth is being used (for example because another + // service starts downloading). Assuming that all services that use + // a significant portion of the bandwidth, the new service included, + // must be throttled (all using the same bandwidth; note that the + // new service is immediately throttled at the same value), then + // the limit should be reduced linear with the fraction: + // max_bandwidth / total_bandwidth. + // + // For example, let max_bandwidth be 1. Let there be two throttled + // services, each using 0.5 (fraction_avg = 1024/2). Lets the new + // service use what it can: also 0.5 - then without reduction the + // total_bandwidth would become 1.5, and throttle_fraction would + // become (1024/2) * 1/1.5 = 1024/3: from 2 to 3 services. + // + // In reality, total_bandwidth would rise linear from 1.0 to 1.5 in + // one second if the throttle fraction wasn't changed. However it is + // changed here. The end result is that any change more or less + // linear fades away in one second. + throttle_fraction = fraction_avg * max_bandwidth / total_bandwidth; + } + if (throttle_fraction > 1024) + throttle_fraction = 1024; + if (total_bandwidth > max_bandwidth) + { + throttle_fraction *= 0.95; + } + + // Throttle this service if it uses too much bandwidth. + if (service_bandwidth > (max_bandwidth * throttle_fraction / 1024)) + { + return false; // wait + } } // Check if it's ok to get a new request based on the total number of requests and increment the threshold if appropriate. diff --git a/indra/llmessage/aihttptimeout.cpp b/indra/llmessage/aihttptimeout.cpp index 1f65b94c0..22dbb1cfd 100644 --- a/indra/llmessage/aihttptimeout.cpp +++ b/indra/llmessage/aihttptimeout.cpp @@ -97,8 +97,9 @@ namespace curlthread { // HTTPTimeout //static -F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency(); // Time between two clock ticks, in 10ms units. -U64 HTTPTimeout::sTime_10ms; // Time in 10ms units, set once per select() exit. +F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency(); // Time between two clock ticks, in 10ms units. +F64 const HTTPTimeout::sClockWidth_40ms = HTTPTimeout::sClockWidth_10ms * 0.25; // Time between two clock ticks, in 40ms units. +U64 HTTPTimeout::sTime_10ms; // Time in 10ms units, set once per select() exit. // CURL-THREAD // This is called when body data was sent to the server socket. diff --git a/indra/llmessage/aihttptimeout.h b/indra/llmessage/aihttptimeout.h index fc6db9919..1af96c9c7 100644 --- a/indra/llmessage/aihttptimeout.h +++ b/indra/llmessage/aihttptimeout.h @@ -89,6 +89,7 @@ class HTTPTimeout : public LLRefCount { U64 mStalled; // The time (sTime_10ms) at which this transaction is considered to be stalling if nothing is transfered anymore. public: static F64 const sClockWidth_10ms; // Time between two clock ticks in 10 ms units. + static F64 const sClockWidth_40ms; // Time between two clock ticks in 40 ms units. static U64 sTime_10ms; // Time since the epoch in 10 ms units. #if defined(CWDEBUG) || defined(DEBUG_CURLIO) ThreadSafeBufferedCurlEasyRequest* mLockObj; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index 8f60d2a7d..814d407d0 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -50,7 +50,6 @@ #include "lldarray.h" #include "lldir.h" #include "llerror.h" -#include "llerrorlegacy.h" #include "llfasttimer.h" #include "llhttpclient.h" #include "llhttpnodeadapter.h" diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 7cefb59c6..fb18720b3 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4921,7 +4921,18 @@ This should be as low as possible, but too low may break functionality Value -1 - DebugStatModeTexture + DebugStatModeHTTPTexture + + Comment + Mode of stat in Statistics floater + Persist + 1 + Type + S32 + Value + -1 + + DebugStatModeUDPTexture Comment Mode of stat in Statistics floater diff --git a/indra/newview/hippogridmanager.cpp b/indra/newview/hippogridmanager.cpp index 446d28282..ce389ff05 100644 --- a/indra/newview/hippogridmanager.cpp +++ b/indra/newview/hippogridmanager.cpp @@ -678,13 +678,6 @@ HippoGridInfo* HippoGridManager::getGrid(const std::string& grid) const } } - -HippoGridInfo* HippoGridManager::getConnectedGrid() const -{ - return (mConnectedGrid)? mConnectedGrid: getCurrentGrid(); -} - - HippoGridInfo* HippoGridManager::getCurrentGrid() const { HippoGridInfo* grid = getGrid(mCurrentGrid); diff --git a/indra/newview/hippogridmanager.h b/indra/newview/hippogridmanager.h index 8d84ba816..92ddd95b4 100644 --- a/indra/newview/hippogridmanager.h +++ b/indra/newview/hippogridmanager.h @@ -157,7 +157,8 @@ public: void discardAndReload(); HippoGridInfo* getGrid(const std::string& grid) const; - HippoGridInfo* getConnectedGrid() const; + HippoGridInfo* getConnectedGrid() const { return mConnectedGrid ? mConnectedGrid : getCurrentGrid(); } + HippoGridInfo* getCurrentGrid() const; const std::string& getDefaultGridNick() const; const std::string& getCurrentGridNick() const; diff --git a/indra/newview/llfloaterstats.cpp b/indra/newview/llfloaterstats.cpp index a60619882..a4311897e 100644 --- a/indra/newview/llfloaterstats.cpp +++ b/indra/newview/llfloaterstats.cpp @@ -238,25 +238,52 @@ void LLFloaterStats::buildStats() // Network statistics LLStatView *net_statviewp = stat_viewp->addStatView("network stat view", "Network", "OpenDebugStatNet", rect); - stat_barp = net_statviewp->addStat("Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn"); + stat_barp = net_statviewp->addStat("UDP Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn"); stat_barp->setUnitLabel("/sec"); - stat_barp = net_statviewp->addStat("Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut"); + stat_barp = net_statviewp->addStat("UDP Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut"); stat_barp->setUnitLabel("/sec"); + stat_barp = net_statviewp->addStat("HTTP Textures", &(LLViewerStats::getInstance()->mHTTPTextureKBitStat), "DebugStatModeHTTPTexture"); + stat_barp->setUnitLabel(" kbps"); + stat_barp->mMinBar = 0.f; + stat_barp->mMaxBar = gSavedSettings.getF32("HTTPThrottleBandwidth") * 2; // Times two because we'll have over shoots. + stat_barp->mTickSpacing = 1.f; + while (stat_barp->mTickSpacing < stat_barp->mMaxBar / 8) + stat_barp->mTickSpacing *= 2.f; + stat_barp->mLabelSpacing = 2 * stat_barp->mTickSpacing; + stat_barp->mPerSec = FALSE; + stat_barp->mDisplayMean = FALSE; + + stat_barp = net_statviewp->addStat("UDP Textures", &(LLViewerStats::getInstance()->mUDPTextureKBitStat), "DebugStatModeUDPTexture"); + stat_barp->setUnitLabel(" kbps"); + stat_barp->mMinBar = 0.f; + stat_barp->mMaxBar = 1024.f; + stat_barp->mTickSpacing = 128.f; + stat_barp->mLabelSpacing = 256.f; + stat_barp = net_statviewp->addStat("Objects", &(LLViewerStats::getInstance()->mObjectKBitStat), "DebugStatModeObjects"); stat_barp->setUnitLabel(" kbps"); + stat_barp->mMinBar = 0.f; + stat_barp->mMaxBar = 1024.f; + stat_barp->mTickSpacing = 128.f; + stat_barp->mLabelSpacing = 256.f; - stat_barp = net_statviewp->addStat("Texture", &(LLViewerStats::getInstance()->mTextureKBitStat), "DebugStatModeTexture"); + stat_barp = net_statviewp->addStat("Assets (UDP)", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset"); stat_barp->setUnitLabel(" kbps"); + stat_barp->mMinBar = 0.f; + stat_barp->mMaxBar = 1024.f; + stat_barp->mTickSpacing = 128.f; + stat_barp->mLabelSpacing = 256.f; - stat_barp = net_statviewp->addStat("Asset", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset"); + stat_barp = net_statviewp->addStat("Layers (UDP)", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers"); stat_barp->setUnitLabel(" kbps"); + stat_barp->mMinBar = 0.f; + stat_barp->mMaxBar = 1024.f; + stat_barp->mTickSpacing = 128.f; + stat_barp->mLabelSpacing = 256.f; - stat_barp = net_statviewp->addStat("Layers", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers"); - stat_barp->setUnitLabel(" kbps"); - - stat_barp = net_statviewp->addStat("Actual In", &(LLViewerStats::getInstance()->mActualInKBitStat), + stat_barp = net_statviewp->addStat("Actual In (UDP)", &(LLViewerStats::getInstance()->mActualInKBitStat), "DebugStatModeActualIn", TRUE, FALSE); stat_barp->setUnitLabel(" kbps"); stat_barp->mMinBar = 0.f; @@ -264,7 +291,7 @@ void LLFloaterStats::buildStats() stat_barp->mTickSpacing = 128.f; stat_barp->mLabelSpacing = 256.f; - stat_barp = net_statviewp->addStat("Actual Out", &(LLViewerStats::getInstance()->mActualOutKBitStat), + stat_barp = net_statviewp->addStat("Actual Out (UDP)", &(LLViewerStats::getInstance()->mActualOutKBitStat), "DebugStatModeActualOut", TRUE, FALSE); stat_barp->setUnitLabel(" kbps"); stat_barp->mMinBar = 0.f; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 24bed73be..cd9f44904 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -36,7 +36,7 @@ #include "lltexturefetch.h" -#include "llcurl.h" +#include "aicurl.h" #include "lldir.h" #include "llhttpclient.h" #include "llhttpstatuscodes.h" @@ -1273,7 +1273,8 @@ bool LLTextureFetchWorker::doWork(S32 param) // Let AICurl decide if we can process more HTTP requests at the moment or not. static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); - if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, mFetcher->getTextureBandwidth() > throttle_bandwidth)) + bool const no_bandwidth_throttling = gHippoGridManager->getConnectedGrid()->isAvination(); + if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, throttle_bandwidth, no_bandwidth_throttling)) { return false ; //wait. } @@ -1290,16 +1291,8 @@ bool LLTextureFetchWorker::doWork(S32 param) mLoaded = FALSE; mGetStatus = 0; mGetReason.clear(); - // Note: comparing mFetcher->getTextureBandwidth() with throttle_bandwidth is a bit like - // comparing apples and oranges, but it's only debug output. The first is the averaged - // bandwidth used for the body of successfully downloaded textures, averaged over roughtly - // 10 seconds, in kbits/s. The latter is the limit of the actual http curl downloads, - // including header and failures for anything (not just textures), averaged over the last - // second, also in kbits/s. - static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset << " Bytes: " << mRequestedSize - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << throttle_bandwidth << LL_ENDL; setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; @@ -2041,7 +2034,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mBadPacketCount(0), mTextureCache(cache), mImageDecodeThread(imagedecodethread), - mTextureBandwidth(0), mHTTPTextureBits(0), mTotalHTTPRequests(0), mQAMode(qa_mode), diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 7d2d86135..d7cc94dcd 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -79,9 +79,6 @@ public: bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data); bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data); - void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; } - F32 getTextureBandwidth() { return mTextureBandwidth; } - // Debug BOOL isFromLocalCache(const LLUUID& id); S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p, @@ -164,7 +161,6 @@ private: queue_t mHTTPTextureQueue; typedef std::map > cancel_queue_t; cancel_queue_t mCancelQueue; - F32 mTextureBandwidth; LLTextureInfo mTextureInfo; U32 mHTTPTextureBits; diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 9e644d51a..944870e32 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -70,6 +70,7 @@ namespace AICurlInterface { U32 getNumHTTPQueued(void); U32 getNumHTTPAdded(void); U32 getNumHTTPRunning(void); + size_t getHTTPBandwidth(void); } // namespace AICurlInterface //////////////////////////////////////////////////////////////////////////// @@ -620,17 +621,15 @@ void LLGLTexMemBar::draw() text_color, LLFontGL::LEFT, LLFontGL::TOP); left += LLFontGL::getFontMonospace()->getWidth(text); - // This bandwidth is averaged over roughly 10 seconds (in kbps) and therefore pretty inaccurate. - // Also, it only takes into account actual texture data (not headers etc). But all it is used for - // is for the color of some text in the texture console, so I guess it doesn't matter. - F32 bandwidth = LLAppViewer::getTextureFetch()->getTextureBandwidth(); + // This bandwidth is averaged over 1 seconds (in kbps). + F32 bandwidth = AICurlInterface::getHTTPBandwidth() / 125.f; // Convert from bytes/s to kbps. // This is the maximum bandwidth allowed for curl transactions (of any type and averaged per second), // that is actually used to limit the number of HTTP texture requests (and only those). // Comparing that with 'bandwidth' is a bit like comparing apples and oranges, but again... who really cares. - F32 max_bandwidth = gSavedSettings.getF32("HTTPThrottleBandwidth"); + static const LLCachedControl max_bandwidth("HTTPThrottleBandwidth", 2000); color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth*.75f ? LLColor4::yellow : text_color; color[VALPHA] = text_color[VALPHA]; - text = llformat("BW:%.0f/%.0f",bandwidth, max_bandwidth); + text = llformat("BW:%.0f/%.0f", bandwidth, max_bandwidth.get()); LLFontGL::getFontMonospace()->renderUTF8(text, 0, left, v_offset + line_height*2, color, LLFontGL::LEFT, LLFontGL::TOP); diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 2d20f8de3..4728f0ffd 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -67,6 +67,10 @@ class AIHTTPTimeoutPolicy; extern AIHTTPTimeoutPolicy viewerStatsResponder_timeout; +namespace AICurlInterface { +size_t getHTTPBandwidth(void); +} + class StatAttributes { public: @@ -210,7 +214,8 @@ LLViewerStats::LLViewerStats() : mLayersKBitStat("layerskbitstat"), mObjectKBitStat("objectkbitstat"), mAssetKBitStat("assetkbitstat"), - mTextureKBitStat("texturekbitstat"), + mHTTPTextureKBitStat("httptexturekbitstat"), + mUDPTextureKBitStat("udptexturekbitstat"), mMallocStat("mallocstat"), mVFSPendingOperations("vfspendingoperations"), mObjectsDrawnStat("objectsdrawnstat"), @@ -300,7 +305,8 @@ void LLViewerStats::resetStats() stats.mKBitStat.reset(); stats.mLayersKBitStat.reset(); stats.mObjectKBitStat.reset(); - stats.mTextureKBitStat.reset(); + stats.mHTTPTextureKBitStat.reset(); + stats.mUDPTextureKBitStat.reset(); stats.mVFSPendingOperations.reset(); stats.mAssetKBitStat.reset(); stats.mPacketsInStat.reset(); @@ -673,13 +679,13 @@ void update_statistics() // Only update texture stats periodically so that they are less noisy { - static const F32 texture_stats_freq = 10.f; + static const F32 texture_stats_freq = 0.25f; static LLFrameTimer texture_stats_timer; if (texture_stats_timer.getElapsedTimeF32() >= texture_stats_freq) { - stats.mTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f); + stats.mHTTPTextureKBitStat.addValue(AICurlInterface::getHTTPBandwidth()/125.f); + stats.mUDPTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f); stats.mTexturePacketsStat.addValue(LLViewerTextureList::sTexturePackets); - LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerTextureList::sTextureBits/1024.f/texture_stats_timer.getElapsedTimeF32()); gTotalTextureBytes += LLViewerTextureList::sTextureBits / 8; LLViewerTextureList::sTextureBits = 0; LLViewerTextureList::sTexturePackets = 0; diff --git a/indra/newview/llviewerstats.h b/indra/newview/llviewerstats.h index f7c7c0c1c..f6300d1e2 100644 --- a/indra/newview/llviewerstats.h +++ b/indra/newview/llviewerstats.h @@ -43,7 +43,8 @@ public: mLayersKBitStat, mObjectKBitStat, mAssetKBitStat, - mTextureKBitStat, + mHTTPTextureKBitStat, + mUDPTextureKBitStat, mVFSPendingOperations, mObjectsDrawnStat, mObjectsCulledStat, diff --git a/indra/newview/llviewertexturelist.cpp b/indra/newview/llviewertexturelist.cpp index 0307e32a2..80efedbed 100644 --- a/indra/newview/llviewertexturelist.cpp +++ b/indra/newview/llviewertexturelist.cpp @@ -687,8 +687,6 @@ void LLViewerTextureList::updateImages(F32 max_time) } cleared = FALSE; - LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerStats::getInstance()->mTextureKBitStat.getMeanPerSec()); - S32 global_raw_memory; { global_raw_memory = *AIAccess(LLImageRaw::sGlobalRawMemory);