Singularity-Viewer/SingularityViewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'aleric/master'

Browse Source
This commit is contained in:
Latif Khalifa
2013-04-25 07:26:45 +02:00
parent ec8212f472 e422a89de8
commit dc4d458e87
22 changed files with 418 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
indra/llmessage/CMakeLists.txt
View File

@@ -22,6 +22,7 @@ include_directories(
)
set(llmessage_SOURCE_FILES
aiaverage.cpp
aicurl.cpp
aicurleasyrequeststatemachine.cpp
aicurlperservice.cpp
@@ -109,6 +110,7 @@ set(llmessage_SOURCE_FILES
set(llmessage_HEADER_FILES
CMakeLists.txt
aiaverage.h
aicurl.h
aicurleasyrequeststatemachine.h
aicurlperservice.h

81
indra/llmessage/aiaverage.cpp Normal file
View File

@@ -0,0 +1,81 @@
/**
* @file aiaverage.cpp
* @brief Implementation of AIAverage
*
* Copyright (c) 2013, Aleric Inglewood.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* There are special exceptions to the terms and conditions of the GPL as
* it is applied to this Source Code. View the full text of the exception
* in the file doc/FLOSS-exception.txt in this software distribution.
*
* CHANGELOG
* and additional copyright holders.
*
* 11/04/2013
* Initial version, written by Aleric Inglewood @ SL
*/
#include "sys.h"
#include "aiaverage.h"
#include "llerror.h" // llassert
void AIAverage::cleanup(U64 clock_tick)
{
// This expression can fail because the curl thread caches the time in
// sTime_10ms for the duration of an entire loop. Therefore, the time can
// go into the next 40ms and a texture fetch worker thread might call
// cleanup() with that time, setting mCurrentClock to a value (one)
// larger than sTime_10ms / 4. Next, the curl thread can continue to call
// this function with the smaller value; in that case just add the new
// data to the current bucket.
//
// Or, this is just the one-time initialization that happens the first
// time this is called. In that case initialize just mCurrentClock:
// the rest is already initialized upon construction.
if (LL_LIKELY(clock_tick > mCurrentClock))
{
// Advance to the next bucket.
++mCurrentBucket;
mCurrentBucket %= mNrOfBuckets;
// Initialize the new bucket.
mData[mCurrentBucket].time = clock_tick;
// Clean up old buckets.
U64 old_time = clock_tick - mNrOfBuckets;
if (LL_UNLIKELY(mTail == mCurrentBucket) || // Extremely unlikely: only happens when data was added EVERY clock tick for the past mNrOfBuckets clock ticks.
mData[mTail].time <= old_time)
{
do
{
mSum -= mData[mTail].sum;
mN -= mData[mTail].n;
mData[mTail].sum = 0;
mData[mTail].n = 0;
++mTail;
if (LL_UNLIKELY(mTail == mNrOfBuckets))
{
mTail = 0;
}
}
while (mData[mTail].time <= old_time);
}
// This was set to zero when mTail passed this point (likely not this call, but a few calls ago).
llassert(mData[mCurrentBucket].sum == 0 &&
mData[mCurrentBucket].n == 0);
}
mCurrentClock = clock_tick;
return;
}

108
indra/llmessage/aiaverage.h Normal file
View File

@@ -0,0 +1,108 @@
/**
* @file aiaverage.h
* @brief Definition of class AIAverage
*
* Copyright (c) 2013, Aleric Inglewood.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* There are special exceptions to the terms and conditions of the GPL as
* it is applied to this Source Code. View the full text of the exception
* in the file doc/FLOSS-exception.txt in this software distribution.
*
* CHANGELOG
* and additional copyright holders.
*
* 11/04/2013
* Initial version, written by Aleric Inglewood @ SL
*/
#ifndef AIAVERAGE_H
#define AIAVERAGE_H
#include "llpreprocessor.h"
#include "stdtypes.h" // U32, U64
#include "llthread.h" // LLMutex
#include <cstddef> // size_t
#include <vector>
class AIAverage {
private:
struct Data {
U32 sum; // Accumulated sum of the 'n' passed to operator()(size_t n, U64 clock_tick) with clock_tick == time.
U32 n; // The number of calls to operator().
U64 time; // The clock_tick as passed to operator()(size_t n, U64 clock_tick) that sum corresponds to.
};
U64 mCurrentClock; // The current (last) time that operator() was called with, or -1 when not initialized.
int mTail; // The oldest bucket with still valid data.
int mCurrentBucket; // The bucket that corresponds to mCurrentClock.
size_t mSum; // The sum of all the 'n' passed to operator()(size_t n, U64 clock_tick) for all passed mNrOfBuckets time units.
U32 mN; // The number of calls to operator().
int const mNrOfBuckets; // Size of mData.
std::vector<Data> mData; // The buckets.
LLMutex mLock; // Mutex for all of the above data.
public:
AIAverage(int number_of_buckets) : mCurrentClock(~(U64)0), mTail(0), mCurrentBucket(0), mSum(0), mN(0), mNrOfBuckets(number_of_buckets), mData(number_of_buckets)
{
// Fill mData with all zeroes (much faster than adding a constructor to Data).
std::memset(&*mData.begin(), 0, number_of_buckets * sizeof(Data));
}
size_t addData(U32 n, U64 clock_tick)
{
DoutEntering(dc::curl, "AIAverage::addData(" << n << ", " << clock_tick << ")");
mLock.lock();
if (LL_UNLIKELY(clock_tick != mCurrentClock))
{
cleanup(clock_tick);
}
mSum += n;
mN += 1;
mData[mCurrentBucket].sum += n;
mData[mCurrentBucket].n += 1;
size_t sum = mSum;
mLock.unlock();
Dout(dc::curl, "Current sum: " << sum << ", average: " << (sum / mN));
return sum;
}
size_t truncateData(U64 clock_tick)
{
mLock.lock();
if (clock_tick != mCurrentClock)
{
cleanup(clock_tick);
}
size_t sum = mSum;
mLock.unlock();
return sum;
}
double getAverage(double avg_no_data)
{
mLock.lock();
double avg = mSum;
llassert(mN != 0 || mSum == 0);
if (LL_UNLIKELY(mN == 0))
avg = avg_no_data;
else
avg /= mN;
mLock.unlock();
return avg;
}
private:
void cleanup(U64 clock_tick);
};
#endif // AIAVERAGE

5
indra/llmessage/aicurl.cpp
View File

@@ -1267,8 +1267,9 @@ static int const HTTP_REDIRECTS_DEFAULT = 10;
LLChannelDescriptors const BufferedCurlEasyRequest::sChannels;
LLMutex BufferedCurlEasyRequest::sResponderCallbackMutex;
bool BufferedCurlEasyRequest::sShuttingDown = false;
AIAverage BufferedCurlEasyRequest::sHTTPBandwidth(25);
BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mResponseTransferedBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER)
BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mTotalRawBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER)
{
AICurlInterface::Stats::BufferedCurlEasyRequest_count++;
}
@@ -1332,7 +1333,7 @@ void BufferedCurlEasyRequest::resetState(void)
mOutput.reset();
mInput.reset();
mRequestTransferedBytes = 0;
mResponseTransferedBytes = 0;
mTotalRawBytes = 0;
mBufferEventsTarget = NULL;
mStatus = HTTP_INTERNAL_ERROR_OTHER;
}

12
indra/llmessage/aicurlperservice.h
View File

@@ -45,6 +45,7 @@
#include <map>
#include <boost/intrusive_ptr.hpp>
#include "aithreadsafe.h"
#include "aiaverage.h"
class AICurlEasyRequest;
class AIPerServiceRequestQueue;
@@ -89,7 +90,7 @@ class AIPerServiceRequestQueue {
static threadsafe_instance_map_type sInstanceMap; // Map of AIPerServiceRequestQueue instances with the hostname as key.
friend class AIThreadSafeSimpleDC<AIPerServiceRequestQueue>; //threadsafe_PerServiceRequestQueue
AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { }
AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false), mHTTPBandwidth(25) { } // 25 = 1000 ms / 40 ms.
public:
typedef instance_map_type::iterator iterator;
@@ -124,6 +125,8 @@ class AIPerServiceRequestQueue {
static bool sQueueFull; // Set to true when sTotalQueued is still larger than zero after popping any queue.
static bool sRequestStarvation; // Set to true when any queue was about to be popped when sTotalQueued was already zero.
AIAverage mHTTPBandwidth; // Keeps track on number of bytes received for this service in the past second.
public:
void added_to_command_queue(void) { ++mQueuedCommands; }
void removed_from_command_queue(void) { --mQueuedCommands; llassert(mQueuedCommands >= 0); }
@@ -141,14 +144,17 @@ class AIPerServiceRequestQueue {
S32 pipelined_requests(void) const { return mQueuedCommands + mQueuedRequests.size() + mAdded; }
static S32 total_queued_size(void) { return sTotalQueued; }
AIAverage& bandwidth(void) { return mHTTPBandwidth; }
AIAverage const& bandwidth(void) const { return mHTTPBandwidth; }
// Returns true if curl can handle another request for this host.
// Should return false if the maximum allowed HTTP bandwidth is reached, or when
// the latency between request and actual delivery becomes too large.
static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth);
static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling);
private:
// Disallow copying.
AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) { }
AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) : mHTTPBandwidth(0) { }
};
namespace AICurlPrivate {

6
indra/llmessage/aicurlprivate.h
View File

@@ -396,6 +396,9 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
// Post-initialization, set the parent to pass the events to.
void send_buffer_events_to(AIBufferedCurlEasyRequestEvents* target) { mBufferEventsTarget = target; }
// Called whenever new body data was (might be) received. Keeps track of the used HTTP bandwidth.
void update_body_bandwidth(void);
protected:
// Events from this class.
/*virtual*/ void received_HTTP_header(void);
@@ -411,13 +414,14 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
U32 mStatus; // HTTP status, decoded from the first header line.
std::string mReason; // The "reason" from the same header line.
U32 mRequestTransferedBytes;
U32 mResponseTransferedBytes;
size_t mTotalRawBytes; // Raw body data (still, possibly, compressed) received from the server so far.
AIBufferedCurlEasyRequestEvents* mBufferEventsTarget;
public:
static LLChannelDescriptors const sChannels; // Channel object for mInput (channel out()) and mOutput (channel in()).
static LLMutex sResponderCallbackMutex; // Locked while calling back any overridden ResponderBase::finished and/or accessing sShuttingDown.
static bool sShuttingDown; // If true, no additional calls to ResponderBase::finished will be made anymore.
static AIAverage sHTTPBandwidth; // HTTP bandwidth usage of all services combined.
private:
// This class may only be created by constructing a ThreadSafeBufferedCurlEasyRequest.

124
indra/llmessage/aicurlthread.cpp
View File

@@ -33,6 +33,7 @@
#include "aihttptimeoutpolicy.h"
#include "aihttptimeout.h"
#include "aicurlperservice.h"
#include "aiaverage.h"
#include "lltimer.h" // ms_sleep, get_clock_count
#include "llhttpstatuscodes.h"
#include "llbuffer.h"
@@ -1829,6 +1830,8 @@ void MultiHandle::check_msg_queue(void)
void MultiHandle::finish_easy_request(AICurlEasyRequest const& easy_request, CURLcode result)
{
AICurlEasyRequest_wat curl_easy_request_w(*easy_request);
// Final body bandwidth update.
curl_easy_request_w->update_body_bandwidth();
// Store the result in the easy handle.
curl_easy_request_w->storeResult(result);
#ifdef CWDEBUG
@@ -2023,10 +2026,10 @@ void BufferedCurlEasyRequest::processOutput(void)
if (responseCode == HTTP_INTERNAL_ERROR_LOW_SPEED)
{
// Rewrite error to something understandable.
responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %u bytes). "
responseReason = llformat("Connection to \"%s\" stalled: download speed dropped below %u bytes/s for %u seconds (up till that point, %s received a total of %lu bytes). "
"To change these values, go to Advanced --> Debug Settings and change CurlTimeoutLowSpeedLimit and CurlTimeoutLowSpeedTime respectively.",
mResponder->getURL().c_str(), mResponder->getHTTPTimeoutPolicy().getLowSpeedLimit(), mResponder->getHTTPTimeoutPolicy().getLowSpeedTime(),
mResponder->getName(), mResponseTransferedBytes);
mResponder->getName(), mTotalRawBytes);
}
setopt(CURLOPT_FRESH_CONNECT, TRUE);
}
@@ -2093,7 +2096,9 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_
// BufferedCurlEasyRequest::setBodyLimit is never called, so buffer_w->mBodyLimit is infinite.
//S32 bytes = llmin(size * nmemb, buffer_w->mBodyLimit); buffer_w->mBodyLimit -= bytes;
self_w->getOutput()->append(sChannels.in(), (U8 const*)data, bytes);
self_w->mResponseTransferedBytes += bytes; // Accumulate data received from the server.
// Update HTTP bandwith.
self_w->update_body_bandwidth();
// Update timeout administration.
if (self_w->httptimeout()->data_received(bytes)) // Update timeout administration.
{
// Transfer timed out. Return 0 which will abort with error CURLE_WRITE_ERROR.
@@ -2102,6 +2107,25 @@ size_t BufferedCurlEasyRequest::curlWriteCallback(char* data, size_t size, size_
return bytes;
}
void BufferedCurlEasyRequest::update_body_bandwidth(void)
{
double size_download; // Total amount of raw bytes received so far (ie. still compressed, 'bytes' is uncompressed).
getinfo(CURLINFO_SIZE_DOWNLOAD, &size_download);
size_t total_raw_bytes = size_download;
size_t raw_bytes = total_raw_bytes - mTotalRawBytes;
mTotalRawBytes = total_raw_bytes;
// Note that in some cases (like HTTP_PARTIAL_CONTENT), the output of CURLINFO_SIZE_DOWNLOAD lags
// behind and will return 0 the first time, and the value of the previous chunk the next time.
// The last call from MultiHandle::finish_easy_request recorrects this, in that case.
if (raw_bytes > 0)
{
U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2;
AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*getPerServicePtr())->bandwidth());
http_bandwidth.addData(raw_bytes, sTime_40ms);
sHTTPBandwidth.addData(raw_bytes, sTime_40ms);
}
}
//static
size_t BufferedCurlEasyRequest::curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data)
{
@@ -2189,6 +2213,11 @@ size_t BufferedCurlEasyRequest::curlHeaderCallback(char* data, size_t size, size
self_w->httptimeout()->being_redirected();
}
}
// Update HTTP bandwidth.
U64 const sTime_40ms = curlthread::HTTPTimeout::sTime_10ms >> 2;
AIAverage& http_bandwidth(PerServiceRequestQueue_wat(*self_w->getPerServicePtr())->bandwidth());
http_bandwidth.addData(header_len, sTime_40ms);
sHTTPBandwidth.addData(header_len, sTime_40ms);
// Update timeout administration. This must be done after the status is already known.
if (self_w->httptimeout()->data_received(header_len/*,*/ ASSERT_ONLY_COMMA(self_w->upload_error_status())))
{
@@ -2533,6 +2562,14 @@ U32 getNumHTTPAdded(void)
return AICurlPrivate::curlthread::MultiHandle::total_added_size();
}
size_t getHTTPBandwidth(void)
{
using namespace AICurlPrivate;
U64 const sTime_40ms = get_clock_count() * curlthread::HTTPTimeout::sClockWidth_40ms;
return BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms);
}
} // namespace AICurlInterface
// Return true if we want at least one more HTTP request for this host.
@@ -2560,7 +2597,7 @@ U32 getNumHTTPAdded(void)
// running requests (in MultiHandle::mAddedEasyRequests)).
//
//static
bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth)
bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, F32 max_kbps, bool no_bandwidth_throttling)
{
using namespace AICurlPrivate;
using namespace AICurlPrivate::curlthread;
@@ -2587,17 +2624,21 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue
// Check if it's ok to get a new request for this particular host and update the per-host threshold.
AIAverage* http_bandwidth_ptr;
// Atomic read max_pipelined_requests_per_service for the below calculations.
S32 const max_pipelined_requests_per_service_cache = max_pipelined_requests_per_service;
{
PerServiceRequestQueue_rat per_service_r(*per_service);
S32 const pipelined_requests_per_service = per_service_r->pipelined_requests();
PerServiceRequestQueue_wat per_service_w(*per_service);
S32 const pipelined_requests_per_service = per_service_w->pipelined_requests();
reject = pipelined_requests_per_service >= max_pipelined_requests_per_service_cache;
equal = pipelined_requests_per_service == max_pipelined_requests_per_service_cache;
increment_threshold = per_service_r->mRequestStarvation;
decrement_threshold = per_service_r->mQueueFull && !per_service_r->mQueueEmpty;
increment_threshold = per_service_w->mRequestStarvation;
decrement_threshold = per_service_w->mQueueFull && !per_service_w->mQueueEmpty;
// Reset flags.
per_service_r->mQueueFull = per_service_r->mQueueEmpty = per_service_r->mRequestStarvation = false;
per_service_w->mQueueFull = per_service_w->mQueueEmpty = per_service_w->mRequestStarvation = false;
// Grab per service bandwidth object.
http_bandwidth_ptr = &per_service_w->bandwidth();
}
if (decrement_threshold)
{
@@ -2621,10 +2662,69 @@ bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueue
return false;
}
//AIFIXME: better bandwidth check here.
if (too_much_bandwidth)
if (!no_bandwidth_throttling)
{
return false; // wait
// Throttle on bandwidth usage.
static size_t throttle_fraction = 1024; // A value between 0 and 1024: each service is throttled when it uses more than max_bandwidth * (throttle_fraction/1024) bandwidth.
static AIAverage fraction(25); // Average over 25 * 40ms = 1 second.
static U64 last_sTime_40ms = 0;
// Truncate the sums to the last second, and get their value.
U64 const sTime_40ms = get_clock_count() * HTTPTimeout::sClockWidth_40ms; // Time in 40ms units.
size_t const max_bandwidth = 125.f * max_kbps; // Convert kbps to bytes per second.
size_t const total_bandwidth = BufferedCurlEasyRequest::sHTTPBandwidth.truncateData(sTime_40ms); // Bytes received in the past second.
size_t const service_bandwidth = http_bandwidth_ptr->truncateData(sTime_40ms); // Idem for just this service.
if (sTime_40ms > last_sTime_40ms)
{
// Only add throttle_fraction once every 40 ms at most.
// It's ok to ignore other values in the same 40 ms because the value only changes on the scale of 1 second.
fraction.addData(throttle_fraction, sTime_40ms);
last_sTime_40ms = sTime_40ms;
}
double fraction_avg = fraction.getAverage(1024.0); // throttle_fraction averaged over the past second, or 1024 if there is no data.
// Adjust throttle_fraction based on total bandwidth usage.
if (total_bandwidth == 0)
throttle_fraction = 1024;
else
{
// This is the main formula. It can be made plausible by assuming
// an equilibrium where total_bandwidth == max_bandwidth and
// thus throttle_fraction == fraction_avg for more than a second.
//
// Then, more bandwidth is being used (for example because another
// service starts downloading). Assuming that all services that use
// a significant portion of the bandwidth, the new service included,
// must be throttled (all using the same bandwidth; note that the
// new service is immediately throttled at the same value), then
// the limit should be reduced linear with the fraction:
// max_bandwidth / total_bandwidth.
//
// For example, let max_bandwidth be 1. Let there be two throttled
// services, each using 0.5 (fraction_avg = 1024/2). Lets the new
// service use what it can: also 0.5 - then without reduction the
// total_bandwidth would become 1.5, and throttle_fraction would
// become (1024/2) * 1/1.5 = 1024/3: from 2 to 3 services.
//
// In reality, total_bandwidth would rise linear from 1.0 to 1.5 in
// one second if the throttle fraction wasn't changed. However it is
// changed here. The end result is that any change more or less
// linear fades away in one second.
throttle_fraction = fraction_avg * max_bandwidth / total_bandwidth;
}
if (throttle_fraction > 1024)
throttle_fraction = 1024;
if (total_bandwidth > max_bandwidth)
{
throttle_fraction *= 0.95;
}
// Throttle this service if it uses too much bandwidth.
if (service_bandwidth > (max_bandwidth * throttle_fraction / 1024))
{
return false; // wait
}
}
// Check if it's ok to get a new request based on the total number of requests and increment the threshold if appropriate.

5
indra/llmessage/aihttptimeout.cpp
View File

@@ -97,8 +97,9 @@ namespace curlthread {
// HTTPTimeout
//static
F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency(); // Time between two clock ticks, in 10ms units.
U64 HTTPTimeout::sTime_10ms; // Time in 10ms units, set once per select() exit.
F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency(); // Time between two clock ticks, in 10ms units.
F64 const HTTPTimeout::sClockWidth_40ms = HTTPTimeout::sClockWidth_10ms * 0.25; // Time between two clock ticks, in 40ms units.
U64 HTTPTimeout::sTime_10ms; // Time in 10ms units, set once per select() exit.
// CURL-THREAD
// This is called when body data was sent to the server socket.

1
indra/llmessage/aihttptimeout.h
View File

@@ -89,6 +89,7 @@ class HTTPTimeout : public LLRefCount {
U64 mStalled; // The time (sTime_10ms) at which this transaction is considered to be stalling if nothing is transfered anymore.
public:
static F64 const sClockWidth_10ms; // Time between two clock ticks in 10 ms units.
static F64 const sClockWidth_40ms; // Time between two clock ticks in 40 ms units.
static U64 sTime_10ms; // Time since the epoch in 10 ms units.
#if defined(CWDEBUG) || defined(DEBUG_CURLIO)
ThreadSafeBufferedCurlEasyRequest* mLockObj;

1
indra/llmessage/message.cpp
View File

@@ -50,7 +50,6 @@
#include "lldarray.h"
#include "lldir.h"
#include "llerror.h"
#include "llerrorlegacy.h"
#include "llfasttimer.h"
#include "llhttpclient.h"
#include "llhttpnodeadapter.h"

13
indra/newview/app_settings/settings.xml
View File

@@ -4921,7 +4921,18 @@ This should be as low as possible, but too low may break functionality</string>
<key>Value</key>
<integer>-1</integer>
</map>
<key>DebugStatModeTexture</key>
<key>DebugStatModeHTTPTexture</key>
<map>
<key>Comment</key>
<string>Mode of stat in Statistics floater</string>
<key>Persist</key>
<integer>1</integer>
<key>Type</key>
<string>S32</string>
<key>Value</key>
<integer>-1</integer>
</map>
<key>DebugStatModeUDPTexture</key>
<map>
<key>Comment</key>
<string>Mode of stat in Statistics floater</string>

24
indra/newview/hippogridmanager.cpp
View File

@@ -374,8 +374,11 @@ void HippoGridInfo::onXmlCharacterData(void* userData, const XML_Char* s, int le
{
case XML_GRIDNICK:
{
if (self->mGridNick == "") self->mGridNick.assign(s, len);
self->mGridNick = sanitizeGridNick(self->mGridNick);
if (self->mGridNick == "")
{
self->mGridNick.assign(s, len);
self->mGridNick = sanitizeGridNick(self->mGridNick);
}
break;
}
@@ -407,7 +410,15 @@ void HippoGridInfo::onXmlCharacterData(void* userData, const XML_Char* s, int le
break;
}
case XML_GRIDNAME: self->mGridName.assign(s, len); break;
case XML_GRIDNAME:
{
if (self->mGridName == "")
{
self->mGridName.assign(s, len);
}
break;
}
case XML_LOGINPAGE: self->mLoginPage.assign(s, len); break;
case XML_WEBSITE: self->mWebSite.assign(s, len); break;
case XML_SUPPORT: self->mSupportUrl.assign(s, len); break;
@@ -678,13 +689,6 @@ HippoGridInfo* HippoGridManager::getGrid(const std::string& grid) const
}
}
HippoGridInfo* HippoGridManager::getConnectedGrid() const
{
return (mConnectedGrid)? mConnectedGrid: getCurrentGrid();
}
HippoGridInfo* HippoGridManager::getCurrentGrid() const
{
HippoGridInfo* grid = getGrid(mCurrentGrid);

3
indra/newview/hippogridmanager.h
View File

@@ -157,7 +157,8 @@ public:
void discardAndReload();
HippoGridInfo* getGrid(const std::string& grid) const;
HippoGridInfo* getConnectedGrid() const;
HippoGridInfo* getConnectedGrid() const { return mConnectedGrid ? mConnectedGrid : getCurrentGrid(); }
HippoGridInfo* getCurrentGrid() const;
const std::string& getDefaultGridNick() const;
const std::string& getCurrentGridNick() const;

4
indra/newview/hippopanelgrids.cpp
View File

@@ -133,7 +133,7 @@ BOOL HippoPanelGridsImpl::postBuild()
requires<LLButton>("btn_add");
requires<LLButton>("btn_copy");
requires<LLButton>("btn_default");
//requires<LLButton>("btn_gridinfo");
requires<LLButton>("btn_gridinfo");
requires<LLButton>("btn_help_render_compat");
if (!checkRequirements()) return false;
@@ -146,7 +146,7 @@ BOOL HippoPanelGridsImpl::postBuild()
childSetAction("btn_add", onClickAdd, this);
childSetAction("btn_copy", onClickCopy, this);
childSetAction("btn_default", onClickDefault, this);
//childSetAction("btn_gridinfo", onClickGridInfo, this);
childSetAction("btn_gridinfo", onClickGridInfo, this);
childSetAction("btn_help_render_compat", onClickHelpRenderCompat, this);
childSetAction("btn_advanced", onClickAdvanced, this);

45
indra/newview/llfloaterstats.cpp
View File

@@ -238,25 +238,52 @@ void LLFloaterStats::buildStats()
// Network statistics
LLStatView *net_statviewp = stat_viewp->addStatView("network stat view", "Network", "OpenDebugStatNet", rect);
stat_barp = net_statviewp->addStat("Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn");
stat_barp = net_statviewp->addStat("UDP Packets In", &(LLViewerStats::getInstance()->mPacketsInStat), "DebugStatModePacketsIn");
stat_barp->setUnitLabel("/sec");
stat_barp = net_statviewp->addStat("Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut");
stat_barp = net_statviewp->addStat("UDP Packets Out", &(LLViewerStats::getInstance()->mPacketsOutStat), "DebugStatModePacketsOut");
stat_barp->setUnitLabel("/sec");
stat_barp = net_statviewp->addStat("HTTP Textures", &(LLViewerStats::getInstance()->mHTTPTextureKBitStat), "DebugStatModeHTTPTexture");
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
stat_barp->mMaxBar = gSavedSettings.getF32("HTTPThrottleBandwidth") * 2; // Times two because we'll have over shoots.
stat_barp->mTickSpacing = 1.f;
while (stat_barp->mTickSpacing < stat_barp->mMaxBar / 8)
stat_barp->mTickSpacing *= 2.f;
stat_barp->mLabelSpacing = 2 * stat_barp->mTickSpacing;
stat_barp->mPerSec = FALSE;
stat_barp->mDisplayMean = FALSE;
stat_barp = net_statviewp->addStat("UDP Textures", &(LLViewerStats::getInstance()->mUDPTextureKBitStat), "DebugStatModeUDPTexture");
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
stat_barp->mMaxBar = 1024.f;
stat_barp->mTickSpacing = 128.f;
stat_barp->mLabelSpacing = 256.f;
stat_barp = net_statviewp->addStat("Objects", &(LLViewerStats::getInstance()->mObjectKBitStat), "DebugStatModeObjects");
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
stat_barp->mMaxBar = 1024.f;
stat_barp->mTickSpacing = 128.f;
stat_barp->mLabelSpacing = 256.f;
stat_barp = net_statviewp->addStat("Texture", &(LLViewerStats::getInstance()->mTextureKBitStat), "DebugStatModeTexture");
stat_barp = net_statviewp->addStat("Assets (UDP)", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset");
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
stat_barp->mMaxBar = 1024.f;
stat_barp->mTickSpacing = 128.f;
stat_barp->mLabelSpacing = 256.f;
stat_barp = net_statviewp->addStat("Asset", &(LLViewerStats::getInstance()->mAssetKBitStat), "DebugStatModeAsset");
stat_barp = net_statviewp->addStat("Layers (UDP)", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers");
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
stat_barp->mMaxBar = 1024.f;
stat_barp->mTickSpacing = 128.f;
stat_barp->mLabelSpacing = 256.f;
stat_barp = net_statviewp->addStat("Layers", &(LLViewerStats::getInstance()->mLayersKBitStat), "DebugStatModeLayers");
stat_barp->setUnitLabel(" kbps");
stat_barp = net_statviewp->addStat("Actual In", &(LLViewerStats::getInstance()->mActualInKBitStat),
stat_barp = net_statviewp->addStat("Actual In (UDP)", &(LLViewerStats::getInstance()->mActualInKBitStat),
"DebugStatModeActualIn", TRUE, FALSE);
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;
@@ -264,7 +291,7 @@ void LLFloaterStats::buildStats()
stat_barp->mTickSpacing = 128.f;
stat_barp->mLabelSpacing = 256.f;
stat_barp = net_statviewp->addStat("Actual Out", &(LLViewerStats::getInstance()->mActualOutKBitStat),
stat_barp = net_statviewp->addStat("Actual Out (UDP)", &(LLViewerStats::getInstance()->mActualOutKBitStat),
"DebugStatModeActualOut", TRUE, FALSE);
stat_barp->setUnitLabel(" kbps");
stat_barp->mMinBar = 0.f;

14
indra/newview/lltexturefetch.cpp
View File

@@ -36,7 +36,7 @@
#include "lltexturefetch.h"
#include "llcurl.h"
#include "aicurl.h"
#include "lldir.h"
#include "llhttpclient.h"
#include "llhttpstatuscodes.h"
@@ -1273,7 +1273,8 @@ bool LLTextureFetchWorker::doWork(S32 param)
// Let AICurl decide if we can process more HTTP requests at the moment or not.
static const LLCachedControl<F32> throttle_bandwidth("HTTPThrottleBandwidth", 2000);
if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, mFetcher->getTextureBandwidth() > throttle_bandwidth))
bool const no_bandwidth_throttling = gHippoGridManager->getConnectedGrid()->isAvination();
if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, throttle_bandwidth, no_bandwidth_throttling))
{
return false ; //wait.
}
@@ -1290,16 +1291,8 @@ bool LLTextureFetchWorker::doWork(S32 param)
mLoaded = FALSE;
mGetStatus = 0;
mGetReason.clear();
// Note: comparing mFetcher->getTextureBandwidth() with throttle_bandwidth is a bit like
// comparing apples and oranges, but it's only debug output. The first is the averaged
// bandwidth used for the body of successfully downloaded textures, averaged over roughtly
// 10 seconds, in kbits/s. The latter is the limit of the actual http curl downloads,
// including header and failures for anything (not just textures), averaged over the last
// second, also in kbits/s.
static const LLCachedControl<F32> throttle_bandwidth("HTTPThrottleBandwidth", 2000);
LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset
<< " Bytes: " << mRequestedSize
<< " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << throttle_bandwidth
<< LL_ENDL;
setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority);
mState = WAIT_HTTP_REQ;
@@ -2041,7 +2034,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image
mBadPacketCount(0),
mTextureCache(cache),
mImageDecodeThread(imagedecodethread),
mTextureBandwidth(0),
mHTTPTextureBits(0),
mTotalHTTPRequests(0),
mQAMode(qa_mode),

4
indra/newview/lltexturefetch.h
View File

@@ -79,9 +79,6 @@ public:
bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data);
bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data);
void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; }
F32 getTextureBandwidth() { return mTextureBandwidth; }
// Debug
BOOL isFromLocalCache(const LLUUID& id);
S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p,
@@ -164,7 +161,6 @@ private:
queue_t mHTTPTextureQueue;
typedef std::map<LLHost,std::set<LLUUID> > cancel_queue_t;
cancel_queue_t mCancelQueue;
F32 mTextureBandwidth;
LLTextureInfo mTextureInfo;
U32 mHTTPTextureBits;

11
indra/newview/lltextureview.cpp
View File

@@ -70,6 +70,7 @@ namespace AICurlInterface {
U32 getNumHTTPQueued(void);
U32 getNumHTTPAdded(void);
U32 getNumHTTPRunning(void);
size_t getHTTPBandwidth(void);
} // namespace AICurlInterface
////////////////////////////////////////////////////////////////////////////
@@ -620,17 +621,15 @@ void LLGLTexMemBar::draw()
text_color, LLFontGL::LEFT, LLFontGL::TOP);
left += LLFontGL::getFontMonospace()->getWidth(text);
// This bandwidth is averaged over roughly 10 seconds (in kbps) and therefore pretty inaccurate.
// Also, it only takes into account actual texture data (not headers etc). But all it is used for
// is for the color of some text in the texture console, so I guess it doesn't matter.
F32 bandwidth = LLAppViewer::getTextureFetch()->getTextureBandwidth();
// This bandwidth is averaged over 1 seconds (in kbps).
F32 bandwidth = AICurlInterface::getHTTPBandwidth() / 125.f; // Convert from bytes/s to kbps.
// This is the maximum bandwidth allowed for curl transactions (of any type and averaged per second),
// that is actually used to limit the number of HTTP texture requests (and only those).
// Comparing that with 'bandwidth' is a bit like comparing apples and oranges, but again... who really cares.
F32 max_bandwidth = gSavedSettings.getF32("HTTPThrottleBandwidth");
static const LLCachedControl<F32> max_bandwidth("HTTPThrottleBandwidth", 2000);
color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth*.75f ? LLColor4::yellow : text_color;
color[VALPHA] = text_color[VALPHA];
text = llformat("BW:%.0f/%.0f",bandwidth, max_bandwidth);
text = llformat("BW:%.0f/%.0f", bandwidth, max_bandwidth.get());
LLFontGL::getFontMonospace()->renderUTF8(text, 0, left, v_offset + line_height*2,
color, LLFontGL::LEFT, LLFontGL::TOP);

16
indra/newview/llviewerstats.cpp
View File

@@ -67,6 +67,10 @@
class AIHTTPTimeoutPolicy;
extern AIHTTPTimeoutPolicy viewerStatsResponder_timeout;
namespace AICurlInterface {
size_t getHTTPBandwidth(void);
}
class StatAttributes
{
public:
@@ -210,7 +214,8 @@ LLViewerStats::LLViewerStats() :
mLayersKBitStat("layerskbitstat"),
mObjectKBitStat("objectkbitstat"),
mAssetKBitStat("assetkbitstat"),
mTextureKBitStat("texturekbitstat"),
mHTTPTextureKBitStat("httptexturekbitstat"),
mUDPTextureKBitStat("udptexturekbitstat"),
mMallocStat("mallocstat"),
mVFSPendingOperations("vfspendingoperations"),
mObjectsDrawnStat("objectsdrawnstat"),
@@ -300,7 +305,8 @@ void LLViewerStats::resetStats()
stats.mKBitStat.reset();
stats.mLayersKBitStat.reset();
stats.mObjectKBitStat.reset();
stats.mTextureKBitStat.reset();
stats.mHTTPTextureKBitStat.reset();
stats.mUDPTextureKBitStat.reset();
stats.mVFSPendingOperations.reset();
stats.mAssetKBitStat.reset();
stats.mPacketsInStat.reset();
@@ -673,13 +679,13 @@ void update_statistics()
// Only update texture stats periodically so that they are less noisy
{
static const F32 texture_stats_freq = 10.f;
static const F32 texture_stats_freq = 0.25f;
static LLFrameTimer texture_stats_timer;
if (texture_stats_timer.getElapsedTimeF32() >= texture_stats_freq)
{
stats.mTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f);
stats.mHTTPTextureKBitStat.addValue(AICurlInterface::getHTTPBandwidth()/125.f);
stats.mUDPTextureKBitStat.addValue(LLViewerTextureList::sTextureBits/1024.f);
stats.mTexturePacketsStat.addValue(LLViewerTextureList::sTexturePackets);
LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerTextureList::sTextureBits/1024.f/texture_stats_timer.getElapsedTimeF32());
gTotalTextureBytes += LLViewerTextureList::sTextureBits / 8;
LLViewerTextureList::sTextureBits = 0;
LLViewerTextureList::sTexturePackets = 0;

3
indra/newview/llviewerstats.h
View File

@@ -43,7 +43,8 @@ public:
mLayersKBitStat,
mObjectKBitStat,
mAssetKBitStat,
mTextureKBitStat,
mHTTPTextureKBitStat,
mUDPTextureKBitStat,
mVFSPendingOperations,
mObjectsDrawnStat,
mObjectsCulledStat,

2
indra/newview/llviewertexturelist.cpp
View File

@@ -687,8 +687,6 @@ void LLViewerTextureList::updateImages(F32 max_time)
}
cleared = FALSE;
LLAppViewer::getTextureFetch()->setTextureBandwidth(LLViewerStats::getInstance()->mTextureKBitStat.getMeanPerSec());
S32 global_raw_memory;
{
global_raw_memory = *AIAccess<S32>(LLImageRaw::sGlobalRawMemory);

14
indra/newview/skins/default/xui/en-us/panel_preferences_grids.xml
View File

@@ -30,17 +30,17 @@
<!-- Buttons -->
<button label="Delete" label_selected="Delete" enabled="true" name="btn_delete"
height="18" width="75" left="120" bottom_delta="-27"
halign="center"
halign="center" tool_type="Delete the selected grid from the list"
follows="left|top" scale_image="true"
font="SansSerifSmall" mouse_opaque="true" />
<button label="Add" label_selected="Add" enabled="true" name="btn_add"
<button label="Create" label_selected="Add" enabled="true" name="btn_add"
height="18" width="75" left_delta="78" bottom_delta="0"
halign="center"
halign="center" tool_tip="Add a new grid to the list"
follows="left|top" scale_image="true"
font="SansSerifSmall" mouse_opaque="true" />
<button label="Copy" label_selected="Copy" enabled="true" name="btn_copy"
height="18" width="75" left_delta="78" bottom_delta="0"
halign="center"
halign="center" tool_tip="Create a new entry using the selected entry as template"
follows="left|top" scale_image="true"
font="SansSerifSmall" mouse_opaque="true" />
<!-- Advanced -->
@@ -65,11 +65,11 @@
font="SansSerifSmall"
follows="left|top|right" border_visible="false" mouse_opaque="false"
drop_shadow_visible="true" border_drop_shadow_visible="false" />
<!--<button label="Get Grid Info" label_selected="Get Grid Info" enabled="true" name="btn_gridinfo"
<button label="Refresh Grid URLs" enabled="true" name="btn_gridinfo"
height="18" width="100" left="120" bottom_delta="-22"
halign="center"
halign="center" tool_tip="Retrieve all grid info from the given Login URI (overwriting the current URLs, see Advanced)"
follows="left|top" scale_image="true"
font="SansSerifSmall" mouse_opaque="true" />-->
font="SansSerifSmall" mouse_opaque="true" />
<!-- Platform -->
<text type="string" length="1" enabled="true" name="platform_label"
height="10" width="100" left="12" bottom_delta="-32"