From 07201a5cfea304c87074ab0bc034bc2ee30456a1 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 5 Apr 2013 20:19:00 +0200 Subject: [PATCH 001/119] Add AICurlInterface::getNumHTTPRunning Replaces LLTextureFetch::getNumHTTPRequests. Returns AICurlInterface::Stats::running_handles. This is work in progress that temporarily doesn't compile because LLTextureFetch::getNumHTTPRequests is still being used somewhere. --- indra/llmessage/aicurl.cpp | 7 +++++++ indra/llmessage/aicurl.h | 5 +++++ indra/llmessage/aicurlthread.cpp | 7 +++++-- indra/newview/lltexturefetch.cpp | 15 ++++++--------- indra/newview/lltexturefetch.h | 1 - indra/newview/lltextureview.cpp | 13 ++++++++++++- indra/newview/llviewertexturelist.cpp | 1 + 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 3e3e510e1..f84ce373f 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -298,6 +298,7 @@ LLAtomicU32 Stats::easy_init_errors; LLAtomicU32 Stats::easy_cleanup_calls; LLAtomicU32 Stats::multi_calls; LLAtomicU32 Stats::multi_errors; +LLAtomicU32 Stats::running_handles; LLAtomicU32 Stats::AICurlEasyRequest_count; LLAtomicU32 Stats::AICurlEasyRequestStateMachine_count; LLAtomicU32 Stats::BufferedCurlEasyRequest_count; @@ -460,6 +461,12 @@ void setCAPath(std::string const& path) CertificateAuthority_w->path = path; } +// THREAD-SAFE +U32 getNumHTTPRunning(void) +{ + return Stats::running_handles; +} + //static void Stats::print(void) { diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index 9244839b7..d8aafe3b3 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -133,6 +133,7 @@ struct Stats { static LLAtomicU32 easy_cleanup_calls; static LLAtomicU32 multi_calls; static LLAtomicU32 multi_errors; + static LLAtomicU32 running_handles; static LLAtomicU32 AICurlEasyRequest_count; static LLAtomicU32 AICurlEasyRequestStateMachine_count; static LLAtomicU32 BufferedCurlEasyRequest_count; @@ -185,6 +186,10 @@ void setCAFile(std::string const& file); // Can be used to set the path to the Certificate Authority file. void setCAPath(std::string const& file); +// This used to be LLAppViewer::getTextureFetch()->getNumHTTPRequests(). +// Returns the number of active curl easy handles (that are actually attempting to download something). +U32 getNumHTTPRunning(void); + } // namespace AICurlInterface // Forward declaration (see aicurlprivate.h). diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 64b35c683..dc68b37b3 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -1653,6 +1653,7 @@ CURLMcode MultiHandle::socket_action(curl_socket_t sockfd, int ev_bitmask) } while(res == CURLM_CALL_MULTI_PERFORM); llassert(mAddedEasyRequests.size() >= (size_t)running_handles); + AICurlInterface::Stats::running_handles = running_handles; return res; } @@ -1696,7 +1697,8 @@ void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) { // ... to here. std::pair res = mAddedEasyRequests.insert(easy_request); llassert(res.second); // May not have been added before. - Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get_ptr().get() << "; now processing " << mAddedEasyRequests.size() << " easy handles."); + Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get_ptr().get() << + "; now processing " << mAddedEasyRequests.size() << " easy handles [running_handles = " << AICurlInterface::Stats::running_handles << "]."); return; } // The request could not be added, we have to queue it. @@ -1748,7 +1750,8 @@ CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator cons #endif mAddedEasyRequests.erase(iter); #if CWDEBUG - Dout(dc::curl, "MultiHandle::remove_easy_request: Removed AICurlEasyRequest " << (void*)lockobj << "; now processing " << mAddedEasyRequests.size() << " easy handles."); + Dout(dc::curl, "MultiHandle::remove_easy_request: Removed AICurlEasyRequest " << (void*)lockobj << + "; now processing " << mAddedEasyRequests.size() << " easy handles [running_handles = " << AICurlInterface::Stats::running_handles << "]."); #endif // Attempt to add a queued request, if any. diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 5efd486e0..22b98c1c5 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1305,6 +1305,12 @@ bool LLTextureFetchWorker::doWork(S32 param) mLoaded = FALSE; mGetStatus = 0; mGetReason.clear(); + // Note: comparing mFetcher->getTextureBandwidth() with throttle_bandwidth is a bit like + // comparing apples and oranges, but it's only debug output. The first is the averaged + // bandwidth used for the body of successfully downloaded textures, averaged over roughtly + // 10 seconds, in kbits/s. The latter is the limit of the actual http curl downloads, + // including header and failures for anything (not just textures), averaged over the last + // second, also in kbits/s. static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset << " Bytes: " << mRequestedSize @@ -2261,15 +2267,6 @@ S32 LLTextureFetch::getNumRequests() return size ; } -S32 LLTextureFetch::getNumHTTPRequests() -{ - mNetworkQueueMutex.lock() ; - S32 size = (S32)mHTTPTextureQueue.size(); - mNetworkQueueMutex.unlock() ; - - return size ; -} - U32 LLTextureFetch::getTotalNumHTTPRequests() { mNetworkQueueMutex.lock() ; diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index f23961083..7d2d86135 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -88,7 +88,6 @@ public: U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http); void dump(); S32 getNumRequests() ; - S32 getNumHTTPRequests() ; U32 getTotalNumHTTPRequests() ; // Public for access by callbacks diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index e57fbc3a2..8541dff47 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -64,6 +64,11 @@ LLTextureSizeView *gTextureCategoryView = NULL; //static std::set LLTextureView::sDebugImages; +// Forward declaration. +namespace AICurlInterface { + U32 getNumHTTPRunning(void); +} // namespace AICurlInterface + //////////////////////////////////////////////////////////////////////////// static std::string title_string1a("Tex UUID Area DDis(Req) DecodePri(Fetch) [download] pk/max"); @@ -601,7 +606,7 @@ void LLGLTexMemBar::draw() LLLFSThread::sLocal->getPending(), LLAppViewer::getImageDecodeThread()->getPending(), LLImageRaw::sRawImageCount, LLImageRaw::sRawImageCachedCount, - LLAppViewer::getTextureFetch()->getNumHTTPRequests(), + AICurlInterface::getNumHTTPRunning(), LLAppViewer::getImageDecodeThread()->getPending(), gTextureList.mCreateTextureList.size()); @@ -609,7 +614,13 @@ void LLGLTexMemBar::draw() text_color, LLFontGL::LEFT, LLFontGL::TOP); left += LLFontGL::getFontMonospace()->getWidth(text); + // This bandwidth is averaged over roughly 10 seconds (in kbps) and therefore pretty inaccurate. + // Also, it only takes into account actual texture data (not headers etc). But all it is used for + // is for the color of some text in the texture console, so I guess it doesn't matter. F32 bandwidth = LLAppViewer::getTextureFetch()->getTextureBandwidth(); + // This is the maximum bandwidth allowed for curl transactions (of any type and averaged per second), + // that is actually used to limit the number of HTTP texture requests (and only those). + // Comparing that with 'bandwidth' is a bit like comparing apples and oranges, but again... who really cares. F32 max_bandwidth = gSavedSettings.getF32("HTTPThrottleBandwidth"); color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth*.75f ? LLColor4::yellow : text_color; color[VALPHA] = text_color[VALPHA]; diff --git a/indra/newview/llviewertexturelist.cpp b/indra/newview/llviewertexturelist.cpp index 7a7b70f9b..1daf21e0c 100644 --- a/indra/newview/llviewertexturelist.cpp +++ b/indra/newview/llviewertexturelist.cpp @@ -1340,6 +1340,7 @@ void LLViewerTextureList::receiveImageHeader(LLMessageSystem *msg, void **user_d { received_size = msg->getReceiveSize() ; } + // Only used for statistics and texture console. gTextureList.sTextureBits += received_size * 8; gTextureList.sTexturePackets++; From f58bc148babbcd0cf0d01367bfedc48ded70cb3f Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 5 Apr 2013 20:24:48 +0200 Subject: [PATCH 002/119] Add a 'size' to command_queue. This adds a size (command_queue_st::size) to the ThreadSafe deque that was AICurlPrivate::command_queue that is kept equal to the number of 'add' commands in the deque minus the number of 'remove' commands in the deque. The deque itself is now command_queue_st::commands. --- indra/llmessage/aicurlthread.cpp | 43 ++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index dc68b37b3..1695d2949 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -264,9 +264,15 @@ void Command::reset(void) // // If at this point addRequest is called again, then it is detected that the ThreadSafeBufferedCurlEasyRequest is active. +struct command_queue_st { + std::deque commands; // The commands + size_t size; // Number of add commands in the queue minus the number of remove commands. +}; + // Multi-threaded queue for passing Command objects from the main-thread to the curl-thread. -AIThreadSafeSimpleDC > command_queue; -typedef AIAccess > command_queue_wat; +AIThreadSafeSimpleDC command_queue; // Fills 'size' with zero, because it's a global. +typedef AIAccess command_queue_wat; +typedef AIAccess command_queue_rat; AIThreadSafeDC command_being_processed; typedef AIWriteAccess command_being_processed_wat; @@ -1289,7 +1295,7 @@ void AICurlThread::process_commands(AICurlMultiHandle_wat const& multi_handle_w) // Access command_queue, and move command to command_being_processed. { command_queue_wat command_queue_w(command_queue); - if (command_queue_w->empty()) + if (command_queue_w->commands.empty()) { mWakeUpFlagMutex.lock(); mWakeUpFlag = false; @@ -1297,8 +1303,22 @@ void AICurlThread::process_commands(AICurlMultiHandle_wat const& multi_handle_w) break; } // Move the next command from the queue into command_being_processed. - *command_being_processed_wat(command_being_processed) = command_queue_w->front(); - command_queue_w->pop_front(); + command_st command; + { + command_being_processed_wat command_being_processed_w(command_being_processed); + *command_being_processed_w = command_queue_w->commands.front(); + command = command_being_processed_w->command(); + } + // Update the size: the number netto number of pending requests in the command queue. + command_queue_w->commands.pop_front(); + if (command == cmd_add) + { + command_queue_w->size--; + } + else if (command == cmd_remove) + { + command_queue_w->size++; + } } // Access command_being_processed only. { @@ -1929,7 +1949,8 @@ void clearCommandQueue(void) { // Clear the command queue now in order to avoid the global deinitialization order fiasco. command_queue_wat command_queue_w(command_queue); - command_queue_w->clear(); + command_queue_w->commands.clear(); + command_queue_w->size = 0; } //----------------------------------------------------------------------------- @@ -2331,7 +2352,7 @@ void AICurlEasyRequest::addRequest(void) // Find the last command added. command_st cmd = cmd_none; - for (std::deque::iterator iter = command_queue_w->begin(); iter != command_queue_w->end(); ++iter) + for (std::deque::iterator iter = command_queue_w->commands.begin(); iter != command_queue_w->commands.end(); ++iter) { if (*iter == *this) { @@ -2357,7 +2378,8 @@ void AICurlEasyRequest::addRequest(void) } #endif // Add a command to add the new request to the multi session to the command queue. - command_queue_w->push_back(Command(*this, cmd_add)); + command_queue_w->commands.push_back(Command(*this, cmd_add)); + command_queue_w->size++; AICurlEasyRequest_wat(*get())->add_queued(); } // Something was added to the queue, wake up the thread to get it. @@ -2381,7 +2403,7 @@ void AICurlEasyRequest::removeRequest(void) // Find the last command added. command_st cmd = cmd_none; - for (std::deque::iterator iter = command_queue_w->begin(); iter != command_queue_w->end(); ++iter) + for (std::deque::iterator iter = command_queue_w->commands.begin(); iter != command_queue_w->commands.end(); ++iter) { if (*iter == *this) { @@ -2418,7 +2440,8 @@ void AICurlEasyRequest::removeRequest(void) } #endif // Add a command to remove this request from the multi session to the command queue. - command_queue_w->push_back(Command(*this, cmd_remove)); + command_queue_w->commands.push_back(Command(*this, cmd_remove)); + command_queue_w->size--; // Suppress warning that would otherwise happen if the callbacks are revoked before the curl thread removed the request. AICurlEasyRequest_wat(*get())->remove_queued(); } From 17455e244273cd8bbf1c5c4f1b09a3206128ade4 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 5 Apr 2013 20:39:42 +0200 Subject: [PATCH 003/119] Add PerHostRequestQueue::sTotalQueued The new variable is updated to contain the sum of the size of PerHostRequestQueue::mQueuedRequests of every PerHostRequestQueue object, and thus the total number of queued requests for all hosts together. Also added PerHostRequestQueue::host_queued_plus_added_size() which returns the the sum of queued requests plus the requests already added to the multi handle for this particular hostname. --- indra/llmessage/aicurlperhost.cpp | 14 ++++++++++++-- indra/llmessage/aicurlperhost.h | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperhost.cpp index b444ba6df..681dd769e 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperhost.cpp @@ -37,6 +37,7 @@ namespace AICurlPrivate { PerHostRequestQueue::threadsafe_instance_map_type PerHostRequestQueue::sInstanceMap; +LLAtomicS32 PerHostRequestQueue::sTotalQueued; U32 curl_concurrent_connections_per_host; //static @@ -72,7 +73,7 @@ void PerHostRequestQueue::release(PerHostRequestQueuePtr& instance) return; } // The reference in the map is the last one; that means there can't be any curl easy requests queued for this host. - llassert(PerHostRequestQueue_wat(*instance)->mQueuedRequests.empty()); + llassert(PerHostRequestQueue_rat(*instance)->mQueuedRequests.empty()); // Find the host and erase it from the map. iterator const end = instance_map_w->end(); for(iterator iter = instance_map_w->begin(); iter != end; ++iter) @@ -111,6 +112,7 @@ void PerHostRequestQueue::removed_from_multi_handle(void) void PerHostRequestQueue::queue(AICurlEasyRequest const& easy_request) { mQueuedRequests.push_back(easy_request.get_ptr()); + sTotalQueued++; } bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) @@ -134,6 +136,8 @@ bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) prev = cur; } mQueuedRequests.pop_back(); // if this is safe. + --sTotalQueued; + llassert(sTotalQueued >= 0); return true; } @@ -143,6 +147,8 @@ void PerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) { multi_handle->add_easy_request(mQueuedRequests.front()); mQueuedRequests.pop_front(); + --sTotalQueued; + llassert(sTotalQueued >= 0); } } @@ -153,7 +159,11 @@ void PerHostRequestQueue::purge(void) for (iterator host = instance_map_w->begin(); host != instance_map_w->end(); ++host) { Dout(dc::curl, "Purging queue of host \"" << host->first << "\"."); - PerHostRequestQueue_wat(*host->second)->mQueuedRequests.clear(); + PerHostRequestQueue_wat per_host_w(*host->second); + size_t s = per_host_w->mQueuedRequests.size(); + per_host_w->mQueuedRequests.clear(); + sTotalQueued -= s; + llassert(sTotalQueued >= 0); } } diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index c464d490c..27107d7c2 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -100,6 +100,8 @@ class PerHostRequestQueue { int mAdded; // Number of active easy handles with this host. queued_request_type mQueuedRequests; // Waiting (throttled) requests. + static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all PerHostRequestQueue objects together. + public: void added_to_multi_handle(void); // Called when an easy handle for this host has been added to the multi handle. void removed_from_multi_handle(void); // Called when an easy handle for this host is removed again from the multi handle. @@ -110,6 +112,10 @@ class PerHostRequestQueue { void add_queued_to(curlthread::MultiHandle* mh); // Add queued easy handle (if any) to the multi handle. The request is removed from the queue, // followed by either a call to added_to_multi_handle() or to queue() to add it back. + + S32 host_queued_plus_added_size(void) const { return mQueuedRequests.size() + mAdded; } + static S32 total_queued_size(void) { return sTotalQueued; } + private: // Disallow copying. PerHostRequestQueue(PerHostRequestQueue const&) { } From 6c9b136d321ac0c9528d1c50dc0ced0b55ae1c64 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 5 Apr 2013 20:46:33 +0200 Subject: [PATCH 004/119] Removal of HTTPRequestRate and dead code. --- indra/newview/app_settings/settings.xml | 11 ----------- indra/newview/lltexturefetch.cpp | 24 ------------------------ 2 files changed, 35 deletions(-) diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 2b7f2fb86..4de95b255 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -172,17 +172,6 @@ 1 - HTTPRequestRate - - Comment - Number of HTTP texture requests fired per second. - Persist - 1 - Type - U32 - Value - 30 - HTTPMaxRequests Comment diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 22b98c1c5..e3aed97c1 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -495,30 +495,6 @@ public: SGHostBlackList::blacklist_t SGHostBlackList::blacklist; -#if 0 -//call every time a connection is opened -//return true if connecting allowed -static bool sgConnectionThrottle() { - const U32 THROTTLE_TIMESTEPS_PER_SECOND = 10; - static const LLCachedControl max_connections_per_second("HTTPRequestRate", 30); - U32 max_connections = max_connections_per_second/THROTTLE_TIMESTEPS_PER_SECOND; - const U32 timestep = USEC_PER_SEC/THROTTLE_TIMESTEPS_PER_SECOND; - U64 now = LLTimer::getTotalTime(); - std::deque timestamps; - while(!timestamps.empty() && (timestamps[0]<=now-timestep)) { - timestamps.pop_front(); - } - if(timestamps.size() < max_connections) { - //llinfos << "throttle pass" << llendl; - timestamps.push_back(now); - return true; - } else { - //llinfos << "throttle fail" << llendl; - return false; - } -} -#endif - ////////////////////////////////////////////////////////////////////////////// // Cross-thread messaging for asset metrics. From db7c3781600c3b41ba40a19881e6e6553520dec6 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 5 Apr 2013 20:49:23 +0200 Subject: [PATCH 005/119] Added MultiHandle::sTotalAdded This new variable is updated to contain the total number of requests in MultiHandle::mAddedEasyRequests. It can be a static because MultiHandle is a singleton (there is only one curl thread) and it has to be an (atomic) static because we don't want to need to take the lock on MultiHandle for accessing this variable. --- indra/llmessage/aicurlthread.cpp | 6 ++++++ indra/llmessage/aicurlthread.h | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 1695d2949..336993da5 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -1581,6 +1581,8 @@ void AICurlThread::run(void) //----------------------------------------------------------------------------- // MultiHandle +LLAtomicU32 MultiHandle::sTotalAdded; + MultiHandle::MultiHandle(void) : mTimeout(-1), mReadPollSet(NULL), mWritePollSet(NULL) { mReadPollSet = new PollSet; @@ -1717,6 +1719,8 @@ void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) { // ... to here. std::pair res = mAddedEasyRequests.insert(easy_request); llassert(res.second); // May not have been added before. + sTotalAdded++; + llassert(sTotalAdded == mAddedEasyRequests.size()); Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get_ptr().get() << "; now processing " << mAddedEasyRequests.size() << " easy handles [running_handles = " << AICurlInterface::Stats::running_handles << "]."); return; @@ -1769,6 +1773,8 @@ CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator cons ThreadSafeBufferedCurlEasyRequest* lockobj = iter->get_ptr().get(); #endif mAddedEasyRequests.erase(iter); + --sTotalAdded; + llassert(sTotalAdded == mAddedEasyRequests.size()); #if CWDEBUG Dout(dc::curl, "MultiHandle::remove_easy_request: Removed AICurlEasyRequest " << (void*)lockobj << "; now processing " << mAddedEasyRequests.size() << " easy handles [running_handles = " << AICurlInterface::Stats::running_handles << "]."); diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index fc29ac1fc..44105eefa 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -75,6 +75,7 @@ class MultiHandle : public CurlMultiHandle typedef std::set addedEasyRequests_type; addedEasyRequests_type mAddedEasyRequests; // All easy requests currently added to the multi handle. long mTimeout; // The last timeout in ms as set by the callback CURLMOPT_TIMERFUNCTION. + static LLAtomicU32 sTotalAdded; // The (sum of the) size of mAddedEasyRequests (of every MultiHandle, but there is only one). private: // Store result and trigger events for easy request. @@ -96,6 +97,9 @@ class MultiHandle : public CurlMultiHandle // Called from the main loop every time select() timed out. void handle_stalls(void); + // Return the total number of added curl requests. + static U32 total_added_size(void) { return sTotalAdded; } + public: //----------------------------------------------------------------------------- // Curl socket administration: From 7866c68ab254cc91bda5ec618a17b6292214a2b2 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sat, 6 Apr 2013 21:54:34 +0200 Subject: [PATCH 006/119] Moved PerHostRequestQueue[Ptr] outside of AICurlPrivate. Renamed PerHostRequestQueue and PerHostRequestQueuePtr to AIPerHostRequestQueue and AIPerHostRequestQueuePtr respectively and moved them to global namespace. This in preparation for using them in the texture fetcher (as function of the hostname of the url that is contructed there). --- indra/llmessage/aicurl.cpp | 6 +-- indra/llmessage/aicurlperhost.cpp | 72 +++++++++++++++++-------------- indra/llmessage/aicurlperhost.h | 55 +++++++++++++---------- indra/llmessage/aicurlprivate.h | 6 +-- indra/llmessage/aicurlthread.cpp | 6 +-- indra/llmessage/aicurlthread.h | 2 +- 6 files changed, 82 insertions(+), 65 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index f84ce373f..b52b69cb5 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -961,7 +961,7 @@ CurlEasyRequest::~CurlEasyRequest() revokeCallbacks(); if (mPerHostPtr) { - PerHostRequestQueue::release(mPerHostPtr); + AIPerHostRequestQueue::release(mPerHostPtr); } // This wasn't freed yet if the request never finished. curl_slist_free_all(mHeaders); @@ -1286,14 +1286,14 @@ void CurlEasyRequest::queued_for_removal(AICurlEasyRequest_wat& curl_easy_reques } #endif -PerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void) +AIPerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void) { if (!mPerHostPtr) { // mPerHostPtr is really just a speed-up cache. // The reason we can cache it is because mLowercaseHostname is only set // in finalizeRequest which may only be called once: it never changes. - mPerHostPtr = PerHostRequestQueue::instance(mLowercaseHostname); + mPerHostPtr = AIPerHostRequestQueue::instance(mLowercaseHostname); } return mPerHostPtr; } diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperhost.cpp index 681dd769e..608bb1857 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperhost.cpp @@ -1,8 +1,8 @@ /** * @file aiperhost.cpp - * @brief Implementation of PerHostRequestQueue + * @brief Implementation of AIPerHostRequestQueue * - * Copyright (c) 2012, Aleric Inglewood. + * Copyright (c) 2012, 2013, Aleric Inglewood. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,36 +26,60 @@ * * 04/11/2012 * Initial version, written by Aleric Inglewood @ SL + * + * 06/04/2013 + * Renamed AICurlPrivate::PerHostRequestQueue[Ptr] to AIPerHostRequestQueue[Ptr] + * to allow public access. */ #include "sys.h" #include "aicurlperhost.h" #include "aicurlthread.h" +AIPerHostRequestQueue::threadsafe_instance_map_type AIPerHostRequestQueue::sInstanceMap; +LLAtomicS32 AIPerHostRequestQueue::sTotalQueued; + #undef AICurlPrivate namespace AICurlPrivate { -PerHostRequestQueue::threadsafe_instance_map_type PerHostRequestQueue::sInstanceMap; -LLAtomicS32 PerHostRequestQueue::sTotalQueued; U32 curl_concurrent_connections_per_host; +// Friend functions of RefCountedThreadSafePerHostRequestQueue + +void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* per_host) +{ + per_host->mReferenceCount++; +} + +void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* per_host) +{ + if (--per_host->mReferenceCount == 0) + { + delete per_host; + } +} + +} // namespace AICurlPrivate + +using namespace AICurlPrivate; + //static -PerHostRequestQueuePtr PerHostRequestQueue::instance(std::string const& hostname) +AIPerHostRequestQueuePtr AIPerHostRequestQueue::instance(std::string const& hostname) { llassert(!hostname.empty()); instance_map_wat instance_map_w(sInstanceMap); - PerHostRequestQueue::iterator iter = instance_map_w->find(hostname); + AIPerHostRequestQueue::iterator iter = instance_map_w->find(hostname); if (iter == instance_map_w->end()) { iter = instance_map_w->insert(instance_map_type::value_type(hostname, new RefCountedThreadSafePerHostRequestQueue)).first; } - // Note: the creation of PerHostRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()). + // Note: the creation of AIPerHostRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()). return iter->second; } //static -void PerHostRequestQueue::release(PerHostRequestQueuePtr& instance) +void AIPerHostRequestQueue::release(AIPerHostRequestQueuePtr& instance) { if (instance->exactly_two_left()) // Being 'instance' and the one in sInstanceMap. { @@ -91,31 +115,31 @@ void PerHostRequestQueue::release(PerHostRequestQueuePtr& instance) instance.reset(); } -bool PerHostRequestQueue::throttled() const +bool AIPerHostRequestQueue::throttled() const { llassert(mAdded <= int(curl_concurrent_connections_per_host)); return mAdded == int(curl_concurrent_connections_per_host); } -void PerHostRequestQueue::added_to_multi_handle(void) +void AIPerHostRequestQueue::added_to_multi_handle(void) { llassert(mAdded < int(curl_concurrent_connections_per_host)); ++mAdded; } -void PerHostRequestQueue::removed_from_multi_handle(void) +void AIPerHostRequestQueue::removed_from_multi_handle(void) { --mAdded; llassert(mAdded >= 0); } -void PerHostRequestQueue::queue(AICurlEasyRequest const& easy_request) +void AIPerHostRequestQueue::queue(AICurlEasyRequest const& easy_request) { mQueuedRequests.push_back(easy_request.get_ptr()); sTotalQueued++; } -bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) +bool AIPerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) { queued_request_type::iterator const end = mQueuedRequests.end(); queued_request_type::iterator cur = std::find(mQueuedRequests.begin(), end, easy_request.get_ptr()); @@ -128,7 +152,7 @@ bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) // the back with swap (could just swap with the end immediately, but I don't // want to break the order in which requests where added). Swap is also not // thread-safe, but OK here because it only touches the objects in the deque, - // and the deque is protected by the lock on the PerHostRequestQueue object. + // and the deque is protected by the lock on the AIPerHostRequestQueue object. queued_request_type::iterator prev = cur; while (++cur != end) { @@ -141,7 +165,7 @@ bool PerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) return true; } -void PerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) +void AIPerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) { if (!mQueuedRequests.empty()) { @@ -153,7 +177,7 @@ void PerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) } //static -void PerHostRequestQueue::purge(void) +void AIPerHostRequestQueue::purge(void) { instance_map_wat instance_map_w(sInstanceMap); for (iterator host = instance_map_w->begin(); host != instance_map_w->end(); ++host) @@ -167,19 +191,3 @@ void PerHostRequestQueue::purge(void) } } -// Friend functions of RefCountedThreadSafePerHostRequestQueue - -void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* per_host) -{ - per_host->mReferenceCount++; -} - -void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* per_host) -{ - if (--per_host->mReferenceCount == 0) - { - delete per_host; - } -} - -} // namespace AICurlPrivate diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 27107d7c2..238cb622e 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -1,8 +1,8 @@ /** * @file aicurlperhost.h - * @brief Definition of class PerHostRequestQueue + * @brief Definition of class AIPerHostRequestQueue * - * Copyright (c) 2012, Aleric Inglewood. + * Copyright (c) 2012, 2013, Aleric Inglewood. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,6 +26,10 @@ * * 04/11/2012 * Initial version, written by Aleric Inglewood @ SL + * + * 06/04/2013 + * Renamed AIPrivate::PerHostRequestQueue[Ptr] to AIPerHostRequestQueue[Ptr] + * to allow public access. */ #ifndef AICURLPERHOST_H @@ -39,68 +43,70 @@ #include "aithreadsafe.h" class AICurlEasyRequest; +class AIPerHostRequestQueue; namespace AICurlPrivate { namespace curlthread { class MultiHandle; } -class PerHostRequestQueue; class RefCountedThreadSafePerHostRequestQueue; class ThreadSafeBufferedCurlEasyRequest; // Forward declaration of BufferedCurlEasyRequestPtr (see aicurlprivate.h). typedef boost::intrusive_ptr BufferedCurlEasyRequestPtr; -// PerHostRequestQueue objects are created by the curl thread and destructed by the main thread. +// AIPerHostRequestQueue objects are created by the curl thread and destructed by the main thread. // We need locking. -typedef AIThreadSafeSimpleDC threadsafe_PerHostRequestQueue; -typedef AIAccessConst PerHostRequestQueue_crat; -typedef AIAccess PerHostRequestQueue_rat; -typedef AIAccess PerHostRequestQueue_wat; +typedef AIThreadSafeSimpleDC threadsafe_PerHostRequestQueue; +typedef AIAccessConst PerHostRequestQueue_crat; +typedef AIAccess PerHostRequestQueue_rat; +typedef AIAccess PerHostRequestQueue_wat; + +} // namespace AICurlPrivate // We can't put threadsafe_PerHostRequestQueue in a std::map because you can't copy a mutex. // Therefore, use an intrusive pointer for the threadsafe type. -typedef boost::intrusive_ptr PerHostRequestQueuePtr; +typedef boost::intrusive_ptr AIPerHostRequestQueuePtr; //----------------------------------------------------------------------------- -// PerHostRequestQueue +// AIPerHostRequestQueue // This class provides a static interface to create and maintain instances -// of PerHostRequestQueue objects, so that at any moment there is at most +// of AIPerHostRequestQueue objects, so that at any moment there is at most // one instance per hostname. Those instances then are used to queue curl // requests when the maximum number of connections for that host already // have been reached. -class PerHostRequestQueue { +class AIPerHostRequestQueue { private: - typedef std::map instance_map_type; + typedef std::map instance_map_type; typedef AIThreadSafeSimpleDC threadsafe_instance_map_type; typedef AIAccess instance_map_rat; typedef AIAccess instance_map_wat; - static threadsafe_instance_map_type sInstanceMap; // Map of PerHostRequestQueue instances with the hostname as key. + static threadsafe_instance_map_type sInstanceMap; // Map of AIPerHostRequestQueue instances with the hostname as key. - friend class AIThreadSafeSimpleDC; //threadsafe_PerHostRequestQueue - PerHostRequestQueue(void) : mAdded(0) { } + friend class AIThreadSafeSimpleDC; //threadsafe_PerHostRequestQueue + AIPerHostRequestQueue(void) : mAdded(0) { } public: typedef instance_map_type::iterator iterator; typedef instance_map_type::const_iterator const_iterator; // Return (possibly create) a unique instance for the given hostname. - static PerHostRequestQueuePtr instance(std::string const& hostname); + static AIPerHostRequestQueuePtr instance(std::string const& hostname); // Release instance (object will be deleted if this was the last instance). - static void release(PerHostRequestQueuePtr& instance); + static void release(AIPerHostRequestQueuePtr& instance); // Remove everything. Called upon viewer exit. static void purge(void); private: - typedef std::deque queued_request_type; + typedef std::deque queued_request_type; int mAdded; // Number of active easy handles with this host. queued_request_type mQueuedRequests; // Waiting (throttled) requests. - static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all PerHostRequestQueue objects together. + static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all AIPerHostRequestQueue objects together. public: void added_to_multi_handle(void); // Called when an easy handle for this host has been added to the multi handle. @@ -110,7 +116,8 @@ class PerHostRequestQueue { void queue(AICurlEasyRequest const& easy_request); // Add easy_request to the queue. bool cancel(AICurlEasyRequest const& easy_request); // Remove easy_request from the queue (if it's there). - void add_queued_to(curlthread::MultiHandle* mh); // Add queued easy handle (if any) to the multi handle. The request is removed from the queue, + void add_queued_to(AICurlPrivate::curlthread::MultiHandle* mh); + // Add queued easy handle (if any) to the multi handle. The request is removed from the queue, // followed by either a call to added_to_multi_handle() or to queue() to add it back. S32 host_queued_plus_added_size(void) const { return mQueuedRequests.size() + mAdded; } @@ -118,16 +125,18 @@ class PerHostRequestQueue { private: // Disallow copying. - PerHostRequestQueue(PerHostRequestQueue const&) { } + AIPerHostRequestQueue(AIPerHostRequestQueue const&) { } }; +namespace AICurlPrivate { + class RefCountedThreadSafePerHostRequestQueue : public threadsafe_PerHostRequestQueue { public: RefCountedThreadSafePerHostRequestQueue(void) : mReferenceCount(0) { } bool exactly_two_left(void) const { return mReferenceCount == 2; } private: - // Used by PerHostRequestQueuePtr. Object is deleted when reference count reaches zero. + // Used by AIPerHostRequestQueuePtr. Object is deleted when reference count reaches zero. LLAtomicU32 mReferenceCount; friend void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* p); diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index cfd4b5d91..9d8d51fb0 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -305,7 +305,7 @@ class CurlEasyRequest : public CurlEasyHandle { AIHTTPTimeoutPolicy const* mTimeoutPolicy; std::string mLowercaseHostname; // Lowercase hostname (canonicalized) extracted from the url. - PerHostRequestQueuePtr mPerHostPtr; // Pointer to the corresponding PerHostRequestQueue. + AIPerHostRequestQueuePtr mPerHostPtr; // Pointer to the corresponding AIPerHostRequestQueue. LLPointer mTimeout;// Timeout administration object associated with last created CurlSocketInfo. bool mTimeoutIsOrphan; // Set to true when mTimeout is not (yet) associated with a CurlSocketInfo. #if defined(CWDEBUG) || defined(DEBUG_CURLIO) @@ -348,8 +348,8 @@ class CurlEasyRequest : public CurlEasyHandle { inline ThreadSafeBufferedCurlEasyRequest const* get_lockobj(void) const; // PerHost API. - PerHostRequestQueuePtr getPerHostPtr(void); // (Optionally create and) return a pointer to the unique - // PerHostRequestQueue corresponding to mLowercaseHostname. + AIPerHostRequestQueuePtr getPerHostPtr(void); // (Optionally create and) return a pointer to the unique + // AIPerHostRequestQueue corresponding to mLowercaseHostname. bool removeFromPerHostQueue(AICurlEasyRequest const&) const; // Remove this request from the per-host queue, if queued at all. // Returns true if it was queued. protected: diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 336993da5..6ed1b19b7 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -1573,7 +1573,7 @@ void AICurlThread::run(void) multi_handle_w->check_msg_queue(); } // Clear the queued requests. - PerHostRequestQueue::purge(); + AIPerHostRequestQueue::purge(); } AICurlMultiHandle::destroyInstance(); } @@ -1700,7 +1700,7 @@ static U32 curl_max_total_concurrent_connections = 32; // Initialized on st void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) { bool throttled = true; // Default. - PerHostRequestQueuePtr per_host; + AIPerHostRequestQueuePtr per_host; { AICurlEasyRequest_wat curl_easy_request_w(*easy_request); per_host = curl_easy_request_w->getPerHostPtr(); @@ -1759,7 +1759,7 @@ CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator const& iter, bool as_per_command) { CURLMcode res; - PerHostRequestQueuePtr per_host; + AIPerHostRequestQueuePtr per_host; { AICurlEasyRequest_wat curl_easy_request_w(**iter); res = curl_easy_request_w->remove_handle_from_multi(curl_easy_request_w, mMultiHandle); diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index 44105eefa..4c96b790c 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -81,7 +81,7 @@ class MultiHandle : public CurlMultiHandle // Store result and trigger events for easy request. void finish_easy_request(AICurlEasyRequest const& easy_request, CURLcode result); // Remove easy request at iter (must exist). - // Note that it's possible that a new request from a PerHostRequestQueue::mQueuedRequests is inserted before iter. + // Note that it's possible that a new request from a AIPerHostRequestQueue::mQueuedRequests is inserted before iter. CURLMcode remove_easy_request(addedEasyRequests_type::iterator const& iter, bool as_per_command); static int socket_callback(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp); From d26241c0f277e980efefaedae273f9194fe12baf Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sat, 6 Apr 2013 22:49:54 +0200 Subject: [PATCH 007/119] Move extract_canonical_hostname to AIPerHostRequestQueue::extract_canonical_hostname. This to make it available to the texture fetcher. --- indra/llmessage/aicurl.cpp | 52 +------------------------------ indra/llmessage/aicurlperhost.cpp | 50 +++++++++++++++++++++++++++++ indra/llmessage/aicurlperhost.h | 3 ++ 3 files changed, 54 insertions(+), 51 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index b52b69cb5..9df34518e 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -1091,56 +1091,6 @@ void CurlEasyRequest::applyDefaultOptions(void) ); } -// url must be of the form -// (see http://www.ietf.org/rfc/rfc3986.txt Appendix A for definitions not given here): -// -// url = sheme ":" hier-part [ "?" query ] [ "#" fragment ] -// hier-part = "//" authority path-abempty -// authority = [ userinfo "@" ] host [ ":" port ] -// path-abempty = *( "/" segment ) -// -// That is, a hier-part of the form '/ path-absolute', '/ path-rootless' or -// '/ path-empty' is NOT allowed here. This should be safe because we only -// call this function for curl access, any file access would use APR. -// -// However, as a special exception, this function allows: -// -// url = authority path-abempty -// -// without the 'sheme ":" "//"' parts. -// -// As follows from the ABNF (see RFC, Appendix A): -// - authority is either terminated by a '/' or by the end of the string because -// neither userinfo, host nor port may contain a '/'. -// - userinfo does not contain a '@', and if it exists, is always terminated by a '@'. -// - port does not contain a ':', and if it exists is always prepended by a ':'. -// -// Only called by CurlEasyRequest::finalizeRequest. -static std::string extract_canonical_hostname(std::string const& url) -{ - std::string::size_type pos; - std::string::size_type authority = 0; // Default if there is no sheme. - if ((pos = url.find("://")) != url.npos && pos < url.find('/')) authority = pos + 3; // Skip the "sheme://" if any, the second find is to avoid finding a "://" as part of path-abempty. - std::string::size_type host = authority; // Default if there is no userinfo. - if ((pos = url.find('@', authority)) != url.npos) host = pos + 1; // Skip the "userinfo@" if any. - authority = url.length() - 1; // Default last character of host if there is no path-abempty. - if ((pos = url.find('/', host)) != url.npos) authority = pos - 1; // Point to last character of host. - std::string::size_type len = url.find_last_not_of(":0123456789", authority) - host + 1; // Skip trailing ":port", if any. - std::string hostname(url, host, len); -#if APR_CHARSET_EBCDIC -#error Not implemented -#else - // Convert hostname to lowercase in a way that we compare two hostnames equal iff libcurl does. - for (std::string::iterator iter = hostname.begin(); iter != hostname.end(); ++iter) - { - int c = *iter; - if (c >= 'A' && c <= 'Z') - *iter = c + ('a' - 'A'); - } -#endif - return hostname; -} - void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolicy const& policy, AICurlEasyRequestStateMachine* state_machine) { DoutCurlEntering("CurlEasyRequest::finalizeRequest(\"" << url << "\", " << policy.name() << ", " << (void*)state_machine << ")"); @@ -1164,7 +1114,7 @@ void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolic setopt(CURLOPT_HTTPHEADER, mHeaders); setoptString(CURLOPT_URL, url); llassert(!mPerHostPtr); - mLowercaseHostname = extract_canonical_hostname(url); + mLowercaseHostname = AIPerHostRequestQueue::extract_canonical_hostname(url); mTimeoutPolicy = &policy; state_machine->setTotalDelayTimeout(policy.getTotalDelay()); // The following line is a bit tricky: we store a pointer to the object without increasing its reference count. diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperhost.cpp index 608bb1857..8e9da1334 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperhost.cpp @@ -64,6 +64,56 @@ void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* per_host) using namespace AICurlPrivate; +// url must be of the form +// (see http://www.ietf.org/rfc/rfc3986.txt Appendix A for definitions not given here): +// +// url = sheme ":" hier-part [ "?" query ] [ "#" fragment ] +// hier-part = "//" authority path-abempty +// authority = [ userinfo "@" ] host [ ":" port ] +// path-abempty = *( "/" segment ) +// +// That is, a hier-part of the form '/ path-absolute', '/ path-rootless' or +// '/ path-empty' is NOT allowed here. This should be safe because we only +// call this function for curl access, any file access would use APR. +// +// However, as a special exception, this function allows: +// +// url = authority path-abempty +// +// without the 'sheme ":" "//"' parts. +// +// As follows from the ABNF (see RFC, Appendix A): +// - authority is either terminated by a '/' or by the end of the string because +// neither userinfo, host nor port may contain a '/'. +// - userinfo does not contain a '@', and if it exists, is always terminated by a '@'. +// - port does not contain a ':', and if it exists is always prepended by a ':'. +// +//static +std::string AIPerHostRequestQueue::extract_canonical_hostname(std::string const& url) +{ + std::string::size_type pos; + std::string::size_type authority = 0; // Default if there is no sheme. + if ((pos = url.find("://")) != url.npos && pos < url.find('/')) authority = pos + 3; // Skip the "sheme://" if any, the second find is to avoid finding a "://" as part of path-abempty. + std::string::size_type host = authority; // Default if there is no userinfo. + if ((pos = url.find('@', authority)) != url.npos) host = pos + 1; // Skip the "userinfo@" if any. + authority = url.length() - 1; // Default last character of host if there is no path-abempty. + if ((pos = url.find('/', host)) != url.npos) authority = pos - 1; // Point to last character of host. + std::string::size_type len = url.find_last_not_of(":0123456789", authority) - host + 1; // Skip trailing ":port", if any. + std::string hostname(url, host, len); +#if APR_CHARSET_EBCDIC +#error Not implemented +#else + // Convert hostname to lowercase in a way that we compare two hostnames equal iff libcurl does. + for (std::string::iterator iter = hostname.begin(); iter != hostname.end(); ++iter) + { + int c = *iter; + if (c >= 'A' && c <= 'Z') + *iter = c + ('a' - 'A'); + } +#endif + return hostname; +} + //static AIPerHostRequestQueuePtr AIPerHostRequestQueue::instance(std::string const& hostname) { diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 238cb622e..502fb3edb 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -91,6 +91,9 @@ class AIPerHostRequestQueue { typedef instance_map_type::iterator iterator; typedef instance_map_type::const_iterator const_iterator; + // Utility function; extract canonical (lowercase) hostname from url. + static std::string extract_canonical_hostname(std::string const& url); + // Return (possibly create) a unique instance for the given hostname. static AIPerHostRequestQueuePtr instance(std::string const& hostname); From c20e33c7f40564b9b0f02a5b57dd2f3c1b63f4f3 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sun, 7 Apr 2013 05:54:07 +0200 Subject: [PATCH 008/119] Added AIPerHostRequestQueue::queued_commands() Returns the number of add commands minus the number of remove commands in the command_queue for this host. --- indra/llmessage/aicurlperhost.h | 6 +++++- indra/llmessage/aicurlthread.cpp | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 502fb3edb..457b2cccd 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -85,7 +85,7 @@ class AIPerHostRequestQueue { static threadsafe_instance_map_type sInstanceMap; // Map of AIPerHostRequestQueue instances with the hostname as key. friend class AIThreadSafeSimpleDC; //threadsafe_PerHostRequestQueue - AIPerHostRequestQueue(void) : mAdded(0) { } + AIPerHostRequestQueue(void) : mQueuedCommands(0), mAdded(0) { } public: typedef instance_map_type::iterator iterator; @@ -106,12 +106,15 @@ class AIPerHostRequestQueue { private: typedef std::deque queued_request_type; + int mQueuedCommands; // Number of add commands (minus remove commands) with this host in the command queue. int mAdded; // Number of active easy handles with this host. queued_request_type mQueuedRequests; // Waiting (throttled) requests. static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all AIPerHostRequestQueue objects together. public: + void added_to_command_queue(void) { ++mQueuedCommands; } + void removed_from_command_queue(void) { --mQueuedCommands; llassert(mQueuedCommands >= 0); } void added_to_multi_handle(void); // Called when an easy handle for this host has been added to the multi handle. void removed_from_multi_handle(void); // Called when an easy handle for this host is removed again from the multi handle. bool throttled(void) const; // Returns true if the maximum number of allowed requests for this host have been added to the multi handle. @@ -123,6 +126,7 @@ class AIPerHostRequestQueue { // Add queued easy handle (if any) to the multi handle. The request is removed from the queue, // followed by either a call to added_to_multi_handle() or to queue() to add it back. + S32 queued_commands(void) const { return mQueuedCommands; } S32 host_queued_plus_added_size(void) const { return mQueuedRequests.size() + mAdded; } static S32 total_queued_size(void) { return sTotalQueued; } diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 6ed1b19b7..62b76d8cb 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -1329,9 +1329,11 @@ void AICurlThread::process_commands(AICurlMultiHandle_wat const& multi_handle_w) case cmd_boost: // FIXME: future stuff break; case cmd_add: + PerHostRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerHostPtr())->removed_from_command_queue(); multi_handle_w->add_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request())); break; case cmd_remove: + PerHostRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerHostPtr())->added_to_command_queue(); // Not really, but this has the same effect as 'removed a remove command'. multi_handle_w->remove_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request()), true); break; } @@ -2386,7 +2388,9 @@ void AICurlEasyRequest::addRequest(void) // Add a command to add the new request to the multi session to the command queue. command_queue_w->commands.push_back(Command(*this, cmd_add)); command_queue_w->size++; - AICurlEasyRequest_wat(*get())->add_queued(); + AICurlEasyRequest_wat curl_easy_request_w(*get()); + PerHostRequestQueue_wat(*curl_easy_request_w->getPerHostPtr())->added_to_command_queue(); + curl_easy_request_w->add_queued(); } // Something was added to the queue, wake up the thread to get it. wakeUpCurlThread(); @@ -2448,8 +2452,10 @@ void AICurlEasyRequest::removeRequest(void) // Add a command to remove this request from the multi session to the command queue. command_queue_w->commands.push_back(Command(*this, cmd_remove)); command_queue_w->size--; + AICurlEasyRequest_wat curl_easy_request_w(*get()); + PerHostRequestQueue_wat(*curl_easy_request_w->getPerHostPtr())->removed_from_command_queue(); // Note really, but this has the same effect as 'added a remove command'. // Suppress warning that would otherwise happen if the callbacks are revoked before the curl thread removed the request. - AICurlEasyRequest_wat(*get())->remove_queued(); + curl_easy_request_w->remove_queued(); } // Something was added to the queue, wake up the thread to get it. wakeUpCurlThread(); From 79bcb1ec07c47f20f7a9b0ecd99d1b42d09461cd Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sun, 7 Apr 2013 05:58:20 +0200 Subject: [PATCH 009/119] Print info in texture console about curl request queuing This includes also non-texture requests (not sure if it's worth it to explicitely separate this data for just textures). The texture console now prints: HTTP:c/q/a/r Where, c = number of 'add' request commands in the command queue (minus the number of 'remove' request in the command queue and not taking into account the command_being_processed, nor entirely being thread-safe with regard to adding requests to 'q' or 'a' next: it is possible that a request is no longer counted in 'c' but not yet is added to 'q' or 'a'. q = number of queued (throttled) requests (by the curl thread). a = number of actually added requests (to the multi handle). r = last returned value of 'running handles' by libcurl. Obviously, c and q should be small (0 or 1) most of the time and a and r should be equal and maxed out. This turns out to be the case. --- indra/llmessage/aicurl.h | 9 +++++++++ indra/llmessage/aicurlthread.cpp | 18 ++++++++++++++++++ indra/newview/lltextureview.cpp | 8 +++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index d8aafe3b3..1b5953e7f 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -186,6 +186,15 @@ void setCAFile(std::string const& file); // Can be used to set the path to the Certificate Authority file. void setCAPath(std::string const& file); +// Returns number of queued 'add' commands minus the number of queued 'remove' commands. +U32 getNumHTTPCommands(void); + +// Returns the number of queued requests. +U32 getNumHTTPQueued(void); + +// Returns the number of curl requests currently added to the multi handle. +U32 getNumHTTPAdded(void); + // This used to be LLAppViewer::getTextureFetch()->getNumHTTPRequests(). // Returns the number of active curl easy handles (that are actually attempting to download something). U32 getNumHTTPRunning(void); diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 62b76d8cb..4cc147473 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -2505,5 +2505,23 @@ bool handleNoVerifySSLCert(LLSD const& newvalue) return true; } +U32 getNumHTTPCommands(void) +{ + using namespace AICurlPrivate; + + command_queue_rat command_queue_r(command_queue); + return command_queue_r->size; +} + +U32 getNumHTTPQueued(void) +{ + return AIPerHostRequestQueue::total_queued_size(); +} + +U32 getNumHTTPAdded(void) +{ + return AICurlPrivate::curlthread::MultiHandle::total_added_size(); +} + } // namespace AICurlInterface diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 8541dff47..9e644d51a 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -66,6 +66,9 @@ std::set LLTextureView::sDebugImages; // Forward declaration. namespace AICurlInterface { + U32 getNumHTTPCommands(void); + U32 getNumHTTPQueued(void); + U32 getNumHTTPAdded(void); U32 getNumHTTPRunning(void); } // namespace AICurlInterface @@ -598,7 +601,7 @@ void LLGLTexMemBar::draw() #endif //---------------------------------------------------------------------------- - text = llformat("Textures: %d Fetch: %d(%d) Pkts:%d(%d) Cache R/W: %d/%d LFS:%d IW:%d RAW:%d(%d) HTP:%d DEC:%d CRE:%d ", + text = llformat("Textures: %d Fetch: %d(%d) Pkts:%d(%d) Cache R/W: %d/%d LFS:%d IW:%d RAW:%d(%d) HTTP:%d/%d/%d/%d DEC:%d CRE:%d ", gTextureList.getNumImages(), LLAppViewer::getTextureFetch()->getNumRequests(), LLAppViewer::getTextureFetch()->getNumDeletes(), LLAppViewer::getTextureFetch()->mPacketCount, LLAppViewer::getTextureFetch()->mBadPacketCount, @@ -606,6 +609,9 @@ void LLGLTexMemBar::draw() LLLFSThread::sLocal->getPending(), LLAppViewer::getImageDecodeThread()->getPending(), LLImageRaw::sRawImageCount, LLImageRaw::sRawImageCachedCount, + AICurlInterface::getNumHTTPCommands(), + AICurlInterface::getNumHTTPQueued(), + AICurlInterface::getNumHTTPAdded(), AICurlInterface::getNumHTTPRunning(), LLAppViewer::getImageDecodeThread()->getPending(), gTextureList.mCreateTextureList.size()); From 9cc801f6696df3e506f85514a7fb88b6f3eb71e6 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sun, 7 Apr 2013 21:52:14 +0200 Subject: [PATCH 010/119] Added LLTextureFetchWorker::PerHostPtr Points to the AIPerHostRequestQueue instance corresponding to the hostname in LLTextureFetchWorker::mUrl. This is basically a cache as we could of course just retrieve that instance from mUrl at any time, everytime. Needed in a future commit. --- indra/newview/lltexturefetch.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index e3aed97c1..b7d5d3a2b 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -253,6 +253,7 @@ private: LLUUID mID; LLHost mHost; std::string mUrl; + AIPerHostRequestQueuePtr mPerHostPtr; // Pointer to the AIPerHostRequestQueue corresponding to the host of mUrl. U8 mType; F32 mImagePriority; U32 mWorkPriority; @@ -795,6 +796,17 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, { mCanUseNET = mUrl.empty() ; + if (!mCanUseNET) + { + // Probably a file://, but well; in that case hostname will be empty. + std::string hostname = AIPerHostRequestQueue::extract_canonical_hostname(mUrl); + if (!hostname.empty()) + { + // Make sure mPerHostPtr is up to date with mUrl. + mPerHostPtr = AIPerHostRequestQueue::instance(hostname); + } + } + calcWorkPriority(); mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL; //llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << " URL:"<< mUrl << llendl; @@ -1150,6 +1162,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. + mPerHostPtr = AIPerHostRequestQueue::instance(AIPerHostRequestQueue::extract_canonical_hostname(http_url)); } else { From 8d6f5c6ffc4a615f8b653a89f1f3fb18db046bd5 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Mon, 8 Apr 2013 19:54:07 +0200 Subject: [PATCH 011/119] Compile error fix. Cannot pass non-trivially copyable type std::string through '...'. --- indra/newview/llviewerwindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indra/newview/llviewerwindow.cpp b/indra/newview/llviewerwindow.cpp index 2088000f8..9da17bf3d 100644 --- a/indra/newview/llviewerwindow.cpp +++ b/indra/newview/llviewerwindow.cpp @@ -372,7 +372,7 @@ public: static const LLCachedControl use_rmse_auto_mask("SHUseRMSEAutoMask",false); static const LLCachedControl alpha_mas_max_rmse("SHAlphaMaskMaxRMSE",.09f); addText(xpos, ypos, llformat("Mask: %s", imagep->getIsAlphaMask(use_rmse_auto_mask ? alpha_mas_max_rmse : -1.f) ? "TRUE":"FALSE")); ypos += y_inc; - addText(xpos, ypos, llformat("ID: %s", imagep->getID().asString())); ypos += y_inc; + addText(xpos, ypos, llformat("ID: %s", imagep->getID().asString().c_str())); ypos += y_inc; } } } From 748d339ee6351d43dc885c486c3c98afdc07f203 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Mon, 8 Apr 2013 22:46:01 +0200 Subject: [PATCH 012/119] Move decision whether or not to add new HTTP request from texture fetcher to AICurl After commit things compile again :). The HTTP bandwidth throttling is not yet implemented. I'll put a temporary fix back the next commit that just does it the "old way"... --- indra/llmessage/aicurl.h | 1 + indra/llmessage/aicurlperhost.cpp | 36 ++++++- indra/llmessage/aicurlperhost.h | 18 +++- indra/llmessage/aicurlthread.cpp | 136 ++++++++++++++++++++++++ indra/newview/app_settings/settings.xml | 24 +---- indra/newview/lltexturefetch.cpp | 30 +++--- 6 files changed, 199 insertions(+), 46 deletions(-) diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index 1b5953e7f..1e1b990db 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -52,6 +52,7 @@ #include "stdtypes.h" // U16, S32, U32, F64 #include "llatomic.h" // LLAtomicU32 #include "aithreadsafe.h" +#include "aicurlperhost.h" // AIPerHostRequestQueuePtr // Debug Settings. extern bool gNoVerifySSLCert; diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperhost.cpp index 8e9da1334..1d2f157a9 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperhost.cpp @@ -38,6 +38,9 @@ AIPerHostRequestQueue::threadsafe_instance_map_type AIPerHostRequestQueue::sInstanceMap; LLAtomicS32 AIPerHostRequestQueue::sTotalQueued; +bool AIPerHostRequestQueue::sQueueEmpty; +bool AIPerHostRequestQueue::sQueueFull; +bool AIPerHostRequestQueue::sRequestStarvation; #undef AICurlPrivate @@ -221,8 +224,37 @@ void AIPerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) { multi_handle->add_easy_request(mQueuedRequests.front()); mQueuedRequests.pop_front(); - --sTotalQueued; - llassert(sTotalQueued >= 0); + llassert(sTotalQueued > 0); + if (!--sTotalQueued) + { + // We obtained a request from the queue, and after that there we no more request in any queue. + sQueueEmpty = true; + } + else + { + // We obtained a request from the queue, and even after that there was at least one more request in some queue. + sQueueFull = true; + } + if (mQueuedRequests.empty()) + { + // We obtained a request from the queue, and after that there we no more request in the queue of this host. + mQueueEmpty = true; + } + else + { + // We obtained a request from the queue, and even after that there was at least one more request in the queue of this host. + mQueueFull = true; + } + } + else + { + // We can add a new request, but there is none in the queue! + mRequestStarvation = true; + if (sTotalQueued == 0) + { + // The queue of every host is empty! + sRequestStarvation = true; + } } } diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 457b2cccd..8cfd3808b 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -85,7 +85,7 @@ class AIPerHostRequestQueue { static threadsafe_instance_map_type sInstanceMap; // Map of AIPerHostRequestQueue instances with the hostname as key. friend class AIThreadSafeSimpleDC; //threadsafe_PerHostRequestQueue - AIPerHostRequestQueue(void) : mQueuedCommands(0), mAdded(0) { } + AIPerHostRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { } public: typedef instance_map_type::iterator iterator; @@ -112,6 +112,14 @@ class AIPerHostRequestQueue { static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all AIPerHostRequestQueue objects together. + bool mQueueEmpty; // Set to true when the queue becomes precisely empty. + bool mQueueFull; // Set to true when the queue is popped and then still isn't empty; + bool mRequestStarvation; // Set to true when the queue was about to be popped but was already empty. + + static bool sQueueEmpty; // Set to true when sTotalQueued becomes precisely zero as the result of popping any queue. + static bool sQueueFull; // Set to true when sTotalQueued is still larger than zero after popping any queue. + static bool sRequestStarvation; // Set to true when any queue was about to be popped when sTotalQueued was already zero. + public: void added_to_command_queue(void) { ++mQueuedCommands; } void removed_from_command_queue(void) { --mQueuedCommands; llassert(mQueuedCommands >= 0); } @@ -126,10 +134,14 @@ class AIPerHostRequestQueue { // Add queued easy handle (if any) to the multi handle. The request is removed from the queue, // followed by either a call to added_to_multi_handle() or to queue() to add it back. - S32 queued_commands(void) const { return mQueuedCommands; } - S32 host_queued_plus_added_size(void) const { return mQueuedRequests.size() + mAdded; } + S32 pipelined_requests(void) const { return mQueuedCommands + mQueuedRequests.size() + mAdded; } static S32 total_queued_size(void) { return sTotalQueued; } + // Returns true if curl can handle another request for this host. + // Should return false if the maximum allowed HTTP bandwidth is reached, or when + // the latency between request and actual delivery becomes too large. + static bool wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host); + private: // Disallow copying. AIPerHostRequestQueue(AIPerHostRequestQueue const&) { } diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 4cc147473..64fc1b780 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -204,6 +204,9 @@ int ioctlsocket(int fd, int, unsigned long* nonblocking_enable) namespace AICurlPrivate { +LLAtomicS32 max_pipelined_requests(32); +LLAtomicS32 max_pipelined_requests_per_host(8); + enum command_st { cmd_none, cmd_add, @@ -2476,6 +2479,8 @@ void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentCo curl_max_total_concurrent_connections = CurlMaxTotalConcurrentConnections; curl_concurrent_connections_per_host = CurlConcurrentConnectionsPerHost; gNoVerifySSLCert = NoVerifySSLCert; + max_pipelined_requests = curl_max_total_concurrent_connections; + max_pipelined_requests_per_host = curl_concurrent_connections_per_host; AICurlThread::sInstance = new AICurlThread; AICurlThread::sInstance->start(); @@ -2483,9 +2488,12 @@ void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentCo bool handleCurlMaxTotalConcurrentConnections(LLSD const& newvalue) { + using namespace AICurlPrivate; using namespace AICurlPrivate::curlthread; + U32 old = curl_max_total_concurrent_connections; curl_max_total_concurrent_connections = newvalue.asInteger(); + max_pipelined_requests += curl_max_total_concurrent_connections - old; llinfos << "CurlMaxTotalConcurrentConnections set to " << curl_max_total_concurrent_connections << llendl; return true; } @@ -2494,7 +2502,9 @@ bool handleCurlConcurrentConnectionsPerHost(LLSD const& newvalue) { using namespace AICurlPrivate; + U32 old = curl_concurrent_connections_per_host; curl_concurrent_connections_per_host = newvalue.asInteger(); + max_pipelined_requests_per_host += curl_concurrent_connections_per_host - old; llinfos << "CurlConcurrentConnectionsPerHost set to " << curl_concurrent_connections_per_host << llendl; return true; } @@ -2525,3 +2535,129 @@ U32 getNumHTTPAdded(void) } // namespace AICurlInterface +// Return true if we want at least one more HTTP request for this host. +// +// It's OK if this function is a bit fuzzy, but we don't want it to return +// true a hundred times on a row when it is called fast in a loop. +// Hence the following consideration: +// +// This function is called only from LLTextureFetchWorker::doWork, and when it returns true +// then doWork will call LLHTTPClient::request with a NULL default engine (signaling that +// it is OK to run in any thread). +// +// At the end, LLHTTPClient::request calls AIStateMachine::run, which in turn calls +// AIStateMachine::reset at the end. Because NULL is passed as default_engine, reset will +// call AIStateMachine::multiplex to immediately start running the state machine. This +// causes it to go through the states bs_reset, bs_initialize and then bs_multiplex with +// run state AICurlEasyRequestStateMachine_addRequest. Finally, in this state, multiplex +// calls AICurlEasyRequestStateMachine::multiplex_impl which then calls AICurlEasyRequest::addRequest +// which causes an increment of command_queue_w->size and AIPerHostRequestQueue::mQueuedCommands. +// +// It is therefore guaranteed that in one loop of LLTextureFetchWorker::doWork, +// this size is incremented; stopping this function from returning true once we reached the +// threshold of "pipelines" requests (the sum of requests in the command queue, the ones +// throttled and queued in AIPerHostRequestQueue::mQueuedRequests and the already +// running requests (in MultiHandle::mAddedEasyRequests)). +// +//static +bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host) +{ + using namespace AICurlPrivate; + using namespace AICurlPrivate::curlthread; + + bool reject, equal, increment_threshold, decrement_threshold; + + // Whether or not we're going to approve a new request, decrement the global threshold first, when appropriate. + + // Atomic read max_pipelined_requests for the below calculations. + S32 const max_pipelined_requests_cache = max_pipelined_requests; + decrement_threshold = sQueueFull && !sQueueEmpty; + // Reset flags. + sQueueEmpty = sQueueFull = false; + if (decrement_threshold) + { + if (max_pipelined_requests_cache > curl_max_total_concurrent_connections) + { + // Decrement the threshold because since the last call to this function at least one curl request finished + // and was replaced with another request from the queue, but the queue never ran empty: we have too many + // queued requests. + --max_pipelined_requests; + } + } + + // Check if it's ok to get a new request for this particular host and update the per-host threshold. + + // Atomic read max_pipelined_requests_per_host for the below calculations. + S32 const max_pipelined_requests_per_host_cache = max_pipelined_requests_per_host; + { + PerHostRequestQueue_rat per_host_r(*per_host); + S32 const pipelined_requests_per_host = per_host_r->pipelined_requests(); + reject = pipelined_requests_per_host >= max_pipelined_requests_per_host_cache; + equal = pipelined_requests_per_host == max_pipelined_requests_per_host_cache; + increment_threshold = per_host_r->mRequestStarvation; + decrement_threshold = per_host_r->mQueueFull && !per_host_r->mQueueEmpty; + // Reset flags. + per_host_r->mQueueFull = per_host_r->mQueueEmpty = per_host_r->mRequestStarvation = false; + } + if (decrement_threshold) + { + if (max_pipelined_requests_per_host_cache > curl_concurrent_connections_per_host) + { + --max_pipelined_requests_per_host; + } + } + else if (increment_threshold && reject) + { + if (max_pipelined_requests_per_host_cache < 2 * curl_concurrent_connections_per_host) + { + max_pipelined_requests_per_host++; + // Immediately take the new threshold into account. + reject = !equal; + } + } + if (reject) + { + // Too many request for this host already. + return false; + } + +#if 0 + //AITODO: better bandwidth check here. + static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); + if (mFetcher->getTextureBandwidth() > throttle_bandwidth) + { + return false; // wait + } +#endif + + // Check if it's ok to get a new request based on the total number of requests and increment the threshold if appropriate. + + { + command_queue_rat command_queue_r(command_queue); + S32 const pipelined_requests = command_queue_r->size + sTotalQueued + MultiHandle::total_added_size(); + // We can't take the command being processed (command_being_processed) into account without + // introducing relatively long waiting times for some mutex (namely between when the command + // is moved from command_queue to command_being_processed, till it's actually being added to + // mAddedEasyRequests). The whole purpose of command_being_processed is to reduce the time + // that things are locked to micro seconds, so we'll just accept an off-by-one fuzziness + // here instead. + + // The maximum number of requests that may be queued in command_queue is equal to the total number of requests + // that may exist in the pipeline minus the number of requests queued in AIPerHostRequestQueue objects, minus + // the number of already running requests. + reject = pipelined_requests >= max_pipelined_requests_cache; + equal = pipelined_requests == max_pipelined_requests_cache; + increment_threshold = sRequestStarvation; + } + if (increment_threshold && reject) + { + if (max_pipelined_requests_cache < 2 * curl_max_total_concurrent_connections) + { + max_pipelined_requests++; + // Immediately take the new threshold into account. + reject = !equal; + } + } + return !reject; +} + diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 4de95b255..0c3da9fba 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -171,29 +171,7 @@ Value 1 - - HTTPMaxRequests - - Comment - Maximum number of simultaneous HTTP requests in progress. - Persist - 1 - Type - U32 - Value - 12 - - HTTPMinRequests - - Comment - Attempt to maintain at least this many HTTP requests in progress by ignoring bandwidth - Persist - 1 - Type - U32 - Value - 2 - + HTTPThrottleBandwidth Comment diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index b7d5d3a2b..4da161c29 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1248,29 +1248,14 @@ bool LLTextureFetchWorker::doWork(S32 param) { if(mCanUseHTTP) { - //NOTE: - //control the number of the http requests issued for: - //1, not opening too many file descriptors at the same time; - //2, control the traffic of http so udp gets bandwidth. - // - static const LLCachedControl max_http_requests("HTTPMaxRequests", 8); - static const LLCachedControl min_http_requests("HTTPMinRequests", 2); - static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); - if(((U32)mFetcher->getNumHTTPRequests() >= max_http_requests) || - ((mFetcher->getTextureBandwidth() > throttle_bandwidth) && - ((U32)mFetcher->getNumHTTPRequests() > min_http_requests))) - { - return false ; //wait. - } - - mFetcher->removeFromNetworkQueue(this, false); - S32 cur_size = 0; if (mFormattedImage.notNull()) { cur_size = mFormattedImage->getDataSize(); // amount of data we already have if (mFormattedImage->getDiscardLevel() == 0) { + // Already have all data. + mFetcher->removeFromNetworkQueue(this, false); // Note sure this is necessary, but it's what the old did --Aleric if(cur_size > 0) { // We already have all the data, just decode it @@ -1284,10 +1269,19 @@ bool LLTextureFetchWorker::doWork(S32 param) } } } + + // Let AICurl decide if we can process more HTTP requests at the moment or not. + if (!AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(mPerHostPtr)) + { + return false ; //wait. + } + + mFetcher->removeFromNetworkQueue(this, false); + mRequestedSize = mDesiredSize - cur_size; mRequestedDiscard = mDesiredDiscard; mRequestedOffset = cur_size; - + bool res = false; if (!mUrl.empty()) { From 3af89dd685312f51b3d1a8b729e4c917fba06ad3 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Mon, 8 Apr 2013 22:56:57 +0200 Subject: [PATCH 013/119] Temporarily add old bandwidth throttle method back. --- indra/llmessage/aicurlperhost.h | 2 +- indra/llmessage/aicurlthread.cpp | 9 +++------ indra/newview/lltexturefetch.cpp | 3 ++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 8cfd3808b..7a90ca2ce 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -140,7 +140,7 @@ class AIPerHostRequestQueue { // Returns true if curl can handle another request for this host. // Should return false if the maximum allowed HTTP bandwidth is reached, or when // the latency between request and actual delivery becomes too large. - static bool wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host); + static bool wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host, bool too_much_bandwidth); private: // Disallow copying. diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 64fc1b780..853b9a339 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -2560,7 +2560,7 @@ U32 getNumHTTPAdded(void) // running requests (in MultiHandle::mAddedEasyRequests)). // //static -bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host) +bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host, bool too_much_bandwidth) { using namespace AICurlPrivate; using namespace AICurlPrivate::curlthread; @@ -2621,14 +2621,11 @@ bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr co return false; } -#if 0 - //AITODO: better bandwidth check here. - static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); - if (mFetcher->getTextureBandwidth() > throttle_bandwidth) + //AIFIXME: better bandwidth check here. + if (too_much_bandwidth) { return false; // wait } -#endif // Check if it's ok to get a new request based on the total number of requests and increment the threshold if appropriate. diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 4da161c29..6cdc3f703 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1271,7 +1271,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } // Let AICurl decide if we can process more HTTP requests at the moment or not. - if (!AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(mPerHostPtr)) + static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); + if (!AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(mPerHostPtr, mFetcher->getTextureBandwidth() > throttle_bandwidth)) { return false ; //wait. } From bb948ce6d56fd8c6eb2592a68474dc0a50aac5ed Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Tue, 9 Apr 2013 04:32:36 +0200 Subject: [PATCH 014/119] Use host:port as key for the "PerHost" request queue, instead of just the hostname. Rationale: LL is doing all throttling per service (host:port), not per service hostname. Also, textures and capabilities use the same host: the sim you are connected to. Splitting the queues up on a per-service basis will stop the textures from blocking a capability request. --- indra/llmessage/aicurl.cpp | 13 +++-- indra/llmessage/aicurlperhost.cpp | 90 +++++++++++++++++++++++-------- indra/llmessage/aicurlperhost.h | 4 +- indra/llmessage/aicurlprivate.h | 7 +-- indra/llmessage/aihttptimeout.cpp | 3 +- indra/newview/lltexturefetch.cpp | 10 ++-- 6 files changed, 90 insertions(+), 37 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 9df34518e..6ca834afe 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -1114,7 +1114,7 @@ void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolic setopt(CURLOPT_HTTPHEADER, mHeaders); setoptString(CURLOPT_URL, url); llassert(!mPerHostPtr); - mLowercaseHostname = AIPerHostRequestQueue::extract_canonical_hostname(url); + mLowercaseServicename = AIPerHostRequestQueue::extract_canonical_servicename(url); mTimeoutPolicy = &policy; state_machine->setTotalDelayTimeout(policy.getTotalDelay()); // The following line is a bit tricky: we store a pointer to the object without increasing its reference count. @@ -1140,7 +1140,7 @@ void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolic // // get less connect time, while it still (also) has to wait for this DNS lookup. void CurlEasyRequest::set_timeout_opts(void) { - setopt(CURLOPT_CONNECTTIMEOUT, mTimeoutPolicy->getConnectTimeout(mLowercaseHostname)); + setopt(CURLOPT_CONNECTTIMEOUT, mTimeoutPolicy->getConnectTimeout(getLowercaseHostname())); setopt(CURLOPT_TIMEOUT, mTimeoutPolicy->getCurlTransaction()); } @@ -1241,9 +1241,9 @@ AIPerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void) if (!mPerHostPtr) { // mPerHostPtr is really just a speed-up cache. - // The reason we can cache it is because mLowercaseHostname is only set + // The reason we can cache it is because mLowercaseServicename is only set // in finalizeRequest which may only be called once: it never changes. - mPerHostPtr = AIPerHostRequestQueue::instance(mLowercaseHostname); + mPerHostPtr = AIPerHostRequestQueue::instance(mLowercaseServicename); } return mPerHostPtr; } @@ -1254,6 +1254,11 @@ bool CurlEasyRequest::removeFromPerHostQueue(AICurlEasyRequest const& easy_reque return mPerHostPtr && PerHostRequestQueue_wat(*mPerHostPtr)->cancel(easy_request); } +std::string CurlEasyRequest::getLowercaseHostname(void) const +{ + return mLowercaseServicename.substr(0, mLowercaseServicename.find_last_of(':')); +} + //----------------------------------------------------------------------------- // BufferedCurlEasyRequest diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperhost.cpp index 1d2f157a9..87d7131b4 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperhost.cpp @@ -92,40 +92,86 @@ using namespace AICurlPrivate; // - port does not contain a ':', and if it exists is always prepended by a ':'. // //static -std::string AIPerHostRequestQueue::extract_canonical_hostname(std::string const& url) +std::string AIPerHostRequestQueue::extract_canonical_servicename(std::string const& url) { - std::string::size_type pos; - std::string::size_type authority = 0; // Default if there is no sheme. - if ((pos = url.find("://")) != url.npos && pos < url.find('/')) authority = pos + 3; // Skip the "sheme://" if any, the second find is to avoid finding a "://" as part of path-abempty. - std::string::size_type host = authority; // Default if there is no userinfo. - if ((pos = url.find('@', authority)) != url.npos) host = pos + 1; // Skip the "userinfo@" if any. - authority = url.length() - 1; // Default last character of host if there is no path-abempty. - if ((pos = url.find('/', host)) != url.npos) authority = pos - 1; // Point to last character of host. - std::string::size_type len = url.find_last_not_of(":0123456789", authority) - host + 1; // Skip trailing ":port", if any. - std::string hostname(url, host, len); + char const* p = url.data(); + char const* const end = p + url.size(); + char const* sheme_colon = NULL; + char const* sheme_slash = NULL; + char const* first_ampersand = NULL; + char const* port_colon = NULL; + std::string servicename; + char const* hostname = p; // Default in the case there is no "sheme://userinfo@". + while (p < end) + { + int c = *p; + if (c == ':') + { + if (!port_colon && std::isdigit(p[1])) + { + port_colon = p; + } + else if (!sheme_colon && !sheme_slash && !first_ampersand && !port_colon) + { + // Found a colon before any slash or ampersand: this has to be the colon between the sheme and the hier-part. + sheme_colon = p; + } + } + else if (c == '/') + { + if (!sheme_slash && sheme_colon && sheme_colon == p - 1 && !first_ampersand && p[1] == '/') + { + // Found the first '/' in the first occurance of the sequence "://". + sheme_slash = p; + hostname = ++p + 1; // Point hostname to the start of the authority, the default when there is no "userinfo@" part. + servicename.clear(); // Remove the sheme. + } + else + { + // Found slash that is not part of the "sheme://" string. Signals end of authority. + // We're done. + break; + } + } + else if (c == '@') + { + if (!first_ampersand) + { + first_ampersand = p; + hostname = p + 1; + servicename.clear(); // Remove the "userinfo@" + } + } + if (p >= hostname) + { + // Convert hostname to lowercase in a way that we compare two hostnames equal iff libcurl does. #if APR_CHARSET_EBCDIC #error Not implemented #else - // Convert hostname to lowercase in a way that we compare two hostnames equal iff libcurl does. - for (std::string::iterator iter = hostname.begin(); iter != hostname.end(); ++iter) - { - int c = *iter; - if (c >= 'A' && c <= 'Z') - *iter = c + ('a' - 'A'); - } + if (c >= 'A' && c <= 'Z') + c += ('a' - 'A'); #endif - return hostname; + servicename += c; + } + ++p; + } + // Strip of any trailing ":80". + if (p - 3 == port_colon && p[-1] == '0' && p[-2] == '8') + { + return servicename.substr(0, p - hostname - 3); + } + return servicename; } //static -AIPerHostRequestQueuePtr AIPerHostRequestQueue::instance(std::string const& hostname) +AIPerHostRequestQueuePtr AIPerHostRequestQueue::instance(std::string const& servicename) { - llassert(!hostname.empty()); + llassert(!servicename.empty()); instance_map_wat instance_map_w(sInstanceMap); - AIPerHostRequestQueue::iterator iter = instance_map_w->find(hostname); + AIPerHostRequestQueue::iterator iter = instance_map_w->find(servicename); if (iter == instance_map_w->end()) { - iter = instance_map_w->insert(instance_map_type::value_type(hostname, new RefCountedThreadSafePerHostRequestQueue)).first; + iter = instance_map_w->insert(instance_map_type::value_type(servicename, new RefCountedThreadSafePerHostRequestQueue)).first; } // Note: the creation of AIPerHostRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()). return iter->second; diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperhost.h index 7a90ca2ce..3e22e4fbe 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperhost.h @@ -91,8 +91,8 @@ class AIPerHostRequestQueue { typedef instance_map_type::iterator iterator; typedef instance_map_type::const_iterator const_iterator; - // Utility function; extract canonical (lowercase) hostname from url. - static std::string extract_canonical_hostname(std::string const& url); + // Utility function; extract canonical (lowercase) hostname and port from url. + static std::string extract_canonical_servicename(std::string const& url); // Return (possibly create) a unique instance for the given hostname. static AIPerHostRequestQueuePtr instance(std::string const& hostname); diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index 9d8d51fb0..23e7e8b12 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -304,7 +304,7 @@ class CurlEasyRequest : public CurlEasyHandle { CURLcode mResult; //AIFIXME: this does not belong in the request object, but belongs in the response object. AIHTTPTimeoutPolicy const* mTimeoutPolicy; - std::string mLowercaseHostname; // Lowercase hostname (canonicalized) extracted from the url. + std::string mLowercaseServicename; // Lowercase hostname:port (canonicalized) extracted from the url. AIPerHostRequestQueuePtr mPerHostPtr; // Pointer to the corresponding AIPerHostRequestQueue. LLPointer mTimeout;// Timeout administration object associated with last created CurlSocketInfo. bool mTimeoutIsOrphan; // Set to true when mTimeout is not (yet) associated with a CurlSocketInfo. @@ -316,7 +316,8 @@ class CurlEasyRequest : public CurlEasyHandle { public: // These two are only valid after finalizeRequest. AIHTTPTimeoutPolicy const* getTimeoutPolicy(void) const { return mTimeoutPolicy; } - std::string const& getLowercaseHostname(void) const { return mLowercaseHostname; } + std::string const& getLowercaseServicename(void) const { return mLowercaseServicename; } + std::string getLowercaseHostname(void) const; // Called by CurlSocketInfo to allow access to the last (after a redirect) HTTPTimeout object related to this request. // This creates mTimeout (unless mTimeoutIsOrphan is set in which case it adopts the orphan). LLPointer& get_timeout_object(void); @@ -349,7 +350,7 @@ class CurlEasyRequest : public CurlEasyHandle { // PerHost API. AIPerHostRequestQueuePtr getPerHostPtr(void); // (Optionally create and) return a pointer to the unique - // AIPerHostRequestQueue corresponding to mLowercaseHostname. + // AIPerHostRequestQueue corresponding to mLowercaseServicename. bool removeFromPerHostQueue(AICurlEasyRequest const&) const; // Remove this request from the per-host queue, if queued at all. // Returns true if it was queued. protected: diff --git a/indra/llmessage/aihttptimeout.cpp b/indra/llmessage/aihttptimeout.cpp index 0f99e23d2..16ed7c170 100644 --- a/indra/llmessage/aihttptimeout.cpp +++ b/indra/llmessage/aihttptimeout.cpp @@ -75,6 +75,7 @@ namespace AICurlPrivate { class BufferedCurlEasyRequest { public: char const* getLowercaseHostname(void) const { return "hostname.com"; } + char const* getLowercaseServicename(void) const { return "hostname.com:12047"; } void getinfo(const int&, double* p) { *p = 0.1; } }; @@ -435,7 +436,7 @@ bool HTTPTimeout::maybe_upload_finished(void) void HTTPTimeout::print_diagnostics(CurlEasyRequest const* curl_easy_request, char const* eff_url) { #ifndef HTTPTIMEOUT_TESTSUITE - llwarns << "Request to \"" << curl_easy_request->getLowercaseHostname() << "\" timed out for " << curl_easy_request->getTimeoutPolicy()->name() << llendl; + llwarns << "Request to \"" << curl_easy_request->getLowercaseServicename() << "\" timed out for " << curl_easy_request->getTimeoutPolicy()->name() << llendl; llinfos << "Effective URL: \"" << eff_url << "\"." << llendl; double namelookup_time, connect_time, appconnect_time, pretransfer_time, starttransfer_time; curl_easy_request->getinfo(CURLINFO_NAMELOOKUP_TIME, &namelookup_time); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 6cdc3f703..dd634490e 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -798,12 +798,12 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, if (!mCanUseNET) { - // Probably a file://, but well; in that case hostname will be empty. - std::string hostname = AIPerHostRequestQueue::extract_canonical_hostname(mUrl); - if (!hostname.empty()) + // Probably a file://, but well; in that case servicename will be empty. + std::string servicename = AIPerHostRequestQueue::extract_canonical_servicename(mUrl); + if (!servicename.empty()) { // Make sure mPerHostPtr is up to date with mUrl. - mPerHostPtr = AIPerHostRequestQueue::instance(hostname); + mPerHostPtr = AIPerHostRequestQueue::instance(servicename); } } @@ -1162,7 +1162,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. - mPerHostPtr = AIPerHostRequestQueue::instance(AIPerHostRequestQueue::extract_canonical_hostname(http_url)); + mPerHostPtr = AIPerHostRequestQueue::instance(AIPerHostRequestQueue::extract_canonical_servicename(http_url)); } else { From 29a1d0e4e15ca52ddcacb4c591855293b53cc036 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Tue, 9 Apr 2013 04:40:47 +0200 Subject: [PATCH 015/119] Renamed aicurlperhost.{cpp,h} --> aicurlperservice.{cpp,h} Next commit will rename everything else. --- indra/llmessage/CMakeLists.txt | 4 ++-- indra/llmessage/aicurl.cpp | 2 +- indra/llmessage/aicurl.h | 2 +- indra/llmessage/{aicurlperhost.cpp => aicurlperservice.cpp} | 4 ++-- indra/llmessage/{aicurlperhost.h => aicurlperservice.h} | 2 +- indra/llmessage/aicurlprivate.h | 2 +- indra/llmessage/aicurlthread.cpp | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) rename indra/llmessage/{aicurlperhost.cpp => aicurlperservice.cpp} (99%) rename indra/llmessage/{aicurlperhost.h => aicurlperservice.h} (99%) diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index ebee8c548..eb45729b9 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -24,7 +24,7 @@ include_directories( set(llmessage_SOURCE_FILES aicurl.cpp aicurleasyrequeststatemachine.cpp - aicurlperhost.cpp + aicurlperservice.cpp aicurlthread.cpp aihttpheaders.cpp aihttptimeout.cpp @@ -111,7 +111,7 @@ set(llmessage_HEADER_FILES aicurl.h aicurleasyrequeststatemachine.h - aicurlperhost.h + aicurlperservice.h aicurlprivate.h aicurlthread.h aihttpheaders.h diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 6ca834afe..09d07a5c3 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -58,7 +58,7 @@ #include "aihttpheaders.h" #include "aihttptimeoutpolicy.h" #include "aicurleasyrequeststatemachine.h" -#include "aicurlperhost.h" +#include "aicurlperservice.h" //================================================================================== // Debug Settings diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index 1e1b990db..cff97cf3e 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -52,7 +52,7 @@ #include "stdtypes.h" // U16, S32, U32, F64 #include "llatomic.h" // LLAtomicU32 #include "aithreadsafe.h" -#include "aicurlperhost.h" // AIPerHostRequestQueuePtr +#include "aicurlperservice.h" // AIPerHostRequestQueuePtr // Debug Settings. extern bool gNoVerifySSLCert; diff --git a/indra/llmessage/aicurlperhost.cpp b/indra/llmessage/aicurlperservice.cpp similarity index 99% rename from indra/llmessage/aicurlperhost.cpp rename to indra/llmessage/aicurlperservice.cpp index 87d7131b4..d3a78c89c 100644 --- a/indra/llmessage/aicurlperhost.cpp +++ b/indra/llmessage/aicurlperservice.cpp @@ -1,5 +1,5 @@ /** - * @file aiperhost.cpp + * @file aiperservice.cpp * @brief Implementation of AIPerHostRequestQueue * * Copyright (c) 2012, 2013, Aleric Inglewood. @@ -33,7 +33,7 @@ */ #include "sys.h" -#include "aicurlperhost.h" +#include "aicurlperservice.h" #include "aicurlthread.h" AIPerHostRequestQueue::threadsafe_instance_map_type AIPerHostRequestQueue::sInstanceMap; diff --git a/indra/llmessage/aicurlperhost.h b/indra/llmessage/aicurlperservice.h similarity index 99% rename from indra/llmessage/aicurlperhost.h rename to indra/llmessage/aicurlperservice.h index 3e22e4fbe..47675ab3f 100644 --- a/indra/llmessage/aicurlperhost.h +++ b/indra/llmessage/aicurlperservice.h @@ -1,5 +1,5 @@ /** - * @file aicurlperhost.h + * @file aicurlperservice.h * @brief Definition of class AIPerHostRequestQueue * * Copyright (c) 2012, 2013, Aleric Inglewood. diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index 23e7e8b12..cb3fd7972 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -34,7 +34,7 @@ #include #include "llatomic.h" #include "llrefcount.h" -#include "aicurlperhost.h" +#include "aicurlperservice.h" #include "aihttptimeout.h" #include "llhttpclient.h" diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 853b9a339..6144e655a 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -32,7 +32,7 @@ #include "aicurlthread.h" #include "aihttptimeoutpolicy.h" #include "aihttptimeout.h" -#include "aicurlperhost.h" +#include "aicurlperservice.h" #include "lltimer.h" // ms_sleep, get_clock_count #include "llhttpstatuscodes.h" #include "llbuffer.h" From fce106f7e20c981a7db1085cbcde4646e2185aea Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Tue, 9 Apr 2013 05:06:32 +0200 Subject: [PATCH 016/119] PerHost became PerService Reflect the fact that we include port number in its name. --- indra/llcrashlogger/llcrashlogger.cpp | 2 +- indra/llmessage/aicurl.cpp | 22 +++---- indra/llmessage/aicurl.h | 6 +- indra/llmessage/aicurlperservice.cpp | 72 +++++++++++---------- indra/llmessage/aicurlperservice.h | 68 ++++++++++--------- indra/llmessage/aicurlprivate.h | 10 +-- indra/llmessage/aicurlthread.cpp | 86 ++++++++++++------------- indra/llmessage/aicurlthread.h | 2 +- indra/newview/app_settings/settings.xml | 4 +- indra/newview/llappviewer.cpp | 2 +- indra/newview/lltexturefetch.cpp | 12 ++-- indra/newview/llviewercontrol.cpp | 2 +- 12 files changed, 148 insertions(+), 140 deletions(-) diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index c09361144..20fde21f8 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -392,7 +392,7 @@ bool LLCrashLogger::init() // Start curl thread. AICurlInterface::startCurlThread(64, // CurlMaxTotalConcurrentConnections - 8, // CurlConcurrentConnectionsPerHost + 8, // CurlConcurrentConnectionsPerService true); // NoVerifySSLCert // We assume that all the logs we're looking for reside on the current drive diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 09d07a5c3..651e0e9eb 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -959,9 +959,9 @@ CurlEasyRequest::~CurlEasyRequest() // be available anymore. send_handle_events_to(NULL); revokeCallbacks(); - if (mPerHostPtr) + if (mPerServicePtr) { - AIPerHostRequestQueue::release(mPerHostPtr); + AIPerServiceRequestQueue::release(mPerServicePtr); } // This wasn't freed yet if the request never finished. curl_slist_free_all(mHeaders); @@ -1113,8 +1113,8 @@ void CurlEasyRequest::finalizeRequest(std::string const& url, AIHTTPTimeoutPolic #endif setopt(CURLOPT_HTTPHEADER, mHeaders); setoptString(CURLOPT_URL, url); - llassert(!mPerHostPtr); - mLowercaseServicename = AIPerHostRequestQueue::extract_canonical_servicename(url); + llassert(!mPerServicePtr); + mLowercaseServicename = AIPerServiceRequestQueue::extract_canonical_servicename(url); mTimeoutPolicy = &policy; state_machine->setTotalDelayTimeout(policy.getTotalDelay()); // The following line is a bit tricky: we store a pointer to the object without increasing its reference count. @@ -1236,22 +1236,22 @@ void CurlEasyRequest::queued_for_removal(AICurlEasyRequest_wat& curl_easy_reques } #endif -AIPerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void) +AIPerServiceRequestQueuePtr CurlEasyRequest::getPerServicePtr(void) { - if (!mPerHostPtr) + if (!mPerServicePtr) { - // mPerHostPtr is really just a speed-up cache. + // mPerServicePtr is really just a speed-up cache. // The reason we can cache it is because mLowercaseServicename is only set // in finalizeRequest which may only be called once: it never changes. - mPerHostPtr = AIPerHostRequestQueue::instance(mLowercaseServicename); + mPerServicePtr = AIPerServiceRequestQueue::instance(mLowercaseServicename); } - return mPerHostPtr; + return mPerServicePtr; } -bool CurlEasyRequest::removeFromPerHostQueue(AICurlEasyRequest const& easy_request) const +bool CurlEasyRequest::removeFromPerServiceQueue(AICurlEasyRequest const& easy_request) const { // Note that easy_request (must) represent(s) this object; it's just passed for convenience. - return mPerHostPtr && PerHostRequestQueue_wat(*mPerHostPtr)->cancel(easy_request); + return mPerServicePtr && PerServiceRequestQueue_wat(*mPerServicePtr)->cancel(easy_request); } std::string CurlEasyRequest::getLowercaseHostname(void) const diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index cff97cf3e..bc97e498f 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -52,7 +52,7 @@ #include "stdtypes.h" // U16, S32, U32, F64 #include "llatomic.h" // LLAtomicU32 #include "aithreadsafe.h" -#include "aicurlperservice.h" // AIPerHostRequestQueuePtr +#include "aicurlperservice.h" // AIPerServiceRequestQueuePtr // Debug Settings. extern bool gNoVerifySSLCert; @@ -155,7 +155,7 @@ struct Stats { // Called to handle changes in Debug Settings. bool handleCurlMaxTotalConcurrentConnections(LLSD const& newvalue); -bool handleCurlConcurrentConnectionsPerHost(LLSD const& newvalue); +bool handleCurlConcurrentConnectionsPerService(LLSD const& newvalue); bool handleNoVerifySSLCert(LLSD const& newvalue); // Called once at start of application (from newview/llappviewer.cpp by main thread (before threads are created)), @@ -163,7 +163,7 @@ bool handleNoVerifySSLCert(LLSD const& newvalue); void initCurl(void); // Called once at start of application (from LLAppViewer::initThreads), starts AICurlThread. -void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerHost, bool NoVerifySSLCert); +void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerService, bool NoVerifySSLCert); // Called once at the end of application before terminating other threads (most notably the texture thread workers) // with the purpose to stop the curl thread from doing any call backs to running responders: the responders sometimes diff --git a/indra/llmessage/aicurlperservice.cpp b/indra/llmessage/aicurlperservice.cpp index d3a78c89c..de7c16859 100644 --- a/indra/llmessage/aicurlperservice.cpp +++ b/indra/llmessage/aicurlperservice.cpp @@ -1,6 +1,6 @@ /** * @file aiperservice.cpp - * @brief Implementation of AIPerHostRequestQueue + * @brief Implementation of AIPerServiceRequestQueue * * Copyright (c) 2012, 2013, Aleric Inglewood. * @@ -30,36 +30,40 @@ * 06/04/2013 * Renamed AICurlPrivate::PerHostRequestQueue[Ptr] to AIPerHostRequestQueue[Ptr] * to allow public access. + * + * 09/04/2013 + * Renamed everything "host" to "service" and use "hostname:port" as key + * instead of just "hostname". */ #include "sys.h" #include "aicurlperservice.h" #include "aicurlthread.h" -AIPerHostRequestQueue::threadsafe_instance_map_type AIPerHostRequestQueue::sInstanceMap; -LLAtomicS32 AIPerHostRequestQueue::sTotalQueued; -bool AIPerHostRequestQueue::sQueueEmpty; -bool AIPerHostRequestQueue::sQueueFull; -bool AIPerHostRequestQueue::sRequestStarvation; +AIPerServiceRequestQueue::threadsafe_instance_map_type AIPerServiceRequestQueue::sInstanceMap; +LLAtomicS32 AIPerServiceRequestQueue::sTotalQueued; +bool AIPerServiceRequestQueue::sQueueEmpty; +bool AIPerServiceRequestQueue::sQueueFull; +bool AIPerServiceRequestQueue::sRequestStarvation; #undef AICurlPrivate namespace AICurlPrivate { -U32 curl_concurrent_connections_per_host; +U32 curl_concurrent_connections_per_service; -// Friend functions of RefCountedThreadSafePerHostRequestQueue +// Friend functions of RefCountedThreadSafePerServiceRequestQueue -void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* per_host) +void intrusive_ptr_add_ref(RefCountedThreadSafePerServiceRequestQueue* per_service) { - per_host->mReferenceCount++; + per_service->mReferenceCount++; } -void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* per_host) +void intrusive_ptr_release(RefCountedThreadSafePerServiceRequestQueue* per_service) { - if (--per_host->mReferenceCount == 0) + if (--per_service->mReferenceCount == 0) { - delete per_host; + delete per_service; } } @@ -92,7 +96,7 @@ using namespace AICurlPrivate; // - port does not contain a ':', and if it exists is always prepended by a ':'. // //static -std::string AIPerHostRequestQueue::extract_canonical_servicename(std::string const& url) +std::string AIPerServiceRequestQueue::extract_canonical_servicename(std::string const& url) { char const* p = url.data(); char const* const end = p + url.size(); @@ -164,21 +168,21 @@ std::string AIPerHostRequestQueue::extract_canonical_servicename(std::string con } //static -AIPerHostRequestQueuePtr AIPerHostRequestQueue::instance(std::string const& servicename) +AIPerServiceRequestQueuePtr AIPerServiceRequestQueue::instance(std::string const& servicename) { llassert(!servicename.empty()); instance_map_wat instance_map_w(sInstanceMap); - AIPerHostRequestQueue::iterator iter = instance_map_w->find(servicename); + AIPerServiceRequestQueue::iterator iter = instance_map_w->find(servicename); if (iter == instance_map_w->end()) { - iter = instance_map_w->insert(instance_map_type::value_type(servicename, new RefCountedThreadSafePerHostRequestQueue)).first; + iter = instance_map_w->insert(instance_map_type::value_type(servicename, new RefCountedThreadSafePerServiceRequestQueue)).first; } - // Note: the creation of AIPerHostRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()). + // Note: the creation of AIPerServiceRequestQueuePtr MUST be protected by the lock on sInstanceMap (see release()). return iter->second; } //static -void AIPerHostRequestQueue::release(AIPerHostRequestQueuePtr& instance) +void AIPerServiceRequestQueue::release(AIPerServiceRequestQueuePtr& instance) { if (instance->exactly_two_left()) // Being 'instance' and the one in sInstanceMap. { @@ -196,7 +200,7 @@ void AIPerHostRequestQueue::release(AIPerHostRequestQueuePtr& instance) return; } // The reference in the map is the last one; that means there can't be any curl easy requests queued for this host. - llassert(PerHostRequestQueue_rat(*instance)->mQueuedRequests.empty()); + llassert(PerServiceRequestQueue_rat(*instance)->mQueuedRequests.empty()); // Find the host and erase it from the map. iterator const end = instance_map_w->end(); for(iterator iter = instance_map_w->begin(); iter != end; ++iter) @@ -214,31 +218,31 @@ void AIPerHostRequestQueue::release(AIPerHostRequestQueuePtr& instance) instance.reset(); } -bool AIPerHostRequestQueue::throttled() const +bool AIPerServiceRequestQueue::throttled() const { - llassert(mAdded <= int(curl_concurrent_connections_per_host)); - return mAdded == int(curl_concurrent_connections_per_host); + llassert(mAdded <= int(curl_concurrent_connections_per_service)); + return mAdded == int(curl_concurrent_connections_per_service); } -void AIPerHostRequestQueue::added_to_multi_handle(void) +void AIPerServiceRequestQueue::added_to_multi_handle(void) { - llassert(mAdded < int(curl_concurrent_connections_per_host)); + llassert(mAdded < int(curl_concurrent_connections_per_service)); ++mAdded; } -void AIPerHostRequestQueue::removed_from_multi_handle(void) +void AIPerServiceRequestQueue::removed_from_multi_handle(void) { --mAdded; llassert(mAdded >= 0); } -void AIPerHostRequestQueue::queue(AICurlEasyRequest const& easy_request) +void AIPerServiceRequestQueue::queue(AICurlEasyRequest const& easy_request) { mQueuedRequests.push_back(easy_request.get_ptr()); sTotalQueued++; } -bool AIPerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) +bool AIPerServiceRequestQueue::cancel(AICurlEasyRequest const& easy_request) { queued_request_type::iterator const end = mQueuedRequests.end(); queued_request_type::iterator cur = std::find(mQueuedRequests.begin(), end, easy_request.get_ptr()); @@ -251,7 +255,7 @@ bool AIPerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) // the back with swap (could just swap with the end immediately, but I don't // want to break the order in which requests where added). Swap is also not // thread-safe, but OK here because it only touches the objects in the deque, - // and the deque is protected by the lock on the AIPerHostRequestQueue object. + // and the deque is protected by the lock on the AIPerServiceRequestQueue object. queued_request_type::iterator prev = cur; while (++cur != end) { @@ -264,7 +268,7 @@ bool AIPerHostRequestQueue::cancel(AICurlEasyRequest const& easy_request) return true; } -void AIPerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) +void AIPerServiceRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) { if (!mQueuedRequests.empty()) { @@ -305,15 +309,15 @@ void AIPerHostRequestQueue::add_queued_to(curlthread::MultiHandle* multi_handle) } //static -void AIPerHostRequestQueue::purge(void) +void AIPerServiceRequestQueue::purge(void) { instance_map_wat instance_map_w(sInstanceMap); for (iterator host = instance_map_w->begin(); host != instance_map_w->end(); ++host) { Dout(dc::curl, "Purging queue of host \"" << host->first << "\"."); - PerHostRequestQueue_wat per_host_w(*host->second); - size_t s = per_host_w->mQueuedRequests.size(); - per_host_w->mQueuedRequests.clear(); + PerServiceRequestQueue_wat per_service_w(*host->second); + size_t s = per_service_w->mQueuedRequests.size(); + per_service_w->mQueuedRequests.clear(); sTotalQueued -= s; llassert(sTotalQueued >= 0); } diff --git a/indra/llmessage/aicurlperservice.h b/indra/llmessage/aicurlperservice.h index 47675ab3f..cf4d8ba71 100644 --- a/indra/llmessage/aicurlperservice.h +++ b/indra/llmessage/aicurlperservice.h @@ -1,6 +1,6 @@ /** * @file aicurlperservice.h - * @brief Definition of class AIPerHostRequestQueue + * @brief Definition of class AIPerServiceRequestQueue * * Copyright (c) 2012, 2013, Aleric Inglewood. * @@ -30,10 +30,14 @@ * 06/04/2013 * Renamed AIPrivate::PerHostRequestQueue[Ptr] to AIPerHostRequestQueue[Ptr] * to allow public access. + * + * 09/04/2013 + * Renamed everything "host" to "service" and use "hostname:port" as key + * instead of just "hostname". */ -#ifndef AICURLPERHOST_H -#define AICURLPERHOST_H +#ifndef AICURLPERSERVICE_H +#define AICURLPERSERVICE_H #include "llerror.h" // llassert #include @@ -43,49 +47,49 @@ #include "aithreadsafe.h" class AICurlEasyRequest; -class AIPerHostRequestQueue; +class AIPerServiceRequestQueue; namespace AICurlPrivate { namespace curlthread { class MultiHandle; } -class RefCountedThreadSafePerHostRequestQueue; +class RefCountedThreadSafePerServiceRequestQueue; class ThreadSafeBufferedCurlEasyRequest; // Forward declaration of BufferedCurlEasyRequestPtr (see aicurlprivate.h). typedef boost::intrusive_ptr BufferedCurlEasyRequestPtr; -// AIPerHostRequestQueue objects are created by the curl thread and destructed by the main thread. +// AIPerServiceRequestQueue objects are created by the curl thread and destructed by the main thread. // We need locking. -typedef AIThreadSafeSimpleDC threadsafe_PerHostRequestQueue; -typedef AIAccessConst PerHostRequestQueue_crat; -typedef AIAccess PerHostRequestQueue_rat; -typedef AIAccess PerHostRequestQueue_wat; +typedef AIThreadSafeSimpleDC threadsafe_PerServiceRequestQueue; +typedef AIAccessConst PerServiceRequestQueue_crat; +typedef AIAccess PerServiceRequestQueue_rat; +typedef AIAccess PerServiceRequestQueue_wat; } // namespace AICurlPrivate -// We can't put threadsafe_PerHostRequestQueue in a std::map because you can't copy a mutex. +// We can't put threadsafe_PerServiceRequestQueue in a std::map because you can't copy a mutex. // Therefore, use an intrusive pointer for the threadsafe type. -typedef boost::intrusive_ptr AIPerHostRequestQueuePtr; +typedef boost::intrusive_ptr AIPerServiceRequestQueuePtr; //----------------------------------------------------------------------------- -// AIPerHostRequestQueue +// AIPerServiceRequestQueue // This class provides a static interface to create and maintain instances -// of AIPerHostRequestQueue objects, so that at any moment there is at most -// one instance per hostname. Those instances then are used to queue curl +// of AIPerServiceRequestQueue objects, so that at any moment there is at most +// one instance per hostname:port. Those instances then are used to queue curl // requests when the maximum number of connections for that host already // have been reached. -class AIPerHostRequestQueue { +class AIPerServiceRequestQueue { private: - typedef std::map instance_map_type; + typedef std::map instance_map_type; typedef AIThreadSafeSimpleDC threadsafe_instance_map_type; typedef AIAccess instance_map_rat; typedef AIAccess instance_map_wat; - static threadsafe_instance_map_type sInstanceMap; // Map of AIPerHostRequestQueue instances with the hostname as key. + static threadsafe_instance_map_type sInstanceMap; // Map of AIPerServiceRequestQueue instances with the hostname as key. - friend class AIThreadSafeSimpleDC; //threadsafe_PerHostRequestQueue - AIPerHostRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { } + friend class AIThreadSafeSimpleDC; //threadsafe_PerServiceRequestQueue + AIPerServiceRequestQueue(void) : mQueuedCommands(0), mAdded(0), mQueueEmpty(false), mQueueFull(false), mRequestStarvation(false) { } public: typedef instance_map_type::iterator iterator; @@ -95,10 +99,10 @@ class AIPerHostRequestQueue { static std::string extract_canonical_servicename(std::string const& url); // Return (possibly create) a unique instance for the given hostname. - static AIPerHostRequestQueuePtr instance(std::string const& hostname); + static AIPerServiceRequestQueuePtr instance(std::string const& servicename); // Release instance (object will be deleted if this was the last instance). - static void release(AIPerHostRequestQueuePtr& instance); + static void release(AIPerServiceRequestQueuePtr& instance); // Remove everything. Called upon viewer exit. static void purge(void); @@ -110,7 +114,7 @@ class AIPerHostRequestQueue { int mAdded; // Number of active easy handles with this host. queued_request_type mQueuedRequests; // Waiting (throttled) requests. - static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all AIPerHostRequestQueue objects together. + static LLAtomicS32 sTotalQueued; // The sum of mQueuedRequests.size() of all AIPerServiceRequestQueue objects together. bool mQueueEmpty; // Set to true when the queue becomes precisely empty. bool mQueueFull; // Set to true when the queue is popped and then still isn't empty; @@ -140,30 +144,30 @@ class AIPerHostRequestQueue { // Returns true if curl can handle another request for this host. // Should return false if the maximum allowed HTTP bandwidth is reached, or when // the latency between request and actual delivery becomes too large. - static bool wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host, bool too_much_bandwidth); + static bool wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth); private: // Disallow copying. - AIPerHostRequestQueue(AIPerHostRequestQueue const&) { } + AIPerServiceRequestQueue(AIPerServiceRequestQueue const&) { } }; namespace AICurlPrivate { -class RefCountedThreadSafePerHostRequestQueue : public threadsafe_PerHostRequestQueue { +class RefCountedThreadSafePerServiceRequestQueue : public threadsafe_PerServiceRequestQueue { public: - RefCountedThreadSafePerHostRequestQueue(void) : mReferenceCount(0) { } + RefCountedThreadSafePerServiceRequestQueue(void) : mReferenceCount(0) { } bool exactly_two_left(void) const { return mReferenceCount == 2; } private: - // Used by AIPerHostRequestQueuePtr. Object is deleted when reference count reaches zero. + // Used by AIPerServiceRequestQueuePtr. Object is deleted when reference count reaches zero. LLAtomicU32 mReferenceCount; - friend void intrusive_ptr_add_ref(RefCountedThreadSafePerHostRequestQueue* p); - friend void intrusive_ptr_release(RefCountedThreadSafePerHostRequestQueue* p); + friend void intrusive_ptr_add_ref(RefCountedThreadSafePerServiceRequestQueue* p); + friend void intrusive_ptr_release(RefCountedThreadSafePerServiceRequestQueue* p); }; -extern U32 curl_concurrent_connections_per_host; +extern U32 curl_concurrent_connections_per_service; } // namespace AICurlPrivate -#endif // AICURLPERHOST_H +#endif // AICURLPERSERVICE_H diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index cb3fd7972..d0fa202ca 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -305,7 +305,7 @@ class CurlEasyRequest : public CurlEasyHandle { AIHTTPTimeoutPolicy const* mTimeoutPolicy; std::string mLowercaseServicename; // Lowercase hostname:port (canonicalized) extracted from the url. - AIPerHostRequestQueuePtr mPerHostPtr; // Pointer to the corresponding AIPerHostRequestQueue. + AIPerServiceRequestQueuePtr mPerServicePtr; // Pointer to the corresponding AIPerServiceRequestQueue. LLPointer mTimeout;// Timeout administration object associated with last created CurlSocketInfo. bool mTimeoutIsOrphan; // Set to true when mTimeout is not (yet) associated with a CurlSocketInfo. #if defined(CWDEBUG) || defined(DEBUG_CURLIO) @@ -348,10 +348,10 @@ class CurlEasyRequest : public CurlEasyHandle { inline ThreadSafeBufferedCurlEasyRequest* get_lockobj(void); inline ThreadSafeBufferedCurlEasyRequest const* get_lockobj(void) const; - // PerHost API. - AIPerHostRequestQueuePtr getPerHostPtr(void); // (Optionally create and) return a pointer to the unique - // AIPerHostRequestQueue corresponding to mLowercaseServicename. - bool removeFromPerHostQueue(AICurlEasyRequest const&) const; // Remove this request from the per-host queue, if queued at all. + // PerService API. + AIPerServiceRequestQueuePtr getPerServicePtr(void); // (Optionally create and) return a pointer to the unique + // AIPerServiceRequestQueue corresponding to mLowercaseServicename. + bool removeFromPerServiceQueue(AICurlEasyRequest const&) const; // Remove this request from the per-host queue, if queued at all. // Returns true if it was queued. protected: // Pass events to parent. diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 6144e655a..ba6fd8507 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -205,7 +205,7 @@ int ioctlsocket(int fd, int, unsigned long* nonblocking_enable) namespace AICurlPrivate { LLAtomicS32 max_pipelined_requests(32); -LLAtomicS32 max_pipelined_requests_per_host(8); +LLAtomicS32 max_pipelined_requests_per_service(8); enum command_st { cmd_none, @@ -1332,11 +1332,11 @@ void AICurlThread::process_commands(AICurlMultiHandle_wat const& multi_handle_w) case cmd_boost: // FIXME: future stuff break; case cmd_add: - PerHostRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerHostPtr())->removed_from_command_queue(); + PerServiceRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerServicePtr())->removed_from_command_queue(); multi_handle_w->add_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request())); break; case cmd_remove: - PerHostRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerHostPtr())->added_to_command_queue(); // Not really, but this has the same effect as 'removed a remove command'. + PerServiceRequestQueue_wat(*AICurlEasyRequest_wat(*command_being_processed_r->easy_request())->getPerServicePtr())->added_to_command_queue(); // Not really, but this has the same effect as 'removed a remove command'. multi_handle_w->remove_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request()), true); break; } @@ -1578,7 +1578,7 @@ void AICurlThread::run(void) multi_handle_w->check_msg_queue(); } // Clear the queued requests. - AIPerHostRequestQueue::purge(); + AIPerServiceRequestQueue::purge(); } AICurlMultiHandle::destroyInstance(); } @@ -1705,17 +1705,17 @@ static U32 curl_max_total_concurrent_connections = 32; // Initialized on st void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) { bool throttled = true; // Default. - AIPerHostRequestQueuePtr per_host; + AIPerServiceRequestQueuePtr per_service; { AICurlEasyRequest_wat curl_easy_request_w(*easy_request); - per_host = curl_easy_request_w->getPerHostPtr(); - PerHostRequestQueue_wat per_host_w(*per_host); - if (mAddedEasyRequests.size() < curl_max_total_concurrent_connections && !per_host_w->throttled()) + per_service = curl_easy_request_w->getPerServicePtr(); + PerServiceRequestQueue_wat per_service_w(*per_service); + if (mAddedEasyRequests.size() < curl_max_total_concurrent_connections && !per_service_w->throttled()) { curl_easy_request_w->set_timeout_opts(); if (curl_easy_request_w->add_handle_to_multi(curl_easy_request_w, mMultiHandle) == CURLM_OK) { - per_host_w->added_to_multi_handle(); // (About to be) added to mAddedEasyRequests. + per_service_w->added_to_multi_handle(); // (About to be) added to mAddedEasyRequests. throttled = false; // Fall through... } } @@ -1731,7 +1731,7 @@ void MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) return; } // The request could not be added, we have to queue it. - PerHostRequestQueue_wat(*per_host)->queue(easy_request); + PerServiceRequestQueue_wat(*per_service)->queue(easy_request); #ifdef SHOW_ASSERT // Not active yet, but it's no longer an error if next we try to remove the request. AICurlEasyRequest_wat(*easy_request)->mRemovedPerCommand = false; @@ -1748,7 +1748,7 @@ CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request #ifdef SHOW_ASSERT bool removed = #endif - easy_request_w->removeFromPerHostQueue(easy_request); + easy_request_w->removeFromPerServiceQueue(easy_request); #ifdef SHOW_ASSERT if (removed) { @@ -1764,12 +1764,12 @@ CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator const& iter, bool as_per_command) { CURLMcode res; - AIPerHostRequestQueuePtr per_host; + AIPerServiceRequestQueuePtr per_service; { AICurlEasyRequest_wat curl_easy_request_w(**iter); res = curl_easy_request_w->remove_handle_from_multi(curl_easy_request_w, mMultiHandle); - per_host = curl_easy_request_w->getPerHostPtr(); - PerHostRequestQueue_wat(*per_host)->removed_from_multi_handle(); // (About to be) removed from mAddedEasyRequests. + per_service = curl_easy_request_w->getPerServicePtr(); + PerServiceRequestQueue_wat(*per_service)->removed_from_multi_handle(); // (About to be) removed from mAddedEasyRequests. #ifdef SHOW_ASSERT curl_easy_request_w->mRemovedPerCommand = as_per_command; #endif @@ -1786,7 +1786,7 @@ CURLMcode MultiHandle::remove_easy_request(addedEasyRequests_type::iterator cons #endif // Attempt to add a queued request, if any. - PerHostRequestQueue_wat(*per_host)->add_queued_to(this); + PerServiceRequestQueue_wat(*per_service)->add_queued_to(this); return res; } @@ -2392,7 +2392,7 @@ void AICurlEasyRequest::addRequest(void) command_queue_w->commands.push_back(Command(*this, cmd_add)); command_queue_w->size++; AICurlEasyRequest_wat curl_easy_request_w(*get()); - PerHostRequestQueue_wat(*curl_easy_request_w->getPerHostPtr())->added_to_command_queue(); + PerServiceRequestQueue_wat(*curl_easy_request_w->getPerServicePtr())->added_to_command_queue(); curl_easy_request_w->add_queued(); } // Something was added to the queue, wake up the thread to get it. @@ -2456,7 +2456,7 @@ void AICurlEasyRequest::removeRequest(void) command_queue_w->commands.push_back(Command(*this, cmd_remove)); command_queue_w->size--; AICurlEasyRequest_wat curl_easy_request_w(*get()); - PerHostRequestQueue_wat(*curl_easy_request_w->getPerHostPtr())->removed_from_command_queue(); // Note really, but this has the same effect as 'added a remove command'. + PerServiceRequestQueue_wat(*curl_easy_request_w->getPerServicePtr())->removed_from_command_queue(); // Note really, but this has the same effect as 'added a remove command'. // Suppress warning that would otherwise happen if the callbacks are revoked before the curl thread removed the request. curl_easy_request_w->remove_queued(); } @@ -2468,7 +2468,7 @@ void AICurlEasyRequest::removeRequest(void) namespace AICurlInterface { -void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerHost, bool NoVerifySSLCert) +void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentConnectionsPerService, bool NoVerifySSLCert) { using namespace AICurlPrivate; using namespace AICurlPrivate::curlthread; @@ -2477,10 +2477,10 @@ void startCurlThread(U32 CurlMaxTotalConcurrentConnections, U32 CurlConcurrentCo // Cache Debug Settings. curl_max_total_concurrent_connections = CurlMaxTotalConcurrentConnections; - curl_concurrent_connections_per_host = CurlConcurrentConnectionsPerHost; + curl_concurrent_connections_per_service = CurlConcurrentConnectionsPerService; gNoVerifySSLCert = NoVerifySSLCert; max_pipelined_requests = curl_max_total_concurrent_connections; - max_pipelined_requests_per_host = curl_concurrent_connections_per_host; + max_pipelined_requests_per_service = curl_concurrent_connections_per_service; AICurlThread::sInstance = new AICurlThread; AICurlThread::sInstance->start(); @@ -2498,14 +2498,14 @@ bool handleCurlMaxTotalConcurrentConnections(LLSD const& newvalue) return true; } -bool handleCurlConcurrentConnectionsPerHost(LLSD const& newvalue) +bool handleCurlConcurrentConnectionsPerService(LLSD const& newvalue) { using namespace AICurlPrivate; - U32 old = curl_concurrent_connections_per_host; - curl_concurrent_connections_per_host = newvalue.asInteger(); - max_pipelined_requests_per_host += curl_concurrent_connections_per_host - old; - llinfos << "CurlConcurrentConnectionsPerHost set to " << curl_concurrent_connections_per_host << llendl; + U32 old = curl_concurrent_connections_per_service; + curl_concurrent_connections_per_service = newvalue.asInteger(); + max_pipelined_requests_per_service += curl_concurrent_connections_per_service - old; + llinfos << "CurlConcurrentConnectionsPerService set to " << curl_concurrent_connections_per_service << llendl; return true; } @@ -2525,7 +2525,7 @@ U32 getNumHTTPCommands(void) U32 getNumHTTPQueued(void) { - return AIPerHostRequestQueue::total_queued_size(); + return AIPerServiceRequestQueue::total_queued_size(); } U32 getNumHTTPAdded(void) @@ -2551,16 +2551,16 @@ U32 getNumHTTPAdded(void) // causes it to go through the states bs_reset, bs_initialize and then bs_multiplex with // run state AICurlEasyRequestStateMachine_addRequest. Finally, in this state, multiplex // calls AICurlEasyRequestStateMachine::multiplex_impl which then calls AICurlEasyRequest::addRequest -// which causes an increment of command_queue_w->size and AIPerHostRequestQueue::mQueuedCommands. +// which causes an increment of command_queue_w->size and AIPerServiceRequestQueue::mQueuedCommands. // // It is therefore guaranteed that in one loop of LLTextureFetchWorker::doWork, // this size is incremented; stopping this function from returning true once we reached the // threshold of "pipelines" requests (the sum of requests in the command queue, the ones -// throttled and queued in AIPerHostRequestQueue::mQueuedRequests and the already +// throttled and queued in AIPerServiceRequestQueue::mQueuedRequests and the already // running requests (in MultiHandle::mAddedEasyRequests)). // //static -bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr const& per_host, bool too_much_bandwidth) +bool AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(AIPerServiceRequestQueuePtr const& per_service, bool too_much_bandwidth) { using namespace AICurlPrivate; using namespace AICurlPrivate::curlthread; @@ -2587,30 +2587,30 @@ bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr co // Check if it's ok to get a new request for this particular host and update the per-host threshold. - // Atomic read max_pipelined_requests_per_host for the below calculations. - S32 const max_pipelined_requests_per_host_cache = max_pipelined_requests_per_host; + // Atomic read max_pipelined_requests_per_service for the below calculations. + S32 const max_pipelined_requests_per_service_cache = max_pipelined_requests_per_service; { - PerHostRequestQueue_rat per_host_r(*per_host); - S32 const pipelined_requests_per_host = per_host_r->pipelined_requests(); - reject = pipelined_requests_per_host >= max_pipelined_requests_per_host_cache; - equal = pipelined_requests_per_host == max_pipelined_requests_per_host_cache; - increment_threshold = per_host_r->mRequestStarvation; - decrement_threshold = per_host_r->mQueueFull && !per_host_r->mQueueEmpty; + PerServiceRequestQueue_rat per_service_r(*per_service); + S32 const pipelined_requests_per_service = per_service_r->pipelined_requests(); + reject = pipelined_requests_per_service >= max_pipelined_requests_per_service_cache; + equal = pipelined_requests_per_service == max_pipelined_requests_per_service_cache; + increment_threshold = per_service_r->mRequestStarvation; + decrement_threshold = per_service_r->mQueueFull && !per_service_r->mQueueEmpty; // Reset flags. - per_host_r->mQueueFull = per_host_r->mQueueEmpty = per_host_r->mRequestStarvation = false; + per_service_r->mQueueFull = per_service_r->mQueueEmpty = per_service_r->mRequestStarvation = false; } if (decrement_threshold) { - if (max_pipelined_requests_per_host_cache > curl_concurrent_connections_per_host) + if (max_pipelined_requests_per_service_cache > curl_concurrent_connections_per_service) { - --max_pipelined_requests_per_host; + --max_pipelined_requests_per_service; } } else if (increment_threshold && reject) { - if (max_pipelined_requests_per_host_cache < 2 * curl_concurrent_connections_per_host) + if (max_pipelined_requests_per_service_cache < 2 * curl_concurrent_connections_per_service) { - max_pipelined_requests_per_host++; + max_pipelined_requests_per_service++; // Immediately take the new threshold into account. reject = !equal; } @@ -2640,7 +2640,7 @@ bool AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(AIPerHostRequestQueuePtr co // here instead. // The maximum number of requests that may be queued in command_queue is equal to the total number of requests - // that may exist in the pipeline minus the number of requests queued in AIPerHostRequestQueue objects, minus + // that may exist in the pipeline minus the number of requests queued in AIPerServiceRequestQueue objects, minus // the number of already running requests. reject = pipelined_requests >= max_pipelined_requests_cache; equal = pipelined_requests == max_pipelined_requests_cache; diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index 4c96b790c..44a4786af 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -81,7 +81,7 @@ class MultiHandle : public CurlMultiHandle // Store result and trigger events for easy request. void finish_easy_request(AICurlEasyRequest const& easy_request, CURLcode result); // Remove easy request at iter (must exist). - // Note that it's possible that a new request from a AIPerHostRequestQueue::mQueuedRequests is inserted before iter. + // Note that it's possible that a new request from a AIPerServiceRequestQueue::mQueuedRequests is inserted before iter. CURLMcode remove_easy_request(addedEasyRequests_type::iterator const& iter, bool as_per_command); static int socket_callback(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp); diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 0c3da9fba..f8e7203b1 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4448,10 +4448,10 @@ This should be as low as possible, but too low may break functionality Value 64 - CurlConcurrentConnectionsPerHost + CurlConcurrentConnectionsPerService Comment - Maximum number of simultaneous curl connections per host + Maximum number of simultaneous curl connections per host:port service Persist 0 Type diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 28b472c70..433aa6f41 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -1911,7 +1911,7 @@ bool LLAppViewer::initThreads() startEngineThread(); AICurlInterface::startCurlThread(gSavedSettings.getU32("CurlMaxTotalConcurrentConnections"), - gSavedSettings.getU32("CurlConcurrentConnectionsPerHost"), + gSavedSettings.getU32("CurlConcurrentConnectionsPerService"), gSavedSettings.getBOOL("NoVerifySSLCert")); LLImage::initClass(); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index dd634490e..670306119 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -253,7 +253,7 @@ private: LLUUID mID; LLHost mHost; std::string mUrl; - AIPerHostRequestQueuePtr mPerHostPtr; // Pointer to the AIPerHostRequestQueue corresponding to the host of mUrl. + AIPerServiceRequestQueuePtr mPerServicePtr; // Pointer to the AIPerServiceRequestQueue corresponding to the host of mUrl. U8 mType; F32 mImagePriority; U32 mWorkPriority; @@ -799,11 +799,11 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, if (!mCanUseNET) { // Probably a file://, but well; in that case servicename will be empty. - std::string servicename = AIPerHostRequestQueue::extract_canonical_servicename(mUrl); + std::string servicename = AIPerServiceRequestQueue::extract_canonical_servicename(mUrl); if (!servicename.empty()) { - // Make sure mPerHostPtr is up to date with mUrl. - mPerHostPtr = AIPerHostRequestQueue::instance(servicename); + // Make sure mPerServicePtr is up to date with mUrl. + mPerServicePtr = AIPerServiceRequestQueue::instance(servicename); } } @@ -1162,7 +1162,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mUrl = http_url + "/?texture_id=" + mID.asString().c_str(); mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. - mPerHostPtr = AIPerHostRequestQueue::instance(AIPerHostRequestQueue::extract_canonical_servicename(http_url)); + mPerServicePtr = AIPerServiceRequestQueue::instance(AIPerServiceRequestQueue::extract_canonical_servicename(http_url)); } else { @@ -1272,7 +1272,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // Let AICurl decide if we can process more HTTP requests at the moment or not. static const LLCachedControl throttle_bandwidth("HTTPThrottleBandwidth", 2000); - if (!AIPerHostRequestQueue::wantsMoreHTTPRequestsFor(mPerHostPtr, mFetcher->getTextureBandwidth() > throttle_bandwidth)) + if (!AIPerServiceRequestQueue::wantsMoreHTTPRequestsFor(mPerServicePtr, mFetcher->getTextureBandwidth() > throttle_bandwidth)) { return false ; //wait. } diff --git a/indra/newview/llviewercontrol.cpp b/indra/newview/llviewercontrol.cpp index 90c6a6e1a..4f0326f46 100644 --- a/indra/newview/llviewercontrol.cpp +++ b/indra/newview/llviewercontrol.cpp @@ -792,7 +792,7 @@ void settings_setup_listeners() gSavedSettings.getControl("AscentAvatarZModifier")->getSignal()->connect(boost::bind(&handleAscentAvatarModifier, _2)); gSavedSettings.getControl("CurlMaxTotalConcurrentConnections")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlMaxTotalConcurrentConnections, _2)); - gSavedSettings.getControl("CurlConcurrentConnectionsPerHost")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlConcurrentConnectionsPerHost, _2)); + gSavedSettings.getControl("CurlConcurrentConnectionsPerService")->getSignal()->connect(boost::bind(&AICurlInterface::handleCurlConcurrentConnectionsPerService, _2)); gSavedSettings.getControl("NoVerifySSLCert")->getSignal()->connect(boost::bind(&AICurlInterface::handleNoVerifySSLCert, _2)); gSavedSettings.getControl("CurlTimeoutDNSLookup")->getValidateSignal()->connect(boost::bind(&validateCurlTimeoutDNSLookup, _2)); From 734d2e658d93c109418eb63d44f010f6acb58ecc Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Tue, 9 Apr 2013 22:22:43 +0200 Subject: [PATCH 017/119] Rename HTTPTimeout::sClockCount (in clock ticks) to sTime_10ms (in 10ms units). Note that HTTPTimeout::sClockWidth is no longer used for HTTPTimeout (as if it's value became a constant of 0.01, the fraction 10ms / 1s). A new variable, HTTPTimeout::sClockWidth_10ms is used to calculate sTime_10ms (only). Also HTTPTimeout::mStalled and HTTPTimeout::mLowSpeedClock changed units to the same as of sTime_10ms (time since epoch in 10ms units). --- indra/llcommon/lltimer.cpp | 2 +- indra/llmessage/aicurlthread.cpp | 4 ++-- indra/llmessage/aihttptimeout.cpp | 21 ++++++++++----------- indra/llmessage/aihttptimeout.h | 10 +++++----- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index 7e5d648fa..e77d8b63a 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -251,7 +251,7 @@ U64 totalTime() } else { - if (current_clock_count >= gLastTotalTimeClockCount) + if (LL_LIKELY(current_clock_count >= gLastTotalTimeClockCount)) { // No wrapping, we're all okay. gTotalTimeClockCount += current_clock_count - gLastTotalTimeClockCount; diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index ba6fd8507..b624daf61 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -1545,8 +1545,8 @@ void AICurlThread::run(void) continue; } // Clock count used for timeouts. - HTTPTimeout::sClockCount = get_clock_count(); - Dout(dc::curl, "HTTPTimeout::sClockCount = " << HTTPTimeout::sClockCount); + HTTPTimeout::sTime_10ms = get_clock_count() * HTTPTimeout::sClockWidth_10ms; + Dout(dc::curl, "HTTPTimeout::sTime_10ms = " << HTTPTimeout::sTime_10ms); if (ready == 0) { multi_handle_w->socket_action(CURL_SOCKET_TIMEOUT, 0); diff --git a/indra/llmessage/aihttptimeout.cpp b/indra/llmessage/aihttptimeout.cpp index 16ed7c170..1f65b94c0 100644 --- a/indra/llmessage/aihttptimeout.cpp +++ b/indra/llmessage/aihttptimeout.cpp @@ -97,8 +97,8 @@ namespace curlthread { // HTTPTimeout //static -F64 const HTTPTimeout::sClockWidth = 1.0 / calc_clock_frequency(); // Time between two clock ticks, in seconds. -U64 HTTPTimeout::sClockCount; // Clock count, set once per select() exit. +F64 const HTTPTimeout::sClockWidth_10ms = 100.0 / calc_clock_frequency(); // Time between two clock ticks, in 10ms units. +U64 HTTPTimeout::sTime_10ms; // Time in 10ms units, set once per select() exit. // CURL-THREAD // This is called when body data was sent to the server socket. @@ -126,7 +126,7 @@ bool HTTPTimeout::data_sent(size_t n, bool finished) // | | void HTTPTimeout::reset_lowspeed(void) { - mLowSpeedClock = sClockCount; + mLowSpeedClock = sTime_10ms; mLowSpeedOn = true; mLastBytesSent = false; // We're just starting! mLastSecond = -1; // This causes lowspeed to initialize the rest. @@ -163,8 +163,8 @@ void HTTPTimeout::upload_finished(void) // We finished uploading (if there was a body to upload at all), so no more transfer rate timeouts. mLowSpeedOn = false; // Timeout if the server doesn't reply quick enough. - mStalled = sClockCount + mPolicy->getReplyDelay() / sClockWidth; - DoutCurl("upload_finished: mStalled set to sClockCount (" << sClockCount << ") + " << (mStalled - sClockCount) << " (" << mPolicy->getReplyDelay() << " seconds)"); + mStalled = sTime_10ms + 100 * mPolicy->getReplyDelay(); + DoutCurl("upload_finished: mStalled set to Time_10ms (" << sTime_10ms << ") + " << (mStalled - sTime_10ms) << " (" << mPolicy->getReplyDelay() << " seconds)"); } // CURL-THREAD @@ -231,8 +231,7 @@ bool HTTPTimeout::lowspeed(size_t bytes, bool finished) // less than low_speed_limit, we abort. // When are we? - S32 second = (sClockCount - mLowSpeedClock) * sClockWidth; - llassert(sClockWidth > 0.0); + S32 second = (sTime_10ms - mLowSpeedClock) / 100; // This REALLY should never happen, but due to another bug it did happened // and caused something so evil and hard to find that... NEVER AGAIN! llassert(second >= 0); @@ -316,8 +315,8 @@ bool HTTPTimeout::lowspeed(size_t bytes, bool finished) // Just give these bytes 4 more seconds to be written to the socket (after which we'll // assume that the 'upload finished' detection failed and we'll wait another ReplyDelay // seconds before finally, actually timing out. - mStalled = sClockCount + 4 / sClockWidth; - DoutCurl("mStalled set to sClockCount (" << sClockCount << ") + " << (mStalled - sClockCount) << " (4 seconds)"); + mStalled = sTime_10ms + 400; // 4 seconds into the future. + DoutCurl("mStalled set to sTime_10ms (" << sTime_10ms << ") + 400 (4 seconds)"); return false; } // The average transfer rate over the passed low_speed_time seconds is too low. Abort the transfer. @@ -369,8 +368,8 @@ bool HTTPTimeout::lowspeed(size_t bytes, bool finished) while(total_bytes >= mintotalbytes); } // If this function isn't called again within max_stall_time seconds, we stalled. - mStalled = sClockCount + max_stall_time / sClockWidth; - DoutCurl("mStalled set to sClockCount (" << sClockCount << ") + " << (mStalled - sClockCount) << " (" << max_stall_time << " seconds)"); + mStalled = sTime_10ms + 100 * max_stall_time; + DoutCurl("mStalled set to sTime_10ms (" << sTime_10ms << ") + " << (mStalled - sTime_10ms) << " (" << max_stall_time << " seconds)"); return false; } diff --git a/indra/llmessage/aihttptimeout.h b/indra/llmessage/aihttptimeout.h index 2c5442075..fc6db9919 100644 --- a/indra/llmessage/aihttptimeout.h +++ b/indra/llmessage/aihttptimeout.h @@ -85,11 +85,11 @@ class HTTPTimeout : public LLRefCount { S32 mLastSecond; // The time at which lowspeed() was last called, in seconds since mLowSpeedClock. S32 mOverwriteSecond; // The second at which the first bucket of this transfer will be overwritten. U32 mTotalBytes; // The sum of all bytes in mBuckets. - U64 mLowSpeedClock; // Clock count at which low speed detection (re)started. - U64 mStalled; // The clock count at which this transaction is considered to be stalling if nothing is transfered anymore. + U64 mLowSpeedClock; // The time (sTime_10ms) at which low speed detection (re)started. + U64 mStalled; // The time (sTime_10ms) at which this transaction is considered to be stalling if nothing is transfered anymore. public: - static F64 const sClockWidth; // Time between two clock ticks in seconds. - static U64 sClockCount; // Clock count used as 'now' during one loop of the main loop. + static F64 const sClockWidth_10ms; // Time between two clock ticks in 10 ms units. + static U64 sTime_10ms; // Time since the epoch in 10 ms units. #if defined(CWDEBUG) || defined(DEBUG_CURLIO) ThreadSafeBufferedCurlEasyRequest* mLockObj; #endif @@ -121,7 +121,7 @@ class HTTPTimeout : public LLRefCount { void done(AICurlEasyRequest_wat const& curlEasyRequest_w, CURLcode code); // Returns true when we REALLY timed out. Might call upload_finished heuristically. - bool has_stalled(void) { return mStalled < sClockCount && !maybe_upload_finished(); } + bool has_stalled(void) { return mStalled < sTime_10ms && !maybe_upload_finished(); } // Called from BufferedCurlEasyRequest::processOutput if a timeout occurred. void print_diagnostics(CurlEasyRequest const* curl_easy_request, char const* eff_url); From b5e70e11c461b3afa3218ec104beea50b06c6046 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 12 Apr 2013 20:55:54 +0200 Subject: [PATCH 018/119] Compile fix for libcwd --- indra/llmath/llmatrix3a.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/indra/llmath/llmatrix3a.cpp b/indra/llmath/llmatrix3a.cpp index ab077abcb..ad008e9d3 100644 --- a/indra/llmath/llmatrix3a.cpp +++ b/indra/llmath/llmatrix3a.cpp @@ -24,6 +24,7 @@ * $/LicenseInfo$ */ +#include "sys.h" #include "llmath.h" static LL_ALIGN_16(const F32 M_IDENT_3A[12]) = From 09965b0cdfd12062aee06b5bb08f26835cf28c19 Mon Sep 17 00:00:00 2001 From: Melanie Date: Mon, 15 Apr 2013 15:55:46 +0200 Subject: [PATCH 019/119] Add support for Avination's "Upload Credits" --- indra/newview/hippogridmanager.cpp | 16 ++++++- indra/newview/hippogridmanager.h | 3 ++ indra/newview/llstartup.cpp | 2 + indra/newview/llstatusbar.cpp | 48 ++++++++++++++++++- indra/newview/llstatusbar.h | 4 ++ indra/newview/llviewergenericmessage.cpp | 2 +- .../default/xui/en-us/panel_status_bar.xml | 14 ++++-- 7 files changed, 83 insertions(+), 6 deletions(-) diff --git a/indra/newview/hippogridmanager.cpp b/indra/newview/hippogridmanager.cpp index bab5eb8dd..8a4e1d8c1 100644 --- a/indra/newview/hippogridmanager.cpp +++ b/indra/newview/hippogridmanager.cpp @@ -58,7 +58,8 @@ HippoGridInfo::HippoGridInfo(const std::string& gridName) : mMaxAgentGroups(-1), mCurrencySymbol("OS$"), mRealCurrencySymbol("US$"), - mDirectoryFee(30) + mDirectoryFee(30), + mUPCSupported(false) { } @@ -676,6 +677,19 @@ void HippoGridInfo::setAutoUpdate(bool b) mAutoUpdate = b; } +bool HippoGridInfo::getUPCSupported() +{ + if(isSecondLife()) + return false; + else + return mUPCSupported; +} + +void HippoGridInfo::setUPCSupported(bool b) +{ + mUPCSupported = b; +} + // ******************************************************************** // ******************************************************************** // HippoGridManager diff --git a/indra/newview/hippogridmanager.h b/indra/newview/hippogridmanager.h index 310033af7..0e6c0f077 100644 --- a/indra/newview/hippogridmanager.h +++ b/indra/newview/hippogridmanager.h @@ -82,6 +82,8 @@ public: void setRenderCompat(bool compat); void setMaxAgentGroups(int max) { mMaxAgentGroups = max; } void setVoiceConnector(const std::string& vc) { mVoiceConnector = vc; } + void setUPCSupported(bool on); + bool getUPCSupported(); void setCurrencySymbol(const std::string& sym); void setRealCurrencySymbol(const std::string& sym); @@ -116,6 +118,7 @@ private: bool mRenderCompat; bool mInvLinks; bool mAutoUpdate; + bool mUPCSupported; int mMaxAgentGroups; std::string mCurrencySymbol; diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp index 3ecc2aa4d..7741d31a9 100644 --- a/indra/newview/llstartup.cpp +++ b/indra/newview/llstartup.cpp @@ -4385,6 +4385,8 @@ bool process_login_success_response(std::string& password) if (!tmp.empty()) gHippoGridManager->getConnectedGrid()->setMaxAgentGroups(atoi(tmp.c_str())); tmp = response["VoiceConnector"].asString(); if (!tmp.empty()) gHippoGridManager->getConnectedGrid()->setVoiceConnector(tmp); + tmp = response["upc_supported"].asString(); + if (!tmp.empty()) gHippoGridManager->getConnectedGrid()->setUPCSupported(true); gHippoGridManager->saveFile(); gHippoLimits->setLimits(); diff --git a/indra/newview/llstatusbar.cpp b/indra/newview/llstatusbar.cpp index 631527d24..0563a150c 100644 --- a/indra/newview/llstatusbar.cpp +++ b/indra/newview/llstatusbar.cpp @@ -68,6 +68,7 @@ #include "llworld.h" #include "llstatgraph.h" #include "llviewercontrol.h" +#include "llviewergenericmessage.h" #include "llviewermenu.h" // for gMenuBarView #include "llviewerparcelmgr.h" #include "llviewerthrottle.h" @@ -137,9 +138,27 @@ std::vector LLStatusBar::sDays; std::vector LLStatusBar::sMonths; const U32 LLStatusBar::MAX_DATE_STRING_LENGTH = 2000; +class LLDispatchUPCBalance : public LLDispatchHandler +{ +public: + virtual bool operator()( + const LLDispatcher* dispatcher, + const std::string& key, + const LLUUID& invoice, + const sparam_t& strings) + { + S32 upc = atoi(strings[0].c_str()); + gStatusBar->setUPC(upc); + return true; + } +}; + +static LLDispatchUPCBalance sDispatchUPCBalance; + LLStatusBar::LLStatusBar(const std::string& name, const LLRect& rect) : LLPanel(name, LLRect(), FALSE), // not mouse opaque mBalance(0), +mUPC(0), mHealth(100), mSquareMetersCredit(0), mSquareMetersCommitted(0), @@ -147,6 +166,11 @@ mRegionCrossingSlot(), mNavMeshSlot(), mIsNavMeshDirty(false) { + mUPCSupported = gHippoGridManager->getConnectedGrid()->getUPCSupported(); + + if (mUPCSupported) + gGenericDispatcher.addHandler("upcbalance", &sDispatchUPCBalance); + // status bar can possible overlay menus? setMouseOpaque(FALSE); setIsChrome(TRUE); @@ -168,10 +192,14 @@ mIsNavMeshDirty(false) mTextParcelName = getChild("ParcelNameText" ); mTextBalance = getChild("BalanceText" ); + mTextUPC = getChild("UPCText" ); mTextHealth = getChild("HealthText" ); mTextTime = getChild("TimeText" ); + if (!mUPCSupported) + mTextUPC->setVisible(false); + childSetAction("scriptout", onClickScriptDebug, this); childSetAction("health", onClickHealth, this); childSetAction("no_fly", onClickFly, this); @@ -595,6 +623,14 @@ void LLStatusBar::refresh() } // Set rects of money, buy money, time + if (mUPCSupported) + { + childGetRect("UPCText", r); + r.translate( new_right - r.mRight, 0); + childSetRect("UPCText", r); + new_right -= r.getWidth() - 18; + } + childGetRect("BalanceText", r); r.translate( new_right - r.mRight, 0); childSetRect("BalanceText", r); @@ -631,6 +667,8 @@ void LLStatusBar::refresh() void LLStatusBar::setVisibleForMouselook(bool visible) { mTextBalance->setVisible(visible); + if (mUPCSupported) + mTextUPC->setVisible(visible); mTextTime->setVisible(visible); childSetVisible("buycurrency", visible); childSetVisible("search_editor", visible); @@ -653,7 +691,7 @@ void LLStatusBar::creditBalance(S32 credit) void LLStatusBar::setBalance(S32 balance) { mTextBalance->setText(gHippoGridManager->getConnectedGrid()->getCurrencySymbol().c_str() + - LLResMgr::getInstance()->getMonetaryString(balance)); + LLResMgr::getInstance()->getMonetaryString(balance - mUPC)); if (mBalance && (fabs((F32)(mBalance - balance)) > gSavedSettings.getF32("UISndMoneyChangeThreshold"))) { @@ -670,6 +708,14 @@ void LLStatusBar::setBalance(S32 balance) } } +void LLStatusBar::setUPC(S32 upc) +{ + mTextUPC->setText("UPC " + LLResMgr::getInstance()->getMonetaryString(upc)); + + mUPC = upc; + + setBalance(mBalance); +} // static void LLStatusBar::sendMoneyBalanceRequest() diff --git a/indra/newview/llstatusbar.h b/indra/newview/llstatusbar.h index 144047166..b24913ef3 100644 --- a/indra/newview/llstatusbar.h +++ b/indra/newview/llstatusbar.h @@ -61,6 +61,7 @@ public: // MANIPULATORS void setBalance(S32 balance); + void setUPC(S32 balance); void debitBalance(S32 debit); void creditBalance(S32 credit); @@ -99,6 +100,7 @@ private: private: LLTextBox *mTextBalance; + LLTextBox *mTextUPC; LLTextBox *mTextHealth; LLTextBox *mTextTime; @@ -110,6 +112,7 @@ private: LLButton *mBtnBuyCurrency; S32 mBalance; + S32 mUPC; S32 mHealth; S32 mSquareMetersCredit; S32 mSquareMetersCommitted; @@ -118,6 +121,7 @@ private: boost::signals2::connection mRegionCrossingSlot; LLPathfindingNavMesh::navmesh_slot_t mNavMeshSlot; bool mIsNavMeshDirty; + bool mUPCSupported; static std::vector sDays; static std::vector sMonths; diff --git a/indra/newview/llviewergenericmessage.cpp b/indra/newview/llviewergenericmessage.cpp index 312fae50a..c4675e7bb 100644 --- a/indra/newview/llviewergenericmessage.cpp +++ b/indra/newview/llviewergenericmessage.cpp @@ -105,7 +105,7 @@ void process_generic_message(LLMessageSystem* msg, void**) } else if (agent_id != gAgent.getID()) { - llwarns << "GenericMessage for wrong agent" << llendl; + llwarns << "GenericMessage for wrong agent " << agent_id << llendl; return; } else diff --git a/indra/newview/skins/default/xui/en-us/panel_status_bar.xml b/indra/newview/skins/default/xui/en-us/panel_status_bar.xml index f0e6af8c1..bf1b1c213 100644 --- a/indra/newview/skins/default/xui/en-us/panel_status_bar.xml +++ b/indra/newview/skins/default/xui/en-us/panel_status_bar.xml @@ -15,21 +15,29 @@ Loading... + + UPC - N/A - +