From 69ca6cd5b227e87e15408d1908c353cc2005aea5 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 27 Jun 2012 12:57:07 +0200 Subject: [PATCH 01/11] WIP: Make curl thread code robust and flexible. Conflicts: indra/llmessage/llcurl.cpp indra/llmessage/llcurl.h indra/newview/app_settings/settings.xml indra/newview/llappviewer.cpp indra/newview/llmeshrepository.cpp Resolved: indra/llmessage/llcurl.cpp: Basically removed (not used anyway) indra/llmessage/llcurl.h: Basically removed (just includes aiculr.h now) indra/newview/app_settings/settings.xml: CurlUseMultipleThreads was remvoved. CurlMaximumNumberOfHandles and CurlRequestTimeOut are still in there, but unused at the moment. indra/newview/llappviewer.cpp: CurlMaximumNumberOfHandles and CurlRequestTimeOut are unused at the moment. indra/newview/llmeshrepository.cpp: Lock mSignal always (is unlocked inside wait()). Use mSignal lock to see if we are waiting; remove mWaiting. Return false from the MeshFetch functions iff we have to retry a HTTP fetch. Catch the error exception thrown by getByteRange instead of using it's return value (always returns true anyway). --- indra/cwdebug/debug.cc | 1 + indra/cwdebug/debug.h | 1 + indra/llcommon/llapp.cpp | 14 + indra/llcommon/llapp.h | 4 + indra/llcommon/llerrorthread.cpp | 6 +- indra/llcommon/llqueuedthread.h | 3 +- indra/llcommon/llthread.cpp | 16 +- indra/llcommon/llthread.h | 36 +- indra/llmessage/CMakeLists.txt | 6 +- indra/llmessage/aicurl.cpp | 1137 +++++++++++++ indra/llmessage/aicurl.h | 317 ++++ indra/llmessage/aicurlprivate.h | 360 ++++ indra/llmessage/aicurlthread.cpp | 1054 ++++++++++++ indra/llmessage/aicurlthread.h | 185 ++ indra/llmessage/llcurl.cpp | 1504 +---------------- indra/llmessage/llcurl.h | 476 +----- indra/llmessage/llhttpclient.cpp | 193 +-- indra/llmessage/llhttpclient.h | 4 +- indra/llmessage/llproxy.cpp | 24 +- indra/llmessage/llproxy.h | 14 +- indra/llmessage/llsdmessage.cpp | 4 +- indra/llmessage/llsdrpcclient.h | 22 +- indra/llmessage/llurlrequest.cpp | 239 +-- indra/llmessage/llurlrequest.h | 11 +- indra/newview/CMakeLists.txt | 2 + indra/newview/app_settings/settings.xml | 11 - indra/newview/hipporestrequest.cpp | 16 +- indra/newview/llagent.cpp | 2 +- indra/newview/llagentlanguage.cpp | 2 +- indra/newview/llappviewer.cpp | 18 +- indra/newview/llcurlrequest.cpp | 147 ++ indra/newview/llcurlrequest.h | 59 + indra/newview/llfloaterregioninfo.cpp | 2 +- indra/newview/llmeshrepository.cpp | 213 ++- indra/newview/llmeshrepository.h | 8 +- indra/newview/llpanelclassified.cpp | 2 +- indra/newview/llstartup.cpp | 8 +- indra/newview/lltexturefetch.cpp | 13 +- indra/newview/lltexturefetch.h | 6 +- indra/newview/llviewerparcelmedia.cpp | 2 +- indra/newview/llviewerparcelmgr.cpp | 2 +- indra/newview/llxmlrpctransaction.cpp | 273 ++- indra/newview/llxmlrpctransaction.h | 4 +- indra/newview/statemachine/CMakeLists.txt | 4 +- .../aicurleasyrequeststatemachine.cpp | 152 ++ .../aicurleasyrequeststatemachine.h | 96 ++ indra/newview/statemachine/aistatemachine.h | 2 +- 47 files changed, 4173 insertions(+), 2502 deletions(-) create mode 100644 indra/llmessage/aicurl.cpp create mode 100644 indra/llmessage/aicurl.h create mode 100644 indra/llmessage/aicurlprivate.h create mode 100644 indra/llmessage/aicurlthread.cpp create mode 100644 indra/llmessage/aicurlthread.h create mode 100644 indra/newview/llcurlrequest.cpp create mode 100644 indra/newview/llcurlrequest.h create mode 100644 indra/newview/statemachine/aicurleasyrequeststatemachine.cpp create mode 100644 indra/newview/statemachine/aicurleasyrequeststatemachine.h diff --git a/indra/cwdebug/debug.cc b/indra/cwdebug/debug.cc index 2a4f1c4fa..bd7d712f4 100644 --- a/indra/cwdebug/debug.cc +++ b/indra/cwdebug/debug.cc @@ -173,6 +173,7 @@ void stop_recording_backtraces(void) channel_ct backtrace DDCN("BACKTRACE"); //!< This debug channel is used for backtraces. channel_ct statemachine DDCN("STATEMACHINE"); //!< This debug channel is used for output related to class AIStateMachine. channel_ct caps DDCN("CAPS"); //!< This debug channel is used for output related to Capabilities. + channel_ct curl DDCN("CURL"); //!< This debug channel is used for output related to Curl. } // namespace dc } // namespace DEBUGCHANNELS diff --git a/indra/cwdebug/debug.h b/indra/cwdebug/debug.h index 786b0c52a..10750f6ab 100644 --- a/indra/cwdebug/debug.h +++ b/indra/cwdebug/debug.h @@ -118,6 +118,7 @@ extern CWD_API channel_ct sdl; extern CWD_API channel_ct backtrace; extern CWD_API channel_ct statemachine; extern CWD_API channel_ct caps; +extern CWD_API channel_ct curl; #endif diff --git a/indra/llcommon/llapp.cpp b/indra/llcommon/llapp.cpp index cb2b3c0ca..316b07e83 100644 --- a/indra/llcommon/llapp.cpp +++ b/indra/llcommon/llapp.cpp @@ -297,6 +297,20 @@ void LLApp::startErrorThread() } } +void LLApp::stopErrorThread() +{ + LLApp::setStopped(); // Signal error thread that we stopped. + int count = 0; + while (mThreadErrorp && !mThreadErrorp->isStopped() && ++count < 100) + { + ms_sleep(10); + } + if (mThreadErrorp && !mThreadErrorp->isStopped()) + { + llwarns << "Failed to stop Error Thread." << llendl; + } +} + void LLApp::setErrorHandler(LLAppErrorHandler handler) { LLApp::sErrorHandler = handler; diff --git a/indra/llcommon/llapp.h b/indra/llcommon/llapp.h index 6ad34c8fa..072f12005 100644 --- a/indra/llcommon/llapp.h +++ b/indra/llcommon/llapp.h @@ -260,6 +260,10 @@ protected: * @ brief This method is called once as soon as logging is initialized. */ void startErrorThread(); + /** + * @brief This method is called at the end, just prior to deinitializing curl. + */ + void stopErrorThread(); private: void setupErrorHandling(); // Do platform-specific error-handling setup (signals, structured exceptions) diff --git a/indra/llcommon/llerrorthread.cpp b/indra/llcommon/llerrorthread.cpp index 8476d0bfe..da6f35089 100644 --- a/indra/llcommon/llerrorthread.cpp +++ b/indra/llcommon/llerrorthread.cpp @@ -194,17 +194,17 @@ void LLErrorThread::run() if (LLApp::isError()) { // The app is in an error state, run the application's error handler. - //llinfos << "thread_error - An error has occurred, running error callback!" << llendl; + Dout(dc::notice, "thread_error - An error has occurred, running error callback!"); // Run the error handling callback LLApp::runErrorHandler(); } else { // Everything is okay, a clean exit. - //llinfos << "thread_error - Application exited cleanly" << llendl; + Dout(dc::notice, "thread_error - Application exited cleanly"); } - //llinfos << "thread_error - Exiting" << llendl; + Dout(dc::notice, "thread_error - Exiting"); LLApp::sErrorThreadRunning = FALSE; } diff --git a/indra/llcommon/llqueuedthread.h b/indra/llcommon/llqueuedthread.h index 2680072bc..73c7d98ee 100644 --- a/indra/llcommon/llqueuedthread.h +++ b/indra/llcommon/llqueuedthread.h @@ -110,6 +110,8 @@ public: return mPriority > second.mPriority; } + virtual void deleteRequest(); // Only method to delete a request + protected: status_t setStatus(status_t newstatus) { @@ -125,7 +127,6 @@ public: virtual bool processRequest() = 0; // Return true when request has completed virtual void finishRequest(bool completed); // Always called from thread after request has completed or aborted - virtual void deleteRequest(); // Only method to delete a request void setPriority(U32 pri) { diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 6f07b5482..d1c02237f 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -68,6 +68,7 @@ U32 ll_thread_local local_thread_ID = 0; U32 LLThread::sIDIter = 0; LLAtomicS32 LLThread::sCount = 0; +LLAtomicS32 LLThread::sRunning = 0; LL_COMMON_API void assert_main_thread() { @@ -119,6 +120,7 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap // the critical area of the mSignal lock)]. lldebugs << "LLThread::staticRun() Exiting: " << name << llendl; + --sRunning; // Would be better to do this after joining with the thread, but we don't join :/ return NULL; } @@ -194,6 +196,7 @@ void LLThread::start() // Set thread state to running mStatus = RUNNING; + sRunning++; apr_status_t status = apr_thread_create(&mAPRThreadp, NULL, staticRun, (void *)this, tldata().mRootPool()); @@ -205,6 +208,7 @@ void LLThread::start() } else { + --sRunning; mStatus = STOPPED; llwarns << "failed to start thread " << mName << llendl; ll_apr_warn_status(status); @@ -315,6 +319,16 @@ LL_COMMON_API bool is_main_thread(void) { return apr_os_thread_equal(main_thread // The thread private handle to access the LLThreadLocalData instance. apr_threadkey_t* LLThreadLocalData::sThreadLocalDataKey; +LLThreadLocalData::LLThreadLocalData(char const* name) : mCurlMultiHandle(NULL), mCurlErrorBuffer(NULL), mName(name) +{ +} + +LLThreadLocalData::~LLThreadLocalData() +{ + delete mCurlMultiHandle; + delete [] mCurlErrorBuffer; +} + //static void LLThreadLocalData::init(void) { @@ -348,7 +362,7 @@ void LLThreadLocalData::destroy(void* thread_local_data) //static void LLThreadLocalData::create(LLThread* threadp) { - LLThreadLocalData* new_tld = new LLThreadLocalData; + LLThreadLocalData* new_tld = new LLThreadLocalData(threadp ? threadp->mName.c_str() : "main thread"); if (threadp) { threadp->mThreadLocalData = new_tld; diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index 5724e0087..b6287bb29 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -57,6 +57,12 @@ class LLCondition; #define ll_thread_local __thread #endif +class LL_COMMON_API LLThreadLocalDataMember +{ +public: + virtual ~LLThreadLocalDataMember() { }; +}; + class LL_COMMON_API LLThreadLocalData { private: @@ -66,11 +72,18 @@ public: // Thread-local memory pool. LLAPRRootPool mRootPool; LLVolatileAPRPool mVolatileAPRPool; + LLThreadLocalDataMember* mCurlMultiHandle; // Initialized by AICurlMultiHandle::getInstance + char* mCurlErrorBuffer; // NULL, or pointing to a buffer used by libcurl. + std::string mName; // "main thread", or a copy of LLThread::mName. static void init(void); static void destroy(void* thread_local_data); static void create(LLThread* pthread); static LLThreadLocalData& tldata(void); + +private: + LLThreadLocalData(char const* name); + ~LLThreadLocalData(); }; class LL_COMMON_API LLThread @@ -78,6 +91,7 @@ class LL_COMMON_API LLThread private: static U32 sIDIter; static LLAtomicS32 sCount; + static LLAtomicS32 sRunning; public: typedef enum e_thread_status @@ -96,8 +110,9 @@ public: static U32 currentID(); // Return ID of current thread static S32 getCount() { return sCount; } + static S32 getRunning() { return sRunning; } static void yield(); // Static because it can be called by the main thread, which doesn't have an LLThread data structure. - + public: // PAUSE / RESUME functionality. See source code for important usage notes. // Called from MAIN THREAD. @@ -204,6 +219,11 @@ protected: apr_thread_mutex_t* mAPRMutexp; mutable U32 mCount; mutable U32 mLockingThread; + +private: + // Disallow copy construction and assignment. + LLMutexBase(LLMutexBase const&); + LLMutexBase& operator=(LLMutexBase const&); }; class LL_COMMON_API LLMutex : public LLMutexBase @@ -223,10 +243,6 @@ public: protected: LLAPRPool mPool; -private: - // Disable copy construction, as si teh bomb!!! -SG - LLMutex(const LLMutex&); - LLMutex& operator=(const LLMutex&); }; #if APR_HAS_THREADS @@ -380,6 +396,16 @@ public: mNoHoldersCondition.signal(); // Tell waiting readers, see [5]. mNoHoldersCondition.unlock(); // Release lock on mHoldersCount. } +#if LL_DEBUG + // Really only intended for debugging purposes: + bool isLocked(void) + { + mNoHoldersCondition.lock(); + bool res = mHoldersCount; + mNoHoldersCondition.unlock(); + return res; + } +#endif }; //============================================================================ diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index b09245ba3..fbf872c7f 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -29,7 +29,8 @@ set(llmessage_SOURCE_FILES llchainio.cpp llcircuit.cpp llclassifiedflags.cpp - llcurl.cpp + aicurl.cpp + aicurlthread.cpp lldatapacker.cpp lldispatcher.cpp llfiltersd2xmlrpc.cpp @@ -117,6 +118,9 @@ set(llmessage_HEADER_FILES llcircuit.h llclassifiedflags.h llcurl.h + aicurl.h + aicurlprivate.h + aicurlthread.h lldatapacker.h lldbstrings.h lldispatcher.h diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp new file mode 100644 index 000000000..6278c91c2 --- /dev/null +++ b/indra/llmessage/aicurl.cpp @@ -0,0 +1,1137 @@ +/** + * @file aicurl.cpp + * @brief Implementation of AICurl. + * + * Copyright (c) 2012, Aleric Inglewood. + * Copyright (C) 2010, Linden Research, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 17/03/2012 + * Initial version, written by Aleric Inglewood @ SL + * + * 20/03/2012 + * Added copyright notice for Linden Lab for those parts that were + * copied or derived from llcurl.cpp. The code of those parts are + * already in their own llcurl.cpp, so they do not ever need to + * even look at this file; the reason I added the copyright notice + * is to make clear that I am not the author of 100% of this code + * and hence I cannot change the license of it. + */ + +#include "linden_common.h" + +#define OPENSSL_THREAD_DEFINES +#include // OPENSSL_THREADS +#include + +#include "aicurl.h" +#include "llbufferstream.h" +#include "llsdserialize.h" +#include "aithreadsafe.h" +#include "llqueuedthread.h" +#include "lltimer.h" // ms_sleep +#include "llproxy.h" +#include "llhttpstatuscodes.h" +#ifdef CWDEBUG +#include +#endif + +//================================================================================== +// Local variables. +// + +namespace { + +struct CertificateAuthority { + std::string file; + std::string path; +}; + +AIThreadSafeSimpleDC gCertificateAuthority; +typedef AIAccess CertificateAuthority_wat; +typedef AIAccessConst CertificateAuthority_rat; + +enum gSSLlib_type { + ssl_unknown, + ssl_openssl, + ssl_gnutls, + ssl_nss +}; + +// No locking needed: initialized before threads are created, and subsequently only read. +gSSLlib_type gSSLlib; +bool gSetoptParamsNeedDup; + +} // namespace + +//----------------------------------------------------------------------------------- +// Needed for thread-safe openSSL operation. + +// Must be defined in global namespace. +struct CRYPTO_dynlock_value +{ + AIRWLock rwlock; +}; + +namespace { + +AIRWLock* ssl_rwlock_array; + +// OpenSSL locking function. +void ssl_locking_function(int mode, int n, char const* file, int line) +{ + if ((mode & CRYPTO_LOCK)) + { + if ((mode & CRYPTO_READ)) + ssl_rwlock_array[n].rdlock(); + else + ssl_rwlock_array[n].wrlock(); + } + else + { + if ((mode & CRYPTO_READ)) + ssl_rwlock_array[n].rdunlock(); + else + ssl_rwlock_array[n].wrunlock(); + } +} + +// OpenSSL uniq id function. +void ssl_id_function(CRYPTO_THREADID* thread_id) +{ +#if 1 // apr_os_thread_current() returns an unsigned long. + CRYPTO_THREADID_set_numeric(thread_id, apr_os_thread_current()); +#else // if it would return a pointer. + CRYPTO_THREADID_set_pointer(thread_id, apr_os_thread_current()); +#endif +} + +// OpenSSL allocate and initialize dynamic crypto lock. +CRYPTO_dynlock_value* ssl_dyn_create_function(char const* file, int line) +{ + return new CRYPTO_dynlock_value; +} + +// OpenSSL destroy dynamic crypto lock. +void ssl_dyn_destroy_function(CRYPTO_dynlock_value* l, char const* file, int line) +{ + delete l; +} + +// OpenSSL dynamic locking function. +void ssl_dyn_lock_function(int mode, CRYPTO_dynlock_value* l, char const* file, int line) +{ + if ((mode & CRYPTO_LOCK)) + { + if ((mode & CRYPTO_READ)) + l->rwlock.rdlock(); + else + l->rwlock.wrlock(); + } + else + { + if ((mode & CRYPTO_READ)) + l->rwlock.rdunlock(); + else + l->rwlock.wrunlock(); + } +} + +typedef void (*ssl_locking_function_type)(int, int, char const*, int); +typedef void (*ssl_id_function_type)(CRYPTO_THREADID*); +typedef CRYPTO_dynlock_value* (*ssl_dyn_create_function_type)(char const*, int); +typedef void (*ssl_dyn_destroy_function_type)(CRYPTO_dynlock_value*, char const*, int); +typedef void (*ssl_dyn_lock_function_type)(int, CRYPTO_dynlock_value*, char const*, int); + +ssl_locking_function_type old_ssl_locking_function; +ssl_id_function_type old_ssl_id_function; +ssl_dyn_create_function_type old_ssl_dyn_create_function; +ssl_dyn_destroy_function_type old_ssl_dyn_destroy_function; +ssl_dyn_lock_function_type old_ssl_dyn_lock_function; + +// Initialize OpenSSL library for thread-safety. +void ssl_init(void) +{ + // Static locks vector. + ssl_rwlock_array = new AIRWLock[CRYPTO_num_locks()]; + // Static locks callbacks. + old_ssl_locking_function = CRYPTO_get_locking_callback(); + old_ssl_id_function = CRYPTO_THREADID_get_callback(); + CRYPTO_set_locking_callback(&ssl_locking_function); + CRYPTO_THREADID_set_callback(&ssl_id_function); // Setting this avoids the need for a thread-local error number facility, which is hard to check. + // Dynamic locks callbacks. + old_ssl_dyn_create_function = CRYPTO_get_dynlock_create_callback(); + old_ssl_dyn_lock_function = CRYPTO_get_dynlock_lock_callback(); + old_ssl_dyn_destroy_function = CRYPTO_get_dynlock_destroy_callback(); + CRYPTO_set_dynlock_create_callback(&ssl_dyn_create_function); + CRYPTO_set_dynlock_lock_callback(&ssl_dyn_lock_function); + CRYPTO_set_dynlock_destroy_callback(&ssl_dyn_destroy_function); +} + +// Cleanup OpenSLL library thread-safety. +void ssl_cleanup(void) +{ + // Dynamic locks callbacks. + CRYPTO_set_dynlock_destroy_callback(old_ssl_dyn_destroy_function); + CRYPTO_set_dynlock_lock_callback(old_ssl_dyn_lock_function); + CRYPTO_set_dynlock_create_callback(old_ssl_dyn_create_function); + // Static locks callbacks. + CRYPTO_THREADID_set_callback(old_ssl_id_function); + CRYPTO_set_locking_callback(old_ssl_locking_function); + // Static locks vector. + delete [] ssl_rwlock_array; +} + +} // namespace openSSL +//----------------------------------------------------------------------------------- + + +//================================================================================== +// External API +// + +namespace AICurlInterface { + +#undef AICurlPrivate +using AICurlPrivate::check_easy_code; + +// MAIN-THREAD +void initCurl(F32 curl_request_timeout, S32 max_number_handles) +{ + DoutEntering(dc::curl, "AICurlInterface::initCurl(" << curl_request_timeout << ", " << max_number_handles << ")"); + + llassert(LLThread::getRunning() == 0); // We must not call curl_global_init unless we are the only thread. + CURLcode res = check_easy_code(curl_global_init(CURL_GLOBAL_ALL)); + if (res != CURLE_OK) + { + llerrs << "curl_global_init(CURL_GLOBAL_ALL) failed." << llendl; + } + + // Print version and do some feature sanity checks. + { + curl_version_info_data* version_info = curl_version_info(CURLVERSION_NOW); + + llassert_always(version_info->age >= 0); + if (version_info->age < 1) + { + llwarns << "libcurl's age is 0; no ares support." << llendl; + } + llassert_always((version_info->features & CURL_VERSION_SSL)); // SSL support, added in libcurl 7.10. + if (!(version_info->features & CURL_VERSION_ASYNCHDNS)); // Asynchronous name lookups (added in libcurl 7.10.7). + { + llwarns << "libcurl was not compiled with support for asynchronous name lookups!" << llendl; + } + + llinfos << "Successful initialization of libcurl " << + version_info->version << " (" << version_info->version_num << "), (" << + version_info->ssl_version << ", libz/" << version_info->libz_version << ")." << llendl; + + // Detect SSL library used. + gSSLlib = ssl_unknown; + std::string ssl_version(version_info->ssl_version); + if (ssl_version.find("OpenSSL") != std::string::npos) + gSSLlib = ssl_openssl; // See http://www.openssl.org/docs/crypto/threads.html#DESCRIPTION + else if (ssl_version.find("GnuTLS") != std::string::npos) + gSSLlib = ssl_gnutls; // See http://www.gnu.org/software/gnutls/manual/html_node/Thread-safety.html + else if (ssl_version.find("NSS") != std::string::npos) + gSSLlib = ssl_nss; // Supposedly thread-safe without any requirements. + + // Set up thread-safety requirements of underlaying SSL library. + // See http://curl.haxx.se/libcurl/c/libcurl-tutorial.html + switch (gSSLlib) + { + case ssl_unknown: + { + llerrs << "Unknown SSL library \"" << version_info->ssl_version << "\", required actions for thread-safe handling are unknown! Bailing out." << llendl; + } + case ssl_openssl: + { +#ifndef OPENSSL_THREADS + llerrs << "OpenSSL was not configured with thread support! Bailing out." << llendl; +#endif + ssl_init(); + } + case ssl_gnutls: + { + // I don't think we ever get here, do we? --Aleric + llassert_always(gSSLlib != ssl_gnutls); + // If we do, then didn't curl_global_init already call gnutls_global_init? + // It seems there is nothing to do for us here. + } + case ssl_nss: + { + break; // No requirements. + } + } + + gSetoptParamsNeedDup = (version_info->version_num < 0x071700); + if (gSetoptParamsNeedDup) + { + llwarns << "Your libcurl version is too old." << llendl; + } + llassert_always(!gSetoptParamsNeedDup); // Might add support later. + } +} + +// MAIN-THREAD +void cleanupCurl(void) +{ + using AICurlPrivate::curlThreadIsRunning; + + DoutEntering(dc::curl, "AICurlInterface::cleanupCurl()"); + + ssl_cleanup(); + + llassert(LLThread::getRunning() <= (curlThreadIsRunning() ? 1 : 0)); // We must not call curl_global_cleanup unless we are the only thread left. + curl_global_cleanup(); +} + +// THREAD-SAFE +std::string getVersionString(void) +{ + // libcurl is thread safe, no locking needed. + return curl_version(); +} + +// THREAD-SAFE +void setCAFile(std::string const& file) +{ + CertificateAuthority_wat CertificateAuthority_w(gCertificateAuthority); + CertificateAuthority_w->file = file; +} + +// This function is not called from anywhere, but provided as part of AICurlInterface because setCAFile is. +// THREAD-SAFE +void setCAPath(std::string const& path) +{ + CertificateAuthority_wat CertificateAuthority_w(gCertificateAuthority); + CertificateAuthority_w->path = path; +} + +// THREAD-SAFE +std::string strerror(CURLcode errorcode) +{ + // libcurl is thread safe, no locking needed. + return curl_easy_strerror(errorcode); +} + +//----------------------------------------------------------------------------- +// class Responder +// + +Responder::Responder(void) : mReferenceCount(0) +{ + DoutEntering(dc::curl, "AICurlInterface::Responder() with this = " << (void*)this); +} + +Responder::~Responder() +{ + DoutEntering(dc::curl, "AICurlInterface::Responder::~Responder() with this = " << (void*)this << "; mReferenceCount = " << mReferenceCount); + llassert(mReferenceCount == 0); +} + +void Responder::setURL(std::string const& url) +{ + // setURL is called from llhttpclient.cpp (request()), before calling any of the below (of course). + // We don't need locking here therefore; it's a case of initializing before use. + mURL = url; +} + +// Called with HTML header. +// virtual +void Responder::completedHeader(U32, std::string const&, LLSD const&) +{ + // Nothing. +} + +// Called with HTML body. +// virtual +void Responder::completedRaw(U32 status, std::string const& reason, LLChannelDescriptors const& channels, LLIOPipe::buffer_ptr_t const& buffer) +{ + LLSD content; + LLBufferStream istr(channels, buffer.get()); + if (!LLSDSerialize::fromXML(content, istr)) + { + llinfos << "Failed to deserialize LLSD. " << mURL << " [" << status << "]: " << reason << llendl; + } + + // Allow derived class to override at this point. + completed(status, reason, content); +} + +void Responder::fatalError(std::string const& reason) +{ + llwarns << "Responder::fatalError(\"" << reason << "\") is called (" << mURL << "). Passing it to Responder::completed with fake HTML error status and empty HTML body!" << llendl; + completed(U32_MAX, reason, LLSD()); +} + +// virtual +void Responder::completed(U32 status, std::string const& reason, LLSD const& content) +{ + // HTML status good? + if (200 <= status && status < 300) + { + // Allow derived class to override at this point. + result(content); + } + else + { + // Allow derived class to override at this point. + errorWithContent(status, reason, content); + } +} + +// virtual +void Responder::errorWithContent(U32 status, std::string const& reason, LLSD const&) +{ + // Allow derived class to override at this point. + error(status, reason); +} + +// virtual +void Responder::error(U32 status, std::string const& reason) +{ + llinfos << mURL << " [" << status << "]: " << reason << llendl; +} + +// virtual +void Responder::result(LLSD const&) +{ + // Nothing. +} + +// Friend functions. + +void intrusive_ptr_add_ref(Responder* responder) +{ + responder->mReferenceCount++; +} + +void intrusive_ptr_release(Responder* responder) +{ + if (--responder->mReferenceCount == 0) + { + delete responder; + } +} + +} // namespace AICurlInterface +//================================================================================== + + +//================================================================================== +// Local implementation. +// + +namespace AICurlPrivate { + +// THREAD-SAFE +CURLcode check_easy_code(CURLcode code) +{ + if (code != CURLE_OK) + { + char* error_buffer = LLThreadLocalData::tldata().mCurlErrorBuffer; + llinfos << "curl easy error detected: " << curl_easy_strerror(code); + if (error_buffer && *error_buffer != '\0') + { + llcont << ": " << error_buffer; + } + llcont << llendl; + } + return code; +} + +// THREAD-SAFE +CURLMcode check_multi_code(CURLMcode code) +{ + if (code != CURLM_OK) + { + llinfos << "curl multi error detected: " << curl_multi_strerror(code) << llendl; + } + return code; +} + +//============================================================================= +// AICurlEasyRequest (base classes) +// + +//----------------------------------------------------------------------------- +// CurlEasyHandle + +LLAtomicU32 CurlEasyHandle::sTotalEasyHandles; + +// Throws AICurlNoEasyHandle. +CurlEasyHandle::CurlEasyHandle(void) : mActiveMultiHandle(NULL), mErrorBuffer(NULL) +{ + mEasyHandle = curl_easy_init(); +#if 0 + //FIXME: for debugging, throw once every 10 times. + static int c = 0; + if (++c % 10 == 5) + { + curl_easy_cleanup(mEasyHandle); + mEasyHandle = NULL; + } +#endif + if (!mEasyHandle) + { + throw AICurlNoEasyHandle("curl_easy_init() returned NULL"); + } + sTotalEasyHandles++; +} + +CurlEasyHandle::~CurlEasyHandle() +{ + llassert(!mActiveMultiHandle); + curl_easy_cleanup(mEasyHandle); + --sTotalEasyHandles; +} + +//static +char* CurlEasyHandle::getTLErrorBuffer(void) +{ + LLThreadLocalData& tldata = LLThreadLocalData::tldata(); + if (!tldata.mCurlErrorBuffer) + { + tldata.mCurlErrorBuffer = new char[CURL_ERROR_SIZE]; + } + return tldata.mCurlErrorBuffer; +} + +void CurlEasyHandle::setErrorBuffer(void) +{ + char* error_buffer = getTLErrorBuffer(); + if (mErrorBuffer != error_buffer) + { + curl_easy_setopt(mEasyHandle, CURLOPT_ERRORBUFFER, error_buffer); + mErrorBuffer = error_buffer; + } +} + +CURLcode CurlEasyHandle::getinfo(CURLINFO info, void* data) +{ + setErrorBuffer(); + return check_easy_code(curl_easy_getinfo(mEasyHandle, info, data)); +} + +char* CurlEasyHandle::escape(char* url, int length) +{ + return curl_easy_escape(mEasyHandle, url, length); +} + +char* CurlEasyHandle::unescape(char* url, int inlength , int* outlength) +{ + return curl_easy_unescape(mEasyHandle, url, inlength, outlength); +} + +CURLcode CurlEasyHandle::perform(void) +{ + llassert(!mActiveMultiHandle); + setErrorBuffer(); + return check_easy_code(curl_easy_perform(mEasyHandle)); +} + +CURLcode CurlEasyHandle::pause(int bitmask) +{ + setErrorBuffer(); + return check_easy_code(curl_easy_pause(mEasyHandle, bitmask)); +} + +CURLMcode CurlEasyHandle::add_handle_to_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi) +{ + llassert_always(!mActiveMultiHandle && multi); + mActiveMultiHandle = multi; + CURLMcode res = check_multi_code(curl_multi_add_handle(multi, mEasyHandle)); + added_to_multi_handle(curl_easy_request_w); + return res; +} + +CURLMcode CurlEasyHandle::remove_handle_from_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi) +{ + llassert_always(mActiveMultiHandle && mActiveMultiHandle == multi); + mActiveMultiHandle = NULL; + CURLMcode res = check_multi_code(curl_multi_remove_handle(multi, mEasyHandle)); + removed_from_multi_handle(curl_easy_request_w); + return res; +} + +void intrusive_ptr_add_ref(ThreadSafeCurlEasyRequest* threadsafe_curl_easy_request) +{ + threadsafe_curl_easy_request->mReferenceCount++; +} + +void intrusive_ptr_release(ThreadSafeCurlEasyRequest* threadsafe_curl_easy_request) +{ + if (--threadsafe_curl_easy_request->mReferenceCount == 0) + { + delete threadsafe_curl_easy_request; + } +} + +//----------------------------------------------------------------------------- +// CurlEasyReqest + +void CurlEasyRequest::setoptString(CURLoption option, std::string const& value) +{ + llassert(!gSetoptParamsNeedDup); + setopt(option, value.c_str()); +} + +void CurlEasyRequest::setPost(char const* postdata, S32 size) +{ + setopt(CURLOPT_POST, 1L); + setopt(CURLOPT_POSTFIELDS, static_cast(const_cast(postdata))); + setopt(CURLOPT_POSTFIELDSIZE, size); +} + +ThreadSafeCurlEasyRequest* CurlEasyRequest::get_lockobj(void) +{ + return static_cast(AIThreadSafeSimpleDC::wrapper_cast(this)); +} + +//static +size_t CurlEasyRequest::headerCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + CurlEasyRequest* self = static_cast(userdata); + ThreadSafeCurlEasyRequest* lockobj = self->get_lockobj(); + AICurlEasyRequest_wat lock_self(*lockobj); + return self->mHeaderCallback(ptr, size, nmemb, self->mHeaderCallbackUserData); +} + +void CurlEasyRequest::setHeaderCallback(curl_write_callback callback, void* userdata) +{ + mHeaderCallback = callback; + mHeaderCallbackUserData = userdata; + setopt(CURLOPT_HEADERFUNCTION, callback ? &CurlEasyRequest::headerCallback : NULL); + setopt(CURLOPT_WRITEHEADER, userdata ? this : NULL); +} + +//static +size_t CurlEasyRequest::writeCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + CurlEasyRequest* self = static_cast(userdata); + ThreadSafeCurlEasyRequest* lockobj = self->get_lockobj(); + AICurlEasyRequest_wat lock_self(*lockobj); + return self->mWriteCallback(ptr, size, nmemb, self->mWriteCallbackUserData); +} + +void CurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) +{ + mWriteCallback = callback; + mWriteCallbackUserData = userdata; + setopt(CURLOPT_WRITEFUNCTION, callback ? &CurlEasyRequest::writeCallback : NULL); + setopt(CURLOPT_WRITEDATA, userdata ? this : NULL); +} + +//static +size_t CurlEasyRequest::readCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + CurlEasyRequest* self = static_cast(userdata); + ThreadSafeCurlEasyRequest* lockobj = self->get_lockobj(); + AICurlEasyRequest_wat lock_self(*lockobj); + return self->mReadCallback(ptr, size, nmemb, self->mReadCallbackUserData); +} + +void CurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) +{ + mReadCallback = callback; + mReadCallbackUserData = userdata; + setopt(CURLOPT_READFUNCTION, callback ? &CurlEasyRequest::readCallback : NULL); + setopt(CURLOPT_READDATA, userdata ? this : NULL); +} + +//static +CURLcode CurlEasyRequest::SSLCtxCallback(CURL* curl, void* sslctx, void* userdata) +{ + CurlEasyRequest* self = static_cast(userdata); + ThreadSafeCurlEasyRequest* lockobj = self->get_lockobj(); + AICurlEasyRequest_wat lock_self(*lockobj); + return self->mSSLCtxCallback(curl, sslctx, self->mSSLCtxCallbackUserData); +} + +void CurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata) +{ + mSSLCtxCallback = callback; + mSSLCtxCallbackUserData = userdata; + setopt(CURLOPT_SSL_CTX_FUNCTION, callback ? &CurlEasyRequest::SSLCtxCallback : NULL); + setopt(CURLOPT_SSL_CTX_DATA, userdata ? this : NULL); +} + +static size_t noHeaderCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + llwarns << "Calling noHeaderCallback(); curl session aborted." << llendl; + return 0; // Cause a CURL_WRITE_ERROR +} + +static size_t noWriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + llwarns << "Calling noWriteCallback(); curl session aborted." << llendl; + return 0; // Cause a CURL_WRITE_ERROR +} + +static size_t noReadCallback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + llwarns << "Calling noReadCallback(); curl session aborted." << llendl; + return CURL_READFUNC_ABORT; // Cause a CURLE_ABORTED_BY_CALLBACK +} + +static CURLcode noSSLCtxCallback(CURL* curl, void* sslctx, void* parm) +{ + llwarns << "Calling noSSLCtxCallback(); curl session aborted." << llendl; + return CURLE_ABORTED_BY_CALLBACK; +} + +void CurlEasyRequest::revokeCallbacks(void) +{ + if (mHeaderCallback == &noHeaderCallback && + mWriteCallback == &noWriteCallback && + mReadCallback == &noReadCallback && + mSSLCtxCallback == &noSSLCtxCallback) + { + // Already revoked. + return; + } + mHeaderCallback = &noHeaderCallback; + mWriteCallback = &noWriteCallback; + mReadCallback = &noReadCallback; + mSSLCtxCallback = &noSSLCtxCallback; + if (active()) + { + llwarns << "Revoking callbacks on a still active CurlEasyRequest object!" << llendl; + } + curl_easy_setopt(getEasyHandle(), CURLOPT_HEADERFUNCTION, &noHeaderCallback); + curl_easy_setopt(getEasyHandle(), CURLOPT_WRITEHEADER, &noWriteCallback); + curl_easy_setopt(getEasyHandle(), CURLOPT_READFUNCTION, &noReadCallback); + curl_easy_setopt(getEasyHandle(), CURLOPT_SSL_CTX_FUNCTION, &noSSLCtxCallback); +} + +CurlEasyRequest::~CurlEasyRequest() +{ + // If the CurlEasyRequest object is destructed then we need to revoke all callbacks, because + // we can't set the lock anymore, and neither will mHeaderCallback, mWriteCallback etc, + // be available anymore. + send_events_to(NULL); + revokeCallbacks(); + // This wasn't freed yet if the request never finished. + curl_slist_free_all(mHeaders); +} + +void CurlEasyRequest::resetState(void) +{ + // This function should not revoke the event call backs! + revokeCallbacks(); + reset(); + curl_slist_free_all(mHeaders); + mHeaders = NULL; + mRequestFinalized = false; + mEventsTarget = NULL; + mResult = CURLE_FAILED_INIT; + applyDefaultOptions(); +} + +void CurlEasyRequest::addHeader(char const* header) +{ + llassert(!mRequestFinalized); + mHeaders = curl_slist_append(mHeaders, header); +} + +#ifdef CWDEBUG +static int curl_debug_callback(CURL*, curl_infotype infotype, char* buf, size_t size, void* user_ptr) +{ + using namespace ::libcwd; + + CurlEasyRequest* request = (CurlEasyRequest*)user_ptr; + std::ostringstream marker; + marker << (void*)request->get_lockobj(); + libcw_do.push_marker(); + libcw_do.marker().assign(marker.str().data(), marker.str().size()); + LibcwDoutScopeBegin(LIBCWD_DEBUGCHANNELS, libcw_do, dc::curl|cond_nonewline_cf(infotype == CURLINFO_TEXT)) + switch (infotype) + { + case CURLINFO_TEXT: + LibcwDoutStream << "* "; + break; + case CURLINFO_HEADER_IN: + LibcwDoutStream << "H> "; + break; + case CURLINFO_HEADER_OUT: + LibcwDoutStream << "H< "; + break; + case CURLINFO_DATA_IN: + LibcwDoutStream << "D> "; + break; + case CURLINFO_DATA_OUT: + LibcwDoutStream << "D< "; + break; + case CURLINFO_SSL_DATA_IN: + LibcwDoutStream << "S> "; + break; + case CURLINFO_SSL_DATA_OUT: + LibcwDoutStream << "S< "; + break; + default: + LibcwDoutStream << "?? "; + } + if (infotype == CURLINFO_TEXT) + LibcwDoutStream.write(buf, size); + else if (infotype == CURLINFO_HEADER_IN || infotype == CURLINFO_HEADER_OUT) + LibcwDoutStream << libcwd::buf2str(buf, size); + else if (infotype == CURLINFO_DATA_IN || infotype == CURLINFO_DATA_OUT) + LibcwDoutStream << size << " bytes: \"" << libcwd::buf2str(buf, size) << '"'; + else + LibcwDoutStream << size << " bytes"; + LibcwDoutScopeEnd; + libcw_do.pop_marker(); + return 0; +} +#endif + +void CurlEasyRequest::applyDefaultOptions(void) +{ + CertificateAuthority_rat CertificateAuthority_r(gCertificateAuthority); + setoptString(CURLOPT_CAINFO, CertificateAuthority_r->file); + // This option forces openssl to use TLS version 1. + // The Linden Lab servers don't support later TLS versions, and libopenssl-1.0.1c has + // a bug were renegotion fails (see http://rt.openssl.org/Ticket/Display.html?id=2828), + // causing the connection to fail completely without this hack. + // For a commandline test of the same, observe the difference between: + // openssl s_client -connect login.agni.lindenlab.com:443 -CAfile packaged/app_settings/CA.pem -debug + // and + // openssl s_client -tls1 -connect login.agni.lindenlab.com:443 -CAfile packaged/app_settings/CA.pem -debug + setopt(CURLOPT_SSLVERSION, (long)CURL_SSLVERSION_TLSv1); + setopt(CURLOPT_NOSIGNAL, 1); + // The old code did this for the 'buffered' version, but I think it's nonsense. + //setopt(CURLOPT_DNS_CACHE_TIMEOUT, 0); + // Set the CURL options for either Socks or HTTP proxy. + LLProxy::getInstance()->applyProxySettings(this); + Debug( + if (dc::curl.is_on()) + { + setopt(CURLOPT_VERBOSE, 1); // Usefull for debugging. + setopt(CURLOPT_DEBUGFUNCTION, &curl_debug_callback); + setopt(CURLOPT_DEBUGDATA, this); + } + ); +} + +void CurlEasyRequest::finalizeRequest(std::string const& url) +{ + llassert(!mRequestFinalized); + mResult = CURLE_FAILED_INIT; // General error code, the final code is written here in MultiHandle::check_run_count when msg is CURLMSG_DONE. + mRequestFinalized = true; + lldebugs << url << llendl; + setopt(CURLOPT_HTTPHEADER, mHeaders); + setoptString(CURLOPT_URL, url); + // The following line is a bit tricky: we store a pointer to the object without increasing it's reference count. + // Of course we could increment the reference count, but that doesn't help much: if then this pointer would + // get "lost" we'd have a memory leak. Either way we must make sure that it is impossible that this pointer + // will be used if the object is deleted [In fact, since this is finalizeRequest() and not addRequest(), + // incrementing the reference counter would be wrong: if addRequest is never called then the object is + // destroyed shortly after and this pointer is never even used.] + // This pointer is used in MultiHandle::check_run_count, which means that addRequest() was called and + // the reference counter was increased and the object is being kept alive, see the comments above + // command_queue in aicurlthread.cpp. In fact, this object survived until MultiHandle::add_easy_request + // was called and is kept alive by MultiHandle::mAddedEasyRequests. The only way to get deleted after + // that is when MultiHandle::remove_easy_request is called, which first removes the easy handle from + // the multi handle. So that it's (hopefully) no longer possible that info_read() in + // MultiHandle::check_run_count returns this easy handle, after the object is destroyed by deleting + // it from MultiHandle::mAddedEasyRequests. + setopt(CURLOPT_PRIVATE, get_lockobj()); +} + +void CurlEasyRequest::getTransferInfo(AICurlInterface::TransferInfo* info) +{ + // Curl explicitly demands a double for these info's. + double size, total_time, speed; + getinfo(CURLINFO_SIZE_DOWNLOAD, &size); + getinfo(CURLINFO_TOTAL_TIME, &total_time); + getinfo(CURLINFO_SPEED_DOWNLOAD, &speed); + // Convert to F64. + info->mSizeDownload = size; + info->mTotalTime = total_time; + info->mSpeedDownload = speed; +} + +void CurlEasyRequest::getResult(CURLcode* result, AICurlInterface::TransferInfo* info) +{ + *result = mResult; + if (info && mResult != CURLE_FAILED_INIT) + { + getTransferInfo(info); + } +} + +void CurlEasyRequest::added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) +{ + if (mEventsTarget) + mEventsTarget->added_to_multi_handle(curl_easy_request_w); +} + +void CurlEasyRequest::finished(AICurlEasyRequest_wat& curl_easy_request_w) +{ + if (mEventsTarget) + mEventsTarget->finished(curl_easy_request_w); +} + +void CurlEasyRequest::removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) +{ + if (mEventsTarget) + mEventsTarget->removed_from_multi_handle(curl_easy_request_w); +} + +//----------------------------------------------------------------------------- +// CurlResponderBuffer + +static unsigned int const MAX_REDIRECTS = 5; +static S32 const CURL_REQUEST_TIMEOUT = 30; // Seconds per operation. + +LLChannelDescriptors const CurlResponderBuffer::sChannels; + +CurlResponderBuffer::CurlResponderBuffer() +{ + ThreadSafeBufferedCurlEasyRequest* lockobj = get_lockobj(); + AICurlEasyRequest_wat curl_easy_request_w(*lockobj); + curl_easy_request_w->send_events_to(this); +} + +// The callbacks need to be revoked when the CurlResponderBuffer is destructed (because that is what the callbacks use). +// The AIThreadSafeSimple is destructed first (right to left), so when we get here then the +// ThreadSafeCurlEasyRequest base class of ThreadSafeBufferedCurlEasyRequest is still intact and we can create +// and use curl_easy_request_w. +CurlResponderBuffer::~CurlResponderBuffer() +{ + ThreadSafeBufferedCurlEasyRequest* lockobj = get_lockobj(); + AICurlEasyRequest_wat curl_easy_request_w(*lockobj); // Wait till possible callbacks have returned. + curl_easy_request_w->send_events_to(NULL); + curl_easy_request_w->revokeCallbacks(); + if (mResponder) + { + llwarns << "Calling ~CurlResponderBuffer() with active responder!" << llendl; + llassert(false); // Does this ever happen? And if so, what does it mean? + // FIXME: Does this really mean it timed out? + mResponder->completedRaw(HTTP_REQUEST_TIME_OUT, "Request timeout, aborted.", sChannels, mOutput); + mResponder = NULL; + } +} + +void CurlResponderBuffer::resetState(AICurlEasyRequest_wat& curl_easy_request_w) +{ + if (mResponder) + { + llwarns << "Calling CurlResponderBuffer::resetState() for active easy handle!" << llendl; + llassert(false); // Does this ever happen? And if so, what does it mean? + // FIXME: Does this really mean it timed out? + mResponder->completedRaw(HTTP_REQUEST_TIME_OUT, "Request timeout, aborted.", sChannels, mOutput); + mResponder = NULL; + } + + curl_easy_request_w->resetState(); + + mOutput.reset(); + + mInput.str(""); + mInput.clear(); + + mHeaderOutput.str(""); + mHeaderOutput.clear(); +} + +ThreadSafeBufferedCurlEasyRequest* CurlResponderBuffer::get_lockobj(void) +{ + return static_cast(AIThreadSafeSimple::wrapper_cast(this)); +} + +void CurlResponderBuffer::prepRequest(AICurlEasyRequest_wat& curl_easy_request_w, std::vector const& headers, AICurlInterface::ResponderPtr responder, S32 time_out, bool post) +{ + if (post) + { + curl_easy_request_w->setoptString(CURLOPT_ENCODING, ""); + } + + mOutput.reset(new LLBufferArray); + mOutput->setThreaded(true); + + ThreadSafeBufferedCurlEasyRequest* lockobj = get_lockobj(); + curl_easy_request_w->setWriteCallback(&curlWriteCallback, lockobj); + curl_easy_request_w->setReadCallback(&curlReadCallback, lockobj); + curl_easy_request_w->setHeaderCallback(&curlHeaderCallback, lockobj); + + // Allow up to five redirects. + if (responder && responder->followRedir()) + { + curl_easy_request_w->setopt(CURLOPT_FOLLOWLOCATION, 1); + curl_easy_request_w->setopt(CURLOPT_MAXREDIRS, MAX_REDIRECTS); + } + + curl_easy_request_w->setopt(CURLOPT_SSL_VERIFYPEER, true); + // Don't verify host name so urls with scrubbed host names will work (improves DNS performance). + curl_easy_request_w->setopt(CURLOPT_SSL_VERIFYHOST, 0); + + curl_easy_request_w->setopt(CURLOPT_TIMEOUT, llmax(time_out, CURL_REQUEST_TIMEOUT)); + + // Keep responder alive. + mResponder = responder; + + if (!post) + { + curl_easy_request_w->addHeader("Connection: keep-alive"); + curl_easy_request_w->addHeader("Keep-alive: 300"); + // Add other headers. + for (std::vector::const_iterator iter = headers.begin(); iter != headers.end(); ++iter) + { + curl_easy_request_w->addHeader((*iter).c_str()); + } + } +} + +//static +size_t CurlResponderBuffer::curlWriteCallback(char* data, size_t size, size_t nmemb, void* user_data) +{ + ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); + + // We need to lock the curl easy request object too, because that lock is used + // to make sure that callbacks and destruction aren't done simulaneously. + AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); + + AICurlResponderBuffer_wat buffer_w(*lockobj); + S32 n = size * nmemb; + buffer_w->getOutput()->append(sChannels.in(), (U8 const*)data, n); + return n; +} + +//static +size_t CurlResponderBuffer::curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) +{ + ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); + + // We need to lock the curl easy request object too, because that lock is used + // to make sure that callbacks and destruction aren't done simulaneously. + AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); + + AICurlResponderBuffer_wat buffer_w(*lockobj); + S32 n = size * nmemb; + S32 startpos = buffer_w->getInput().tellg(); + buffer_w->getInput().seekg(0, std::ios::end); + S32 endpos = buffer_w->getInput().tellg(); + buffer_w->getInput().seekg(startpos, std::ios::beg); + S32 maxn = endpos - startpos; + n = llmin(n, maxn); + buffer_w->getInput().read(data, n); + return n; +} + +//static +size_t CurlResponderBuffer::curlHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data) +{ + ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); + + // We need to lock the curl easy request object too, because that lock is used + // to make sure that callbacks and destruction aren't done simulaneously. + AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); + + AICurlResponderBuffer_wat buffer_w(*lockobj); + size_t n = size * nmemb; + buffer_w->getHeaderOutput().write(data, n); + return n; +} + +void CurlResponderBuffer::added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) +{ + Dout(dc::curl, "Calling CurlResponderBuffer::added_to_multi_handle(@" << (void*)&*curl_easy_request_w << ") for this = " << (void*)this); +} + +void CurlResponderBuffer::finished(AICurlEasyRequest_wat& curl_easy_request_w) +{ + Dout(dc::curl, "Calling CurlResponderBuffer::finished(@" << (void*)&*curl_easy_request_w << ") for this = " << (void*)this); +} + +void CurlResponderBuffer::removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) +{ + Dout(dc::curl, "Calling CurlResponderBuffer::removed_from_multi_handle(@" << (void*)&*curl_easy_request_w << ") for this = " << (void*)this); + + // Lock self. + ThreadSafeBufferedCurlEasyRequest* lockobj = get_lockobj(); + llassert(dynamic_cast(static_cast(ThreadSafeCurlEasyRequest::wrapper_cast(&*curl_easy_request_w))) == lockobj); + AICurlResponderBuffer_wat buffer_w(*lockobj); + llassert(&*buffer_w == this); + + processOutput(curl_easy_request_w); +} + +void CurlResponderBuffer::processOutput(AICurlEasyRequest_wat& curl_easy_request_w) +{ + U32 responseCode = 0; + std::string responseReason; + + CURLcode code; + curl_easy_request_w->getResult(&code); + if (code == CURLE_OK) + { + curl_easy_request_w->getinfo(CURLINFO_RESPONSE_CODE, &responseCode); + //*TODO: get reason from first line of mHeaderOutput + } + else + { + responseCode = 499; + responseReason = AICurlInterface::strerror(code) + " : "; + if (code == CURLE_FAILED_INIT) + { + responseReason += "Curl Easy Handle initialization failed."; + } + else + { + responseReason += curl_easy_request_w->getErrorString(); + } + curl_easy_request_w->setopt(CURLOPT_FRESH_CONNECT, TRUE); + } + + if (mResponder) + { + mResponder->completedRaw(responseCode, responseReason, sChannels, mOutput); + mResponder = NULL; + } + + resetState(curl_easy_request_w); +} + +//----------------------------------------------------------------------------- +// CurlMultiHandle + +LLAtomicU32 CurlMultiHandle::sTotalMultiHandles; + +CurlMultiHandle::CurlMultiHandle(void) +{ + mMultiHandle = curl_multi_init(); + if (!mMultiHandle) + { + throw AICurlNoMultiHandle("curl_multi_init() returned NULL"); + } + sTotalMultiHandles++; +} + +CurlMultiHandle::~CurlMultiHandle() +{ + curl_multi_cleanup(mMultiHandle); + --sTotalMultiHandles; +} + +} // namespace AICurlPrivate diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h new file mode 100644 index 000000000..ffff2a01e --- /dev/null +++ b/indra/llmessage/aicurl.h @@ -0,0 +1,317 @@ +/** + * @file aicurl.h + * @brief Thread safe wrapper for libcurl. + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 17/03/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AICURL_H +#define AICURL_H + +#include +#include +#include +#include +#include +#include +#include // CURL, CURLM, CURLMcode, CURLoption, curl_*_callback + +// Make sure we don't use this option: it is not thread-safe. +#undef CURLOPT_DNS_USE_GLOBAL_CACHE + +#include "stdtypes.h" // U32 +#include "lliopipe.h" // LLIOPipe::buffer_ptr_t +#include "llatomic.h" // LLAtomicU32 +#include "aithreadsafe.h" + +class LLSD; + +//----------------------------------------------------------------------------- +// Exceptions. +// + +// A general curl exception. +// +class AICurlError : public std::runtime_error { + public: + AICurlError(std::string const& message) : std::runtime_error(message) { } +}; + +class AICurlNoEasyHandle : public AICurlError { + public: + AICurlNoEasyHandle(std::string const& message) : AICurlError(message) { } +}; + +class AICurlNoMultiHandle : public AICurlError { + public: + AICurlNoMultiHandle(std::string const& message) : AICurlError(message) { } +}; + +// End Exceptions. +//----------------------------------------------------------------------------- + +// Things defined in this namespace are called from elsewhere in the viewer code. +namespace AICurlInterface { + +// Output parameter of AICurlPrivate::CurlEasyRequest::getResult. +// Only used by LLXMLRPCTransaction::Impl. +struct TransferInfo { + TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) { } + F64 mSizeDownload; + F64 mTotalTime; + F64 mSpeedDownload; +}; + +//----------------------------------------------------------------------------- +// Global functions. + +// Called once at start of application (from newview/llappviewer.cpp by main thread (before threads are created)), +// with main purpose to initialize curl. +void initCurl(F32 curl_request_timeout = 120.f, S32 max_number_handles = 256); + +// Called once at start of application (from LLAppViewer::initThreads), starts AICurlThread. +void startCurlThread(void); + +// Called once at end of application (from newview/llappviewer.cpp by main thread), +// with purpose to stop curl threads, free curl resources and deinitialize curl. +void cleanupCurl(void); + +// Called from indra/llmessage/llurlrequest.cpp to print debug output regarding +// an error code returned by EasyRequest::getResult. +// Just returns curl_easy_strerror(errorcode). +std::string strerror(CURLcode errorcode); + +// Called from indra/newview/llfloaterabout.cpp for the About floater, and +// from newview/llappviewer.cpp in behalf of debug output. +// Just returns curl_version(). +std::string getVersionString(void); + +// Called from newview/llappviewer.cpp (and llcrashlogger/llcrashlogger.cpp) to set +// the Certificate Authority file used to verify HTTPS certs. +void setCAFile(std::string const& file); + +// Not called from anywhere. +// Can be used to set the path to the Certificate Authority file. +void setCAPath(std::string const& file); + +//----------------------------------------------------------------------------- +// Global classes. + +// Responder - base class for Request::get* and Request::post API. +// +// The life cycle of classes derived from this class is as follows: +// They are allocated with new on the line where get(), getByteRange() or post() is called, +// and the pointer to the allocated object is then put in a reference counting ResponderPtr. +// This ResponderPtr is passed to CurlResponderBuffer::prepRequest which stores it in it's +// member mResponder. Hence, the life time of a Responder is never longer than it's +// associated CurlResponderBuffer, however, if everything works correct, then normally a +// responder is deleted in CurlResponderBuffer::removed_from_multi_handle by setting +// mReponder to NULL. +// +// Note that the life time of CurlResponderBuffer is (a bit) shorter than the associated +// CurlEasyRequest (because of the order of base classes of ThreadSafeBufferedCurlEasyRequest) +// and the callbacks, as set by prepRequest, only use those two. +// A callback locks the CurlEasyRequest before actually making the callback, and the +// destruction of CurlResponderBuffer also first locks the CurlEasyRequest, and then revokes +// the callbacks. This assures that a Responder is never used when the objects it uses are +// destructed. Also, if any of those are destructed then the responder is automatically +// destructed too. +// +class Responder { + protected: + Responder(void); + virtual ~Responder(); + + private: + // Associated URL, used for debug output. + std::string mURL; + + public: + // Called to set the url of the current request for this responder, + // used only when printing debug output regarding activity of the responder. + void setURL(std::string const& url); + + public: + // Called from LLHTTPClientURLAdaptor::complete(): + + // Derived classes can override this to get the HTML header that was received, when the message is completed. + // The default does nothing. + virtual void completedHeader(U32 status, std::string const& reason, LLSD const& content); + + // Derived classes can override this to get the raw data of the body of the HTML message that was received. + // The default is to interpret the content as LLSD and call completed(). + virtual void completedRaw(U32 status, std::string const& reason, LLChannelDescriptors const& channels, LLIOPipe::buffer_ptr_t const& buffer); + + // Called from LLHTTPClient request calls, if an error occurs even before we can call one of the above. + // It calls completed() with a fake status U32_MAX, as that is what some derived clients expect (bad design). + // This means that if a derived class overrides completedRaw() it now STILL has to override completed() to catch this error. + void fatalError(std::string const& reason); + + // A derived class should return true if curl should follow redirections. + // The default is not to follow redirections. + virtual bool followRedir(void) { return false; } + + protected: + // ... or, derived classes can override this to get the LLSD content when the message is completed. + // The default is to call result() (or errorWithContent() in case of a HTML status indicating an error). + virtual void completed(U32 status, std::string const& reason, LLSD const& content); + + // ... or, derived classes can override this to received the content of a body upon success. + // The default does nothing. + virtual void result(LLSD const& content); + + // Derived classes can override this to get informed when a bad HTML status code is received. + // The default calls error(). + virtual void errorWithContent(U32 status, std::string const& reason, LLSD const& content); + + // ... or, derived classes can override this to get informed when a bad HTML statis code is received. + // The default prints the error to llinfos. + virtual void error(U32 status, std::string const& reason); + + public: + // Called from LLSDMessage::ResponderAdapter::listener. + // LLSDMessage::ResponderAdapter is a hack, showing among others by fact that these function needs to be public. + + void pubErrorWithContent(U32 status, std::string const& reason, LLSD const& content) { errorWithContent(status, reason, content); } + void pubResult(LLSD const& content) { result(content); } + + private: + // Used by ResponderPtr. Object is deleted when reference count reaches zero. + LLAtomicU32 mReferenceCount; + + friend void intrusive_ptr_add_ref(Responder* p); // Called by boost::intrusive_ptr when a new copy of a boost::intrusive_ptr is made. + friend void intrusive_ptr_release(Responder* p); // Called by boost::intrusive_ptr when a boost::intrusive_ptr is destroyed. + // This function must delete the Responder object when the reference count reaches zero. +}; + +// A Responder is passed around as ResponderPtr, which causes it to automatically +// destruct when there are no pointers left pointing to it. +typedef boost::intrusive_ptr ResponderPtr; + +} // namespace AICurlInterface + +// Forward declaration (see aicurlprivate.h). +namespace AICurlPrivate { + class CurlEasyRequest; +} // namespace AICurlPrivate + +// Define access types (_crat = Const Read Access Type, _rat = Read Access Type, _wat = Write Access Type). +// Typical usage is: +// AICurlEasyRequest h1; // Create easy handle. +// AICurlEasyRequest h2(h1); // Make lightweight copies. +// AICurlEasyRequest_wat h2_w(*h2); // Lock and obtain write access to the easy handle. +// Use *h2_w, which is a reference to the locked CurlEasyRequest instance. +// Note: As it is not allowed to use curl easy handles in any way concurrently, +// read access would at most give access to a CURL const*, which will turn out +// to be completely useless; therefore it is sufficient and efficient to use +// an AIThreadSafeSimple and it's unlikely that AICurlEasyRequest_rat will be used. +typedef AIAccessConst AICurlEasyRequest_rat; +typedef AIAccess AICurlEasyRequest_wat; + +// Events generated by AICurlPrivate::CurlEasyHandle. +struct AICurlEasyHandleEvents { + // Events. + virtual void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0; + virtual void finished(AICurlEasyRequest_wat& curl_easy_request_w) = 0; + virtual void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0; +}; + +#include "aicurlprivate.h" + +// AICurlPrivate::CurlEasyRequestPtr, a boost::intrusive_ptr, is no more threadsafe than a +// builtin type, but wrapping it in AIThreadSafe is obviously not going to help here. +// Therefore we use the following trick: we wrap CurlEasyRequestPtr too, and only allow +// read accesses on it. + +// AICurlEasyRequest: a thread safe, reference counting, auto cleaning curl easy handle. +class AICurlEasyRequest { + public: + // Initial construction is allowed (thread-safe). + // Note: If ThreadSafeCurlEasyRequest() throws then the memory allocated is still freed. + // 'new' never returned however and the constructor nor destructor of mCurlEasyRequest is called in this case. + // This might throw AICurlNoEasyHandle. + AICurlEasyRequest(bool buffered) : + mCurlEasyRequest(buffered ? new AICurlPrivate::ThreadSafeBufferedCurlEasyRequest : new AICurlPrivate::ThreadSafeCurlEasyRequest) { } + AICurlEasyRequest(AICurlEasyRequest const& orig) : mCurlEasyRequest(orig.mCurlEasyRequest) { } + + // For the rest, only allow read operations. + AIThreadSafeSimple& operator*(void) const { llassert(mCurlEasyRequest.get()); return *mCurlEasyRequest; } + AIThreadSafeSimple* operator->(void) const { llassert(mCurlEasyRequest.get()); return mCurlEasyRequest.get(); } + AIThreadSafeSimple* get(void) const { return mCurlEasyRequest.get(); } + + // Returns true if this object points to the same CurlEasyRequest object. + bool operator==(AICurlEasyRequest const& cer) const { return mCurlEasyRequest == cer.mCurlEasyRequest; } + + // Returns true if this object points to a different CurlEasyRequest object. + bool operator!=(AICurlEasyRequest const& cer) const { return mCurlEasyRequest != cer.mCurlEasyRequest; } + + // Queue this request for insertion in the multi session. + void addRequest(void); + + // Queue a command to remove this request from the multi session (or cancel a queued command to add it). + void removeRequest(void); + + private: + // The actual pointer to the ThreadSafeCurlEasyRequest instance. + AICurlPrivate::CurlEasyRequestPtr mCurlEasyRequest; + + private: + // Assignment would not be thread-safe; we may create this object and read from it. + // Note: Destruction is implicitly assumed thread-safe, as it would be a logic error to + // destruct it while another thread still needs it, concurrent or not. + AICurlEasyRequest& operator=(AICurlEasyRequest const&) { return *this; } + + public: + // The more exotic member functions of this class, to deal with passing this class + // as CURLOPT_PRIVATE pointer to a curl handle and afterwards restore it. + // For "internal use" only; don't use things from AICurlPrivate yourself. + + // It's thread-safe to give read access the underlaying boost::intrusive_ptr. + // It's not OK to then call get() on that and store the AICurlPrivate::ThreadSafeCurlEasyRequest* separately. + AICurlPrivate::CurlEasyRequestPtr const& get_ptr(void) const { return mCurlEasyRequest; } + + // If we have a correct (with regard to reference counting) AICurlPrivate::CurlEasyRequestPtr, + // then it's OK to construct a AICurlEasyRequest from it. + // Note that the external AICurlPrivate::CurlEasyRequestPtr needs it's own locking, because + // it's not thread-safe in itself. + AICurlEasyRequest(AICurlPrivate::CurlEasyRequestPtr const& ptr) : mCurlEasyRequest(ptr) { } + + // This one is obviously dangerous. It's for use only in MultiHandle::check_run_count. + // See also the long comment in CurlEasyRequest::finalizeRequest with regard to CURLOPT_PRIVATE. + explicit AICurlEasyRequest(AICurlPrivate::ThreadSafeCurlEasyRequest* ptr) : mCurlEasyRequest(ptr) { } +}; + +// Write Access Type for the buffer. +struct AICurlResponderBuffer_wat : public AIAccess { + explicit AICurlResponderBuffer_wat(AICurlPrivate::ThreadSafeBufferedCurlEasyRequest& lockobj) : + AIAccess(lockobj) { } + AICurlResponderBuffer_wat(AIThreadSafeSimple& lockobj) : + AIAccess(static_cast(lockobj)) { } +}; + +#define AICurlPrivate DONTUSE_AICurlPrivate + +#endif diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h new file mode 100644 index 000000000..9a7f50180 --- /dev/null +++ b/indra/llmessage/aicurlprivate.h @@ -0,0 +1,360 @@ +/** + * @file aicurlprivate.h + * @brief Thread safe wrapper for libcurl. + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 28/04/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AICURLPRIVATE_H +#define AICURLPRIVATE_H + +#include + +namespace AICurlPrivate { +namespace curlthread { class MultiHandle; } + +CURLcode check_easy_code(CURLcode code); +CURLMcode check_multi_code(CURLMcode code); + +bool curlThreadIsRunning(void); +void wakeUpCurlThread(void); + +class ThreadSafeCurlEasyRequest; +class ThreadSafeBufferedCurlEasyRequest; + +// This class wraps CURL*'s. +// It guarantees that a pointer is cleaned up when no longer needed, as required by libcurl. +class CurlEasyHandle : public boost::noncopyable, protected AICurlEasyHandleEvents { + public: + CurlEasyHandle(void); + ~CurlEasyHandle(); + + private: + // Disallow assignment. + CurlEasyHandle& operator=(CurlEasyHandle const*); + + public: + // Reset all options of a libcurl session handle. + void reset(void) { llassert(!mActiveMultiHandle); curl_easy_reset(mEasyHandle); } + + // Set options for a curl easy handle. + template + CURLcode setopt(CURLoption option, BUILTIN parameter); + + // Clone a libcurl session handle using all the options previously set. + CurlEasyHandle(CurlEasyHandle const& orig) : mEasyHandle(curl_easy_duphandle(orig.mEasyHandle)), mActiveMultiHandle(NULL), mErrorBuffer(NULL) { } + + // URL encode/decode the given string. + char* escape(char* url, int length); + char* unescape(char* url, int inlength , int* outlength); + + // Extract information from a curl handle. + CURLcode getinfo(CURLINFO info, void* data); +#if _WIN64 || __x86_64__ || __ppc64__ + // Overload for integer types that are too small (libcurl demands a long). + CURLcode getinfo(CURLINFO info, S32* data) { long ldata; CURLcode res = getinfo(info, &ldata); *data = static_cast(ldata); return res; } + CURLcode getinfo(CURLINFO info, U32* data) { long ldata; CURLcode res = getinfo(info, &ldata); *data = static_cast(ldata); return res; } +#endif + + // Perform a file transfer (blocking). + CURLcode perform(void); + // Pause and unpause a connection. + CURLcode pause(int bitmask); + + private: + CURL* mEasyHandle; + CURLM* mActiveMultiHandle; + char* mErrorBuffer; + static LLAtomicU32 sTotalEasyHandles; + + private: + // This should only be called from MultiHandle; add/remove an easy handle to/from a multi handle. + friend class curlthread::MultiHandle; + CURLMcode add_handle_to_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi_handle); + CURLMcode remove_handle_from_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi_handle); + + public: + // Retuns total number of existing CURL* handles (excluding ones created outside this class). + static U32 getTotalEasyHandles(void) { return sTotalEasyHandles; } + + // Returns true if this easy handle was added to a curl multi handle. + bool active(void) const { return mActiveMultiHandle; } + + // Call this prior to every curl_easy function whose return value is passed to check_easy_code. + void setErrorBuffer(void); + + // If there was an error code as result, then this returns a human readable error string. + // Only valid when setErrorBuffer was called and the curl_easy function returned an error. + std::string getErrorString(void) const { return mErrorBuffer; } + + // Used for debugging purposes. + bool operator==(CURL* easy_handle) const { return mEasyHandle == easy_handle; } + + protected: + // Return the underlaying curl easy handle. + CURL* getEasyHandle(void) const { return mEasyHandle; } + + private: + // Return, and possibly create, the curl (easy) error buffer used by the current thread. + static char* getTLErrorBuffer(void); +}; + +template +CURLcode CurlEasyHandle::setopt(CURLoption option, BUILTIN parameter) +{ + llassert(!mActiveMultiHandle); + setErrorBuffer(); + return check_easy_code(curl_easy_setopt(mEasyHandle, option, parameter)); +} + +// CurlEasyRequest adds a slightly more powerful interface that can be used +// to set the options on a curl easy handle. +// +// Calling sendRequest() will then connect to the given URL and perform +// the data exchange. If an error occurs related to this handle, it can +// be read by calling getErrorString(). +// +// Note that the life cycle of a CurlEasyRequest is controlled by AICurlEasyRequest: +// a CurlEasyRequest is only ever created as base class of a ThreadSafeCurlEasyRequest, +// which is only created by creating a AICurlEasyRequest. When the last copy of such +// AICurlEasyRequest is deleted, then also the ThreadSafeCurlEasyRequest is deleted +// and the CurlEasyRequest destructed. +class CurlEasyRequest : public CurlEasyHandle { + public: + void setoptString(CURLoption option, std::string const& value); + void setPost(char const* postdata, S32 size); + void addHeader(char const* str); + + private: + // Call back stubs. + static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* userdata); + static size_t writeCallback(char* ptr, size_t size, size_t nmemb, void* userdata); + static size_t readCallback(char* ptr, size_t size, size_t nmemb, void* userdata); + static CURLcode SSLCtxCallback(CURL* curl, void* sslctx, void* userdata); + + curl_write_callback mHeaderCallback; + void* mHeaderCallbackUserData; + curl_write_callback mWriteCallback; + void* mWriteCallbackUserData; + curl_read_callback mReadCallback; + void* mReadCallbackUserData; + curl_ssl_ctx_callback mSSLCtxCallback; + void* mSSLCtxCallbackUserData; + + public: + void setHeaderCallback(curl_write_callback callback, void* userdata); + void setWriteCallback(curl_write_callback callback, void* userdata); + void setReadCallback(curl_read_callback callback, void* userdata); + void setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata); + + // Call this if the set callbacks are about to be invalidated. + void revokeCallbacks(void); + + // Reset everything to the state it was in when this object was just created. + void resetState(void); + + public: + // Set default options that we want applied to all curl easy handles. + void applyDefaultOptions(void); + + // Prepare the request for adding it to a multi session, or calling perform. + // This actually adds the headers that were collected with addHeader. + void finalizeRequest(std::string const& url); + + // Store result code that is returned by getResult. + void store_result(CURLcode result) { mResult = result; } + + // Called when the curl easy handle is done. + void done(AICurlEasyRequest_wat& curl_easy_request_w) { finished(curl_easy_request_w); } + + // Fill info with the transfer info. + void getTransferInfo(AICurlInterface::TransferInfo* info); + + // If result != CURLE_FAILED_INIT then also info was filled. + void getResult(CURLcode* result, AICurlInterface::TransferInfo* info = NULL); + + private: + curl_slist* mHeaders; + bool mRequestFinalized; + AICurlEasyHandleEvents* mEventsTarget; + CURLcode mResult; + + private: + // This class may only be created by constructing a ThreadSafeCurlEasyRequest. + friend class ThreadSafeCurlEasyRequest; + // Throws AICurlNoEasyHandle. + CurlEasyRequest(void) : + mHeaders(NULL), mRequestFinalized(false), mEventsTarget(NULL), mResult(CURLE_FAILED_INIT) + { applyDefaultOptions(); } + public: + ~CurlEasyRequest(); + + public: + // Post initialization, set the parent to which to pass the events to. + void send_events_to(AICurlEasyHandleEvents* target) { mEventsTarget = target; } + + // For debugging purposes + bool is_finalized(void) const { return mRequestFinalized; } + + // Return pointer to the ThreadSafe (wrapped) version of this object. + ThreadSafeCurlEasyRequest* get_lockobj(void); + + protected: + // Pass events to parent. + /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void finished(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); +}; + +// Buffers used by the AICurlInterface::Request API. +// Curl callbacks write into and read from these buffers. +// The interface with the rest of the code is through AICurlInterface::Responder. +// +// The life time of a CurlResponderBuffer is slightly shorter than it's +// associated CurlEasyRequest; this class can only be created as base class +// of ThreadSafeBufferedCurlEasyRequest, and is therefore constructed after +// the construction of the associated CurlEasyRequest and destructed before it. +// Hence, it's safe to use get_lockobj() and through that access the CurlEasyRequest +// object at all times. +// +// A CurlResponderBuffer is thus created when a ThreadSafeBufferedCurlEasyRequest +// is created which only happens by creating a AICurlEasyRequest(true) instance, +// and when the last AICurlEasyRequest is deleted, then the ThreadSafeBufferedCurlEasyRequest +// is deleted and the CurlResponderBuffer destructed. +class CurlResponderBuffer : protected AICurlEasyHandleEvents { + public: + void resetState(AICurlEasyRequest_wat& curl_easy_request_w); + void prepRequest(AICurlEasyRequest_wat& buffered_curl_easy_request_w, std::vector const& headers, AICurlInterface::ResponderPtr responder, S32 time_out = 0, bool post = false); + + std::stringstream& getInput() { return mInput; } + std::stringstream& getHeaderOutput() { return mHeaderOutput; } + LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } + + // Called after removed_from_multi_handle was called. + void processOutput(AICurlEasyRequest_wat& curl_easy_request_w); + + protected: + /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void finished(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); + + private: + std::stringstream mInput; + std::stringstream mHeaderOutput; + LLIOPipe::buffer_ptr_t mOutput; + AICurlInterface::ResponderPtr mResponder; + + public: + static LLChannelDescriptors const sChannels; // Channel object for mOutput: we ONLY use channel 0, so this can be a constant. + + private: + // This class may only be created by constructing a ThreadSafeBufferedCurlEasyRequest. + friend class ThreadSafeBufferedCurlEasyRequest; + CurlResponderBuffer(void); + public: + ~CurlResponderBuffer(); + + private: + static size_t curlWriteCallback(char* data, size_t size, size_t nmemb, void* user_data); + static size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data); + static size_t curlHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data); + + public: + // Return pointer to the ThreadSafe (wrapped) version of this object. + ThreadSafeBufferedCurlEasyRequest* get_lockobj(void); +}; + +// This class wraps CurlEasyRequest for thread-safety and adds a reference counter so we can +// copy it around cheaply and it gets destructed automatically when the last instance is deleted. +// It guarantees that the CURL* handle is never used concurrently, which is not allowed by libcurl. +// As AIThreadSafeSimpleDC contains a mutex, it cannot be copied. Therefore we need a reference counter for this object. +class ThreadSafeCurlEasyRequest : public AIThreadSafeSimple { + public: + // Throws AICurlNoEasyHandle. + ThreadSafeCurlEasyRequest(void) : mReferenceCount(0) + { new (ptr()) CurlEasyRequest; + Dout(dc::curl, "Creating ThreadSafeCurlEasyRequest with this = " << (void*)this); } + virtual ~ThreadSafeCurlEasyRequest() + { Dout(dc::curl, "Destructing ThreadSafeCurlEasyRequest with this = " << (void*)this); } + + private: + LLAtomicU32 mReferenceCount; + + friend void intrusive_ptr_add_ref(ThreadSafeCurlEasyRequest* p); // Called by boost::intrusive_ptr when a new copy of a boost::intrusive_ptr is made. + friend void intrusive_ptr_release(ThreadSafeCurlEasyRequest* p); // Called by boost::intrusive_ptr when a boost::intrusive_ptr is destroyed. +}; + +// Same as the above but adds a CurlResponderBuffer. The latter has it's own locking in order to +// allow casting the underlaying CurlEasyRequest to ThreadSafeCurlEasyRequest, independent of +// what class it is part of: ThreadSafeCurlEasyRequest or ThreadSafeBufferedCurlEasyRequest. +// The virtual destructor of ThreadSafeCurlEasyRequest allows to treat each easy handle transparently +// as a ThreadSafeCurlEasyRequest object, or optionally dynamic_cast it to a ThreadSafeBufferedCurlEasyRequest. +// Note: the order of these base classes is important: AIThreadSafeSimple is now +// destructed before ThreadSafeCurlEasyRequest is. +class ThreadSafeBufferedCurlEasyRequest : public ThreadSafeCurlEasyRequest, public AIThreadSafeSimple { + public: + // Throws AICurlNoEasyHandle. + ThreadSafeBufferedCurlEasyRequest(void) { new (AIThreadSafeSimple::ptr()) CurlResponderBuffer; } +}; + +// The curl easy request type wrapped in a reference counting pointer. +typedef boost::intrusive_ptr CurlEasyRequestPtr; + +// This class wraps CURLM*'s. +// It guarantees that a pointer is cleaned up when no longer needed, as required by libcurl. +class CurlMultiHandle : public boost::noncopyable { + public: + CurlMultiHandle(void); + ~CurlMultiHandle(); + + private: + // Disallow assignment. + CurlMultiHandle& operator=(CurlMultiHandle const*); + + private: + static LLAtomicU32 sTotalMultiHandles; + + protected: + CURLM* mMultiHandle; + + public: + // Set options for a curl multi handle. + template + CURLMcode setopt(CURLMoption option, BUILTIN parameter); + + // Returns total number of existing CURLM* handles (excluding ones created outside this class). + static U32 getTotalMultiHandles(void) { return sTotalMultiHandles; } +}; + +template +CURLMcode CurlMultiHandle::setopt(CURLMoption option, BUILTIN parameter) +{ + return check_multi_code(curl_multi_setopt(mMultiHandle, option, parameter)); +} + +} // namespace AICurlPrivate + +#endif diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp new file mode 100644 index 000000000..6b2d1d68d --- /dev/null +++ b/indra/llmessage/aicurlthread.cpp @@ -0,0 +1,1054 @@ +/** + * @file aicurlthread.cpp + * @brief Implementation of AICurl, curl thread functions. + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 28/04/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#include "linden_common.h" +#include "aicurlthread.h" +#include "lltimer.h" // ms_sleep +#include +#include +#include +#include +#include + +#undef AICurlPrivate + +namespace AICurlPrivate { + +enum command_st { + cmd_none, + cmd_add, + cmd_boost, + cmd_remove +}; + +class Command { + public: + Command(void) : mCommand(cmd_none) { } + Command(AICurlEasyRequest const& easy_request, command_st command) : mCurlEasyRequest(easy_request.get_ptr()), mCommand(command) { } + + command_st command(void) const { return mCommand; } + CurlEasyRequestPtr const& easy_request(void) const { return mCurlEasyRequest; } + + bool operator==(AICurlEasyRequest const& easy_request) const { return mCurlEasyRequest == easy_request.get_ptr(); } + + void reset(void); + + private: + CurlEasyRequestPtr mCurlEasyRequest; + command_st mCommand; +}; + +void Command::reset(void) +{ + mCurlEasyRequest.reset(); + mCommand = cmd_none; +} + +// The following two globals have separate locks for speed considerations (in order not +// to block the main thread unnecessarily) but have the following correlation: +// +// MAIN-THREAD (AICurlEasyRequest::addRequest) +// * command_queue locked +// - A non-active (mActiveMultiHandle is NULL) ThreadSafeCurlEasyRequest (by means of an AICurlEasyRequest pointing to it) is added to command_queue with as command cmd_add. +// * command_queue unlocked +// +// If at this point addRequest is called again, then it is detected that the last command added to the queue +// for this ThreadSafeCurlEasyRequest is cmd_add. +// +// CURL-THREAD (AICurlThread::wakeup): +// * command_queue locked +// * command_being_processed is write-locked +// - command_being_processed is assigned the value of the command in the queue. +// * command_being_processed is unlocked +// - The command is removed from command_queue +// * command_queue unlocked +// +// If at this point addRequest is called again, then it is detected that command_being_processed adds the same ThreadSafeCurlEasyRequest. +// +// * command_being_processed is read-locked +// - mActiveMultiHandle is set to point to the curl multi handle +// - The easy handle is added to the multi handle +// * command_being_processed is write-locked +// - command_being_processed is reset +// * command_being_processed is unlocked +// +// If at this point addRequest is called again, then it is detected that the ThreadSafeCurlEasyRequest is active. + +// Multi-threaded queue for passing Command objects from the main-thread to the curl-thread. +AIThreadSafeSimpleDC > command_queue; +typedef AIAccess > command_queue_wat; + +AIThreadSafeDC command_being_processed; +typedef AIWriteAccess command_being_processed_wat; +typedef AIReadAccess command_being_processed_rat; + +namespace curlthread { +// All functions in this namespace are only run by the curl thread, unless they are marked with MAIN-THREAD. + +//----------------------------------------------------------------------------- +// PollSet + +// A PollSet can store at least 1024 file descriptors, or FD_SETSIZE if that is larger than 1024 [MAXSIZE]. +// The number of stored file descriptors is mNrFds [0 <= mNrFds <= MAXSIZE]. +// The largest file descriptor is stored is mMaxFd, which is -1 iff mNrFds == 0. +// The file descriptors are stored contiguous in mFileDescriptors[i], with 0 <= i < mNrFds. +// File descriptors with the highest priority should be stored first (low index). +// +// mNext is an index into mFileDescriptors that is copied first, the next call to refresh(). +// It is set to 0 when mNrFds < FD_SETSIZE, even if mNrFds == 0. +// +// After a call to refresh(): +// +// mFdSet has bits set for at most FD_SETSIZE - 1 file descriptors, copied from mFileDescriptors starting +// at index mNext (wrapping around to 0). If mNrFds < FD_SETSIZE then mNext is reset to 0 before copying starts. +// If mNrFds >= FD_SETSIZE then mNext is set to the next filedescriptor that was not copied (otherwise it is left at 0). +// +// mMaxFdSet is the largest filedescriptor in mFdSet or -1 if it is empty. + +// Create an empty PollSet. +PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [std::max(1024, FD_SETSIZE)]), mSize(std::max(1024, FD_SETSIZE)), + mNrFds(0), mMaxFd(-1), mNext(0), mMaxFdSet(-1) +{ + FD_ZERO(&mFdSet); +} + +// Add file descriptor s to the PollSet. +void PollSet::add(curl_socket_t s) +{ + llassert_always(mNrFds < mSize); + mFileDescriptors[mNrFds++] = s; + mMaxFd = std::max(mMaxFd, s); +} + +// Remove file descriptor s from the PollSet. +void PollSet::remove(curl_socket_t s) +{ + // The number of open filedescriptors is relatively small, + // and on top of that we rather do something CPU intensive + // than bandwidth intensive (lookup table). Hence that this + // is a linear search in an array containing just the open + // filedescriptors. Then, since we are reading this memory + // page anyway, we might as well write to it without losing + // much clock cycles. Therefore, shift the whole vector + // back, keeping it compact and keeping the filedescriptors + // in the same order (which is supposedly their priority). + llassert(mNrFds > 0); + int i = --mNrFds; + int prev = mFileDescriptors[i]; + int max = -1; + for (--i; i >= 0 && prev != s; --i) + { + int cur = mFileDescriptors[i]; + mFileDescriptors[i] = prev; + max = std::max(max, prev); + prev = cur; + } + if (s == mMaxFd) + { + while (i >= 0) + { + int cur = mFileDescriptors[i]; + max = std::max(max, cur); + --i; + } + mMaxFd = max; + llassert(mMaxFd < s); + llassert((mMaxFd == -1) == (mNrFds == 0)); + } + if (mNext == mNrFds) + mNext = 0; +} + +bool PollSet::contains(curl_socket_t fd) const +{ + for (int i = 0; i < mNrFds; ++i) + if (mFileDescriptors[i] == fd) + return true; + return false; +} + +inline bool PollSet::is_set(curl_socket_t fd) const +{ + return FD_ISSET(fd, &mFdSet); +} + +inline void PollSet::clr(curl_socket_t fd) +{ + return FD_CLR(fd, &mFdSet); +} + +// This function fills mFdSet with at most FD_SETSIZE - 1 filedescriptors, +// starting at index mNext (updating mNext when not all could be added), +// and updates mMaxFdSet to be the largest fd added to mFdSet, or -1 if it's empty. +refresh_t PollSet::refresh(void) +{ + FD_ZERO(&mFdSet); + mCopiedFileDescriptors.clear(); + + if (mNrFds == 0) + { + mMaxFdSet = -1; + return empty_and_complete; + } + + llassert_always(mNext < mNrFds); + + // Test if mNrFds is larger or equal FD_SETSIZE; equal, because we reserve one + // file descriptor for the wakeup fd: we copy maximal FD_SETSIZE - 1 file descriptors. + // If not then we're going to copy everything so that we can save on CPU cycles + // by not calculating mMaxFdSet here. + if (mNrFds >= FD_SETSIZE) + { + llwarns << "PollSet::reset: More than FD_SETSIZE (" << FD_SETSIZE << ") file descriptors active!" << llendl; + // Calculate mMaxFdSet. + int max = -1; + int count = 0; + int end = mNrFds; + int i = mNext; + while (++count < FD_SETSIZE) + { + max = std::max(max, mFileDescriptors[i]); + if (++i == end) + { + if (end == mNext) + break; + end = mNext; + i = 0; + llassert(i < end); // If mNext == 0 then the while loop terminates before we get here. + } + } + mMaxFdSet = max; + } + else + { + mNext = 0; // Start at the beginning if we copy everything anyway. + mMaxFdSet = mMaxFd; + } + int count = 0; + int end = mNrFds; + int i = mNext; + for(;;) + { + if (++count == FD_SETSIZE) + { + mNext = i; + return not_complete_not_empty; + } + FD_SET(mFileDescriptors[i], &mFdSet); + mCopiedFileDescriptors.push_back(mFileDescriptors[i]); + if (++i == end) + { + if (mNext == 0) + break; + // If this was true then we got here a second time, which means that we accessed all + // filedescriptors with mNext != 0, but count is still < FD_SETSIZE, which is not + // possible because mNext is only non-zero when mNrFds >= FD_SETSIZE. + llassert(end != mNext); + end = mNext; + i = 0; + } + } + return complete_not_empty; +} + +// FIXME: This needs a rewrite on windows, as FD_ISSET is slow there; it would make +// more sense to iterate directly over the fd's in mFdSet on windows. +void PollSet::reset(void) +{ + llassert((mNrFds == 0) == mCopiedFileDescriptors.empty()); + if (mCopiedFileDescriptors.empty()) + mIter = mCopiedFileDescriptors.end(); + else + { + mIter = mCopiedFileDescriptors.begin(); + if (!FD_ISSET(*mIter, &mFdSet)) + next(); + } +} + +inline int PollSet::get(void) const +{ + return (mIter == mCopiedFileDescriptors.end()) ? -1 : *mIter; +} + +// FIXME: This needs a rewrite on windows, as FD_ISSET is slow there; it would make +// more sense to iterate directly over the fd's in mFdSet on windows. +void PollSet::next(void) +{ + llassert(mIter != mCopiedFileDescriptors.end()); // Only call next() if the last call to get() didn't return -1. + while (++mIter != mCopiedFileDescriptors.end() && !FD_ISSET(*mIter, &mFdSet)); +} + +//----------------------------------------------------------------------------- +// MergeIterator + +class MergeIterator +{ + public: + MergeIterator(PollSet& readPollSet, PollSet& writePollSet); + + bool next(int& fd_out, int& ev_bitmask_out); + + private: + PollSet& mReadPollSet; + PollSet& mWritePollSet; + int readIndx; + int writeIndx; +}; + +MergeIterator::MergeIterator(PollSet& readPollSet, PollSet& writePollSet) : + mReadPollSet(readPollSet), mWritePollSet(writePollSet), readIndx(0), writeIndx(0) +{ + mReadPollSet.reset(); + mWritePollSet.reset(); +} + +bool MergeIterator::next(int& fd_out, int& ev_bitmask_out) +{ + int rfd = mReadPollSet.get(); + int wfd = mWritePollSet.get(); + + if (rfd == -1 && wfd == -1) + return false; + + if (rfd == wfd) + { + fd_out = rfd; + ev_bitmask_out = CURL_CSELECT_IN | CURL_CSELECT_OUT; + mReadPollSet.next(); + + } + else if ((unsigned int)rfd < (unsigned int)wfd) // Use and increment smaller one, unless it's -1. + { + fd_out = rfd; + ev_bitmask_out = CURL_CSELECT_IN; + mReadPollSet.next(); + if (wfd != -1 && mWritePollSet.is_set(rfd)) + { + ev_bitmask_out |= CURL_CSELECT_OUT; + mWritePollSet.clr(rfd); + } + } + else + { + fd_out = wfd; + ev_bitmask_out = CURL_CSELECT_OUT; + mWritePollSet.next(); + if (rfd != -1 && mReadPollSet.is_set(wfd)) + { + ev_bitmask_out |= CURL_CSELECT_IN; + mReadPollSet.clr(wfd); + } + } + + return true; +} + +//----------------------------------------------------------------------------- +// CurlSocketInfo + +// A class with info for each socket that is in use by curl. +class CurlSocketInfo +{ + public: + CurlSocketInfo(MultiHandle& multi_handle, CURL* easy, curl_socket_t s, int action); + ~CurlSocketInfo(); + + void set_action(int action); + + private: + MultiHandle& mMultiHandle; + CURL const* mEasy; + curl_socket_t mSocketFd; + int mAction; +}; + +CurlSocketInfo::CurlSocketInfo(MultiHandle& multi_handle, CURL* easy, curl_socket_t s, int action) : + mMultiHandle(multi_handle), mEasy(easy), mSocketFd(s), mAction(CURL_POLL_NONE) +{ + mMultiHandle.assign(s, this); + llassert(!mMultiHandle.mReadPollSet.contains(s)); + llassert(!mMultiHandle.mWritePollSet.contains(s)); + set_action(action); +} + +CurlSocketInfo::~CurlSocketInfo() +{ + set_action(CURL_POLL_NONE); +} + +void CurlSocketInfo::set_action(int action) +{ + int toggle_action = mAction ^ action; + mAction = action; + if ((toggle_action & CURL_POLL_IN)) + { + if ((action & CURL_POLL_IN)) + mMultiHandle.mReadPollSet.add(mSocketFd); + else + mMultiHandle.mReadPollSet.remove(mSocketFd); + } + if ((toggle_action & CURL_POLL_OUT)) + { + if ((action & CURL_POLL_OUT)) + mMultiHandle.mWritePollSet.add(mSocketFd); + else + mMultiHandle.mWritePollSet.remove(mSocketFd); + } +} + +//----------------------------------------------------------------------------- +// AICurlThread + +class AICurlThread : public LLThread +{ + public: + static AICurlThread* sInstance; + + public: + // MAIN-THREAD + AICurlThread(void); + virtual ~AICurlThread(); + + // MAIN-THREAD + void wakeup_thread(void); + + protected: + virtual void run(void); + void wakeup(AICurlMultiHandle_wat const& multi_handle_w); + + private: + // MAIN-THREAD + void create_wakeup_fds(void); + void cleanup_wakeup_fds(void); + + int mWakeUpFd_in; + int mWakeUpFd; + + int mZeroTimeOut; +}; + +// Only the main thread is accessing this. +AICurlThread* AICurlThread::sInstance = NULL; + +// MAIN-THREAD +AICurlThread::AICurlThread(void) : LLThread("AICurlThread"), mWakeUpFd_in(-1), mWakeUpFd(-1), mZeroTimeOut(0) +{ + create_wakeup_fds(); + sInstance = this; +} + +// MAIN-THREAD +AICurlThread::~AICurlThread() +{ + sInstance = NULL; + cleanup_wakeup_fds(); +} + +// MAIN-THREAD +void AICurlThread::create_wakeup_fds(void) +{ +#ifdef WINDOWS +// Probably need to use sockets here, cause windows select doesn't work for a pipe. +#error Missing implementation +#else + int pipefd[2]; + if (pipe(pipefd)) + { + llerrs << "Failed to create wakeup pipe: " << strerror(errno) << llendl; + } + long flags = O_NONBLOCK; + for (int i = 0; i < 2; ++i) + { + if (fcntl(pipefd[i], F_SETFL, flags)) + { + llerrs << "Failed to set pipe to non-blocking: " << strerror(errno) << llendl; + } + } + mWakeUpFd = pipefd[0]; // Read-end of the pipe. + mWakeUpFd_in = pipefd[1]; // Write-end of the pipe. +#endif +} + +// MAIN-THREAD +void AICurlThread::cleanup_wakeup_fds(void) +{ + if (mWakeUpFd_in != -1) + close(mWakeUpFd_in); + if (mWakeUpFd != -1) + close(mWakeUpFd); +} + +// MAIN-THREAD +void AICurlThread::wakeup_thread(void) +{ + DoutEntering(dc::curl, "AICurlThread::wakeup_thread"); + +#ifdef WINDOWS +#error Missing implementation +#else + // If write() is interrupted by a signal before it writes any data, it shall return -1 with errno set to [EINTR]. + // If write() is interrupted by a signal after it successfully writes some data, it shall return the number of bytes written. + // Write requests to a pipe or FIFO shall be handled in the same way as a regular file with the following exceptions: + // If the O_NONBLOCK flag is set, write() requests shall be handled differently, in the following ways: + // A write request for {PIPE_BUF} or fewer bytes shall have the following effect: + // if there is sufficient space available in the pipe, write() shall transfer all the data and return the number + // of bytes requested. Otherwise, write() shall transfer no data and return -1 with errno set to [EAGAIN]. + ssize_t len; + do + { + len = write(mWakeUpFd_in, "!", 1); + if (len == -1 && errno == EAGAIN) + return; // Unread characters are still in the pipe, so no need to add more. + } + while(len == -1 && errno == EINTR); + if (len == -1) + { + llerrs << "write(3) to mWakeUpFd_in: " << strerror(errno) << llendl; + } + llassert_always(len == 1); +#endif +} + +void AICurlThread::wakeup(AICurlMultiHandle_wat const& multi_handle_w) +{ + DoutEntering(dc::curl, "AICurlThread::wakeup"); + +#ifdef WINDOWS +#error Missing implementation +#else + // If a read() is interrupted by a signal before it reads any data, it shall return -1 with errno set to [EINTR]. + // If a read() is interrupted by a signal after it has successfully read some data, it shall return the number of bytes read. + // When attempting to read from an empty pipe or FIFO: + // If no process has the pipe open for writing, read() shall return 0 to indicate end-of-file. + // If some process has the pipe open for writing and O_NONBLOCK is set, read() shall return -1 and set errno to [EAGAIN]. + char buf[256]; + ssize_t len; + do + { + len = read(mWakeUpFd, buf, sizeof(buf)); + if (len == -1 && errno == EAGAIN) + return; + } + while(len == -1 && errno == EINTR); + if (len == -1) + { + llerrs << "read(3) from mWakeUpFd: " << strerror(errno) << llendl; + } + if (LL_UNLIKELY(len == 0)) + { + llwarns << "read(3) from mWakeUpFd returned 0, indicating that the pipe on the other end was closed! Shutting down curl thread." << llendl; + close(mWakeUpFd); + mWakeUpFd = -1; + shutdown(); + return; + } +#endif + // If we get here then the main thread called wakeup_thread() recently. + for(;;) + { + // Access command_queue, and move command to command_being_processed. + { + command_queue_wat command_queue_w(command_queue); + if (command_queue_w->empty()) + break; + // Move the next command from the queue into command_being_processed. + *command_being_processed_wat(command_being_processed) = command_queue_w->front(); + command_queue_w->pop_front(); + } + // Access command_being_processed only. + { + command_being_processed_rat command_being_processed_r(command_being_processed); + switch(command_being_processed_r->command()) + { + case cmd_none: + case cmd_boost: // FIXME: future stuff + break; + case cmd_add: + multi_handle_w->add_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request())); + break; + case cmd_remove: + multi_handle_w->remove_easy_request(AICurlEasyRequest(command_being_processed_r->easy_request())); + break; + } + // Done processing. + command_being_processed_wat command_being_processed_w(command_being_processed_r); + command_being_processed_w->reset(); // This destroys the CurlEasyRequest in case of a cmd_remove. + } + } +} + +// The main loop of the curl thread. +void AICurlThread::run(void) +{ + DoutEntering(dc::curl, "AICurlThread::run()"); + + AICurlMultiHandle_wat multi_handle_w(AICurlMultiHandle::getInstance()); + for(;;) + { + refresh_t rres = multi_handle_w->mReadPollSet.refresh(); + refresh_t wres = multi_handle_w->mWritePollSet.refresh(); + fd_set* read_fd_set = multi_handle_w->mReadPollSet.access(); + if (LL_LIKELY(mWakeUpFd != -1)) + FD_SET(mWakeUpFd, read_fd_set); + else if ((rres & empty)) + read_fd_set = NULL; + fd_set* write_fd_set = ((wres & empty)) ? NULL : multi_handle_w->mWritePollSet.access(); + int const max_rfd = std::max(multi_handle_w->mReadPollSet.get_max_fd(), mWakeUpFd); + int const max_wfd = multi_handle_w->mWritePollSet.get_max_fd(); + int nfds = std::max(max_rfd, max_wfd) + 1; + llassert(0 <= nfds && nfds <= FD_SETSIZE); + llassert((max_rfd == -1) == (read_fd_set == NULL) && + (max_wfd == -1) == (write_fd_set == NULL)); // Needed on windows. + llassert((max_rfd == -1 || multi_handle_w->mReadPollSet.is_set(max_rfd)) && + (max_wfd == -1 || multi_handle_w->mWritePollSet.is_set(max_wfd))); + int ready = 0; + if (LL_UNLIKELY(nfds == 0)) // Only happens when the thread is shutting down. + ms_sleep(1000); + else + { + struct timeval timeout; + long timeout_ms = multi_handle_w->getTimeOut(); + // If no timeout is set, sleep 1 second. + if (timeout_ms < 0) + timeout_ms = 1000; + if (timeout_ms == 0) + { + if (mZeroTimeOut >= 10000) + { + if (mZeroTimeOut == 10000) + llwarns << "Detected more than 10000 zero-timeout calls of select() by curl thread (more than 101 seconds)!" << llendl; + } + else if (mZeroTimeOut >= 1000) + timeout_ms = 10; + else if (mZeroTimeOut >= 100) + timeout_ms = 1; + } + else + { + if (mZeroTimeOut >= 10000) + llinfos << "Timeout of select() call by curl thread reset (to " << timeout_ms << " ms)." << llendl; + mZeroTimeOut = 0; + } + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_usec = (timeout_ms % 1000) * 1000; +#ifdef CWDEBUG + static int last_nfds = -1; + static long last_timeout_ms = -1; + static int same_count = 0; + bool same = (nfds == last_nfds && timeout_ms == last_timeout_ms); + if (!same) + { + if (same_count > 1) + Dout(dc::curl, "Last select() call repeated " << same_count << " times."); + Dout(dc::curl|flush_cf|continued_cf, "select(" << nfds << ", ..., timeout = " << timeout_ms << " ms) = "); + same_count = 1; + } + else + { + ++same_count; + } +#endif + ready = select(nfds, read_fd_set, write_fd_set, NULL, &timeout); +#ifdef CWDEBUG + static int last_ready = -2; + static int last_errno = 0; + if (!same) + Dout(dc::finish|cond_error_cf(ready == -1), ready); + else if (ready != last_ready || (ready == -1 && errno != last_errno)) + { + if (same_count > 1) + Dout(dc::curl, "Last select() call repeated " << same_count << " times."); + Dout(dc::curl|cond_error_cf(ready == -1), "select(" << last_nfds << ", ..., timeout = " << last_timeout_ms << " ms) = " << ready); + same_count = 1; + } + last_nfds = nfds; + last_timeout_ms = timeout_ms; + last_ready = ready; + if (ready == -1) + last_errno = errno; +#endif + } + // Select returns the total number of bits set in each of the fd_set's (upon return), + // or -1 when an error occurred. A value of 0 means that a timeout occurred. + if (ready == -1) + { + llwarns << "select() failed: " << errno << ", " << strerror(errno) << llendl; + continue; + } + else if (ready == 0) + { + multi_handle_w->socket_action(CURL_SOCKET_TIMEOUT, 0); + } + else + { + if (multi_handle_w->mReadPollSet.is_set(mWakeUpFd)) + { + wakeup(multi_handle_w); + --ready; + } + MergeIterator iter(multi_handle_w->mReadPollSet, multi_handle_w->mWritePollSet); + int fd, ev_bitmask; + while (ready > 0 && iter.next(fd, ev_bitmask)) + { + ready -= (ev_bitmask == (CURL_CSELECT_IN|CURL_CSELECT_OUT)) ? 2 : 1; + multi_handle_w->socket_action(fd, ev_bitmask); + llassert(ready >= 0); + } + llassert(ready == 0); + } + multi_handle_w->check_run_count(); + } +} + +//----------------------------------------------------------------------------- +// MultiHandle + +MultiHandle::MultiHandle(void) : mHandleAddedOrRemoved(false), mPrevRunningHandles(0), mRunningHandles(0), mTimeOut(-1) +{ + curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETFUNCTION, &MultiHandle::socket_callback); + curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETDATA, this); + curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERFUNCTION, &MultiHandle::timer_callback); + curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERDATA, this); +} + +MultiHandle::~MultiHandle() +{ + // This thread was terminated. + // Curl demands that all handles are removed from the multi session handle before calling curl_multi_cleanup. + for(addedEasyRequests_type::iterator iter = mAddedEasyRequests.begin(); iter != mAddedEasyRequests.end(); iter = mAddedEasyRequests.begin()) + { + remove_easy_request(*iter); + } +} + +#ifdef CWDEBUG +#undef AI_CASE_RETURN +#define AI_CASE_RETURN(x) do { case x: return #x; } while(0) +char const* action_str(int action) +{ + switch(action) + { + AI_CASE_RETURN(CURL_POLL_NONE); + AI_CASE_RETURN(CURL_POLL_IN); + AI_CASE_RETURN(CURL_POLL_OUT); + AI_CASE_RETURN(CURL_POLL_INOUT); + AI_CASE_RETURN(CURL_POLL_REMOVE); + } + return ""; +} +#endif + +//static +int MultiHandle::socket_callback(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp) +{ + DoutEntering(dc::curl, "MultiHandle::socket_callback(" << (void*)easy << ", " << s << ", " << action_str(action) << ", " << (void*)userp << ", " << (void*)socketp << ")"); + MultiHandle& self = *static_cast(userp); + CurlSocketInfo* sock_info = static_cast(socketp); + if (action == CURL_POLL_REMOVE) + { + delete sock_info; + } + else + { + if (!sock_info) + { + sock_info = new CurlSocketInfo(self, easy, s, action); + } + else + { + sock_info->set_action(action); + } + } + return 0; +} + +//static +int MultiHandle::timer_callback(CURLM* multi, long timeout_ms, void* userp) +{ + MultiHandle& self = *static_cast(userp); + llassert(multi == self.mMultiHandle); + self.mTimeOut = timeout_ms; + Dout(dc::curl, "MultiHandle::timer_callback(): timeout set to " << timeout_ms << " ms."); + return 0; +} + +CURLMcode MultiHandle::socket_action(curl_socket_t sockfd, int ev_bitmask) +{ + CURLMcode res; + do + { + res = check_multi_code(curl_multi_socket_action(mMultiHandle, sockfd, ev_bitmask, &mRunningHandles)); + } + while(res == CURLM_CALL_MULTI_PERFORM); + return res; +} + +CURLMcode MultiHandle::assign(curl_socket_t sockfd, void* sockptr) +{ + return check_multi_code(curl_multi_assign(mMultiHandle, sockfd, sockptr)); +} + +CURLMsg const* MultiHandle::info_read(int* msgs_in_queue) const +{ + return curl_multi_info_read(mMultiHandle, msgs_in_queue); +} + +CURLMcode MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) +{ + std::pair res = mAddedEasyRequests.insert(easy_request); + llassert(res.second); // May not have been added before. + CURLMcode ret; + { + AICurlEasyRequest_wat curl_easy_request_w(*easy_request); + ret = curl_easy_request_w->add_handle_to_multi(curl_easy_request_w, mMultiHandle); + } + mHandleAddedOrRemoved = true; + Dout(dc::curl, "MultiHandle::add_easy_request: Added AICurlEasyRequest " << (void*)easy_request.get() << "; now processing " << mAddedEasyRequests.size() << " easy handles."); + return ret; +} + +CURLMcode MultiHandle::remove_easy_request(AICurlEasyRequest const& easy_request) +{ + addedEasyRequests_type::iterator iter = mAddedEasyRequests.find(easy_request); + if (iter == mAddedEasyRequests.end()) + return (CURLMcode)-2; // Was already removed before. + CURLMcode res; + { + AICurlEasyRequest_wat curl_easy_request_w(**iter); + res = curl_easy_request_w->remove_handle_from_multi(curl_easy_request_w, mMultiHandle); + } + mAddedEasyRequests.erase(iter); + mHandleAddedOrRemoved = true; + Dout(dc::curl, "MultiHandle::remove_easy_request: Removed AICurlEasyRequest " << (void*)easy_request.get() << "; now processing " << mAddedEasyRequests.size() << " easy handles."); + return res; +} + +void MultiHandle::check_run_count(void) +{ + if (mHandleAddedOrRemoved || mRunningHandles < mPrevRunningHandles) + { + CURLMsg const* msg; + int msgs_left; + while ((msg = info_read(&msgs_left))) + { + if (msg->msg == CURLMSG_DONE) + { + CURL* easy = msg->easy_handle; + ThreadSafeCurlEasyRequest* ptr; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ptr); + AICurlEasyRequest easy_request(ptr); + llassert(*AICurlEasyRequest_wat(*easy_request) == easy); + // Store the result and transfer info in the easy handle. + { + AICurlEasyRequest_wat curl_easy_request_w(*easy_request); + curl_easy_request_w->store_result(msg->data.result); +#ifdef CWDEBUG + char* eff_url; + curl_easy_request_w->getinfo(CURLINFO_EFFECTIVE_URL, &eff_url); + Dout(dc::curl, "Finished: " << eff_url << " (" << msg->data.result << ")"); +#endif + // Signal that this easy handle finished. + curl_easy_request_w->done(curl_easy_request_w); + } + // This invalidates msg, but not easy_request. + CURLMcode res = remove_easy_request(easy_request); + // This should hold, I think, because the handles are obviously ok and + // the only error we could get is when remove_easy_request() was already + // called before (by this thread); but if that was the case then the easy + // handle should not have been be returned by info_read()... + llassert(res == CURLM_OK); + // Nevertheless, if it was already removed then just ignore it. + if (res == CURLM_OK) + { + } + else if (res == -2) + { + llwarns << "Curl easy handle returned by curl_multi_info_read() that is not (anymore) in MultiHandle::mAddedEasyRequests!?!" << llendl; + } + // Destruction of easy_request at this point, causes the CurlEasyRequest to be deleted. + } + } + mHandleAddedOrRemoved = false; + } + mPrevRunningHandles = mRunningHandles; +} + +} // namespace curlthread +} // namespace AICurlPrivate + +//============================================================================= +// MAIN-THREAD (needing to access the above declarations). + +//static +AICurlMultiHandle& AICurlMultiHandle::getInstance(void) +{ + LLThreadLocalData& tldata = LLThreadLocalData::tldata(); + if (!tldata.mCurlMultiHandle) + { + llinfos << "Creating AICurlMultiHandle for thread \"" << tldata.mName << "\"." << llendl; + tldata.mCurlMultiHandle = new AICurlMultiHandle; + } + return *static_cast(tldata.mCurlMultiHandle); +} + +namespace AICurlPrivate { + +bool curlThreadIsRunning(void) +{ + using curlthread::AICurlThread; + return AICurlThread::sInstance && !AICurlThread::sInstance->isStopped(); +} + +void wakeUpCurlThread(void) +{ + using curlthread::AICurlThread; + if (AICurlThread::sInstance) + AICurlThread::sInstance->wakeup_thread(); +} + +} // namespace AICurlPrivate + +//----------------------------------------------------------------------------- +// AICurlEasyRequest + +void AICurlEasyRequest::addRequest(void) +{ + using namespace AICurlPrivate; + + { + // Write-lock the command queue. + command_queue_wat command_queue_w(command_queue); +#ifdef SHOW_ASSERT + // This debug code checks if we aren't calling addRequest() twice for the same object. + // That means that the main thread already called (and finished, this is also the + // main thread) this function, which also follows from that we just locked command_queue. + // That leaves three options: It's still in the queue, or it was removed and is currently + // processed by the curl thread with again two options: either it was already added + // to the multi session handle or not yet. + + // Find the last command added. + command_st cmd = cmd_none; + for (std::deque::iterator iter = command_queue_w->begin(); iter != command_queue_w->end(); ++iter) + { + if (*iter == *this) + { + cmd = iter->command(); + break; + } + } + llassert(cmd == cmd_none || cmd == cmd_remove); // Not in queue, or last command was to remove it. + if (cmd == cmd_none) + { + // Read-lock command_being_processed. + command_being_processed_rat command_being_processed_r(command_being_processed); + if (*command_being_processed_r == *this) + { + // May not be inbetween being removed from the command queue but not added to the multi session handle yet. + llassert(command_being_processed_r->command() == cmd_remove); + } + else + { + // May not already be added to the multi session handle. + llassert(!AICurlEasyRequest_wat(*get())->active()); + } + } +#endif + // Add a command to add the new request to the multi session to the command queue. + command_queue_w->push_back(Command(*this, cmd_add)); + } + // Something was added to the queue, wake up the thread to get it. + wakeUpCurlThread(); +} + +void AICurlEasyRequest::removeRequest(void) +{ + using namespace AICurlPrivate; + + { + // Write-lock the command queue. + command_queue_wat command_queue_w(command_queue); +#ifdef SHOW_ASSERT + // This debug code checks if we aren't calling removeRequest() twice for the same object. + // That means that the thread calling this function already finished it, following from that + // we just locked command_queue. + // That leaves three options: It's still in the queue, or it was removed and is currently + // processed by the curl thread with again two options: either it was already removed + // from the multi session handle or not yet. + + // Find the last command added. + command_st cmd = cmd_none; + for (std::deque::iterator iter = command_queue_w->begin(); iter != command_queue_w->end(); ++iter) + { + if (*iter == *this) + { + cmd = iter->command(); + break; + } + } + llassert(cmd == cmd_none || cmd != cmd_remove); // Not in queue, or last command was not a remove command. + if (cmd == cmd_none) + { + // Read-lock command_being_processed. + command_being_processed_rat command_being_processed_r(command_being_processed); + if (*command_being_processed_r == *this) + { + // May not be inbetween being removed from the command queue but not removed from the multi session handle yet. + llassert(command_being_processed_r->command() != cmd_remove); + } + else + { + // May not already have been removed from multi session handle. + llassert(AICurlEasyRequest_wat(*get())->active()); + } + } +#endif + // Add a command to remove this request from the multi session to the command queue. + command_queue_w->push_back(Command(*this, cmd_remove)); + } + // Something was added to the queue, wake up the thread to get it. + wakeUpCurlThread(); +} + +//----------------------------------------------------------------------------- + +namespace AICurlInterface { + +void startCurlThread(void) +{ + using namespace AICurlPrivate::curlthread; + + llassert(is_main_thread()); + AICurlThread::sInstance = new AICurlThread; + AICurlThread::sInstance->start(); +} + +} // namespace AICurlInterface + diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h new file mode 100644 index 000000000..448e53ac0 --- /dev/null +++ b/indra/llmessage/aicurlthread.h @@ -0,0 +1,185 @@ +/** + * @file aicurlthread.h + * @brief Thread safe wrapper for libcurl. + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 28/04/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AICURLTHREAD_H +#define AICURLTHREAD_H + +#include "aicurl.h" +#include + +#undef AICurlPrivate + +namespace AICurlPrivate { +namespace curlthread { + +// For ordering a std::set with AICurlEasyRequest objects. +struct AICurlEasyRequestCompare { + bool operator()(AICurlEasyRequest const& h1, AICurlEasyRequest const& h2) { return h1.get() < h2.get(); } +}; + +//----------------------------------------------------------------------------- +// PollSet + +int const empty = 0x1; +int const complete = 0x2; + +enum refresh_t { + not_complete_not_empty = 0, + complete_not_empty = complete, + empty_and_complete = complete|empty +}; + +class PollSet +{ + public: + PollSet(void); + + // Add/remove a filedescriptor to/from mFileDescriptors. + void add(curl_socket_t s); + void remove(curl_socket_t s); + + // Copy mFileDescriptors to an internal fd_set that is returned by access(). + // Returns if all fds could be copied (complete) and/or if the resulting fd_set is empty. + refresh_t refresh(void); + + // Return a pointer to the underlaying fd_set. + fd_set* access(void) { return &mFdSet; } + + // Return the largest fd set in mFdSet by refresh. + int get_max_fd(void) const { return mMaxFdSet; } + + // Return true if a filedescriptor is set in mFileDescriptors (used for debugging). + bool contains(curl_socket_t s) const; + + // Return true if a filedescriptor is set in mFdSet. + bool is_set(curl_socket_t s) const; + + // Clear filedescriptor in mFdSet. + void clr(curl_socket_t fd); + + // Iterate over all file descriptors that were set by refresh and are still set in mFdSet. + void reset(void); // Reset the iterator. + int get(void) const; // Return next filedescriptor, or -1 when there are no more. + // Only valid if reset() was called after the last call to refresh(). + void next(void); // Advance to next filedescriptor. + + private: + curl_socket_t* mFileDescriptors; + size_t mSize; // Size of mFileDescriptors array. + int mNrFds; // The number of filedescriptors in the array. + int mMaxFd; // The largest filedescriptor in the array, or -1 when it is empty. + int mNext; // The index of the first file descriptor to start copying, the next call to refresh(). + + fd_set mFdSet; // Output variable for select(). (Re)initialized by calling refresh(). + int mMaxFdSet; // The largest filedescriptor set in mFdSet by refresh(), or -1 when it was empty. + + std::vector mCopiedFileDescriptors; // File descriptors copied by refresh to mFdSet. + std::vector::iterator mIter; // Index into mCopiedFileDescriptors for next(); loop variable. +}; + +//----------------------------------------------------------------------------- +// MultiHandle + +// This class adds member functions that will only be called from the AICurlThread thread. +// This class guarantees that all added easy handles will be removed from the multi handle +// before the multi handle is cleaned up, as is required by libcurl. +class MultiHandle : public CurlMultiHandle +{ + public: + MultiHandle(void); + ~MultiHandle(); + + // Add/remove an easy handle to/from a multi session. + CURLMcode add_easy_request(AICurlEasyRequest const& easy_request); + CURLMcode remove_easy_request(AICurlEasyRequest const& easy_request); + + // Reads/writes available data from a particular socket (non-blocking). + CURLMcode socket_action(curl_socket_t sockfd, int ev_bitmask); + + // Set data to association with an internal socket. + CURLMcode assign(curl_socket_t sockfd, void* sockptr); + + // Read multi stack informationals. + CURLMsg const* info_read(int* msgs_in_queue) const; + + private: + typedef std::set addedEasyRequests_type; + addedEasyRequests_type mAddedEasyRequests; + + bool mHandleAddedOrRemoved; // Set when an easy handle was added or removed, reset in check_run_count(). + int mPrevRunningHandles; // The last value of mRunningHandles that check_run_count() was called with. + int mRunningHandles; // The last value returned by curl_multi_socket_action. + long mTimeOut; // The last time out in ms as set by the call back CURLMOPT_TIMERFUNCTION. + + private: + static int socket_callback(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp); + static int timer_callback(CURLM* multi, long timeout_ms, void* userp); + + public: + // Returns the number of active easy handles as reported by the last call to curl_multi_socket_action. + int getRunningHandles(void) const { return mRunningHandles; } + + // Returns how long to wait for socket action before calling socket_action(CURL_SOCKET_TIMEOUT, 0), in ms. + int getTimeOut(void) const { return mTimeOut; } + + // This is called before sleeping, after calling (one or more times) socket_action. + void check_run_count(void); + + public: + //----------------------------------------------------------------------------- + // Curl socket administration: + + PollSet mReadPollSet; + PollSet mWritePollSet; +}; + +} // namespace curlthread +} // namespace AICurlPrivate + +// Thread safe, noncopyable curl multi handle. +// This class wraps MultiHandle for thread-safety. +// AIThreadSafeSingleThreadDC cannot be copied, but that is OK as we don't need that (or want that); +// this class provides a thread-local singleton (exactly one instance per thread), and because it +// can't be copied, that guarantees that the CURLM* handle is never used concurrently, which is +// not allowed by libcurl. +class AICurlMultiHandle : public AIThreadSafeSingleThreadDC, public LLThreadLocalDataMember { + public: + static AICurlMultiHandle& getInstance(void); + private: + // Use getInstance(). + AICurlMultiHandle(void) { } +}; + +typedef AISTAccessConst AICurlMultiHandle_rat; +typedef AISTAccess AICurlMultiHandle_wat; + +#define AICurlPrivate DONTUSE_AICurlPrivate + +#endif diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index b2f5ce471..585e4644b 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -26,32 +26,6 @@ * $/LicenseInfo$ */ -#if LL_WINDOWS -#define SAFE_SSL 1 -#elif LL_DARWIN -#define SAFE_SSL 1 -#else -#define SAFE_SSL 1 -#endif - -#include "linden_common.h" - -#include "llcurl.h" - -#include -#include -#include -#if SAFE_SSL -#include -#endif - -#include "llbufferstream.h" -#include "llproxy.h" -#include "llsdserialize.h" -#include "llstl.h" -#include "llthread.h" -#include "lltimer.h" - ////////////////////////////////////////////////////////////////////////////// /* The trick to getting curl to do keep-alives is to reuse the @@ -75,179 +49,37 @@ static const S32 MULTI_PERFORM_CALL_REPEAT = 5; static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; -// DEBUG // -S32 gCurlEasyCount = 0; -S32 gCurlMultiCount = 0; - -////////////////////////////////////////////////////////////////////////////// - //static -std::vector LLCurl::sSSLMutex; -std::string LLCurl::sCAPath; -std::string LLCurl::sCAFile; -LLCurlThread* LLCurl::sCurlThread = NULL ; -LLMutex* LLCurl::sHandleMutexp = NULL ; -S32 LLCurl::sTotalHandles = 0 ; -bool LLCurl::sNotQuitting = true; -F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds +F32 LLCurl::sCurlRequestTimeOut = 120.f; //seconds S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined). -void check_curl_code(CURLcode code) -{ - if (code != CURLE_OK) - { - // linux appears to throw a curl error once per session for a bad initialization - // at a pretty random time (when enabling cookies). - llinfos << "curl error detected: " << curl_easy_strerror(code) << llendl; - } -} - -void check_curl_multi_code(CURLMcode code) -{ - if (code != CURLM_OK) - { - // linux appears to throw a curl error once per session for a bad initialization - // at a pretty random time (when enabling cookies). - llinfos << "curl multi error detected: " << curl_multi_strerror(code) << llendl; - } -} - -//static -void LLCurl::setCAPath(const std::string& path) -{ - sCAPath = path; -} - -//static -void LLCurl::setCAFile(const std::string& file) -{ - sCAFile = file; -} - -//static -std::string LLCurl::getVersionString() -{ - return std::string(curl_version()); -} - ////////////////////////////////////////////////////////////////////////////// -LLCurl::Responder::Responder() - : mReferenceCount(0) -{ -} - -LLCurl::Responder::~Responder() -{ -} - -// virtual -void LLCurl::Responder::errorWithContent( - U32 status, - const std::string& reason, - const LLSD&) -{ - error(status, reason); -} - -// virtual -void LLCurl::Responder::error(U32 status, const std::string& reason) -{ - llinfos << mURL << " [" << status << "]: " << reason << llendl; -} - -// virtual -void LLCurl::Responder::result(const LLSD& content) -{ -} - -void LLCurl::Responder::setURL(const std::string& url) -{ - mURL = url; -} - -// virtual -void LLCurl::Responder::completedRaw( - U32 status, - const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) -{ - LLSD content; - LLBufferStream istr(channels, buffer.get()); - if (!LLSDSerialize::fromXML(content, istr)) - { - llinfos << "Failed to deserialize LLSD. " << mURL << " [" << status << "]: " << reason << llendl; - } - - completed(status, reason, content); -} - -// virtual -void LLCurl::Responder::completed(U32 status, const std::string& reason, const LLSD& content) -{ - if (isGoodStatus(status)) - { - result(content); - } - else - { - errorWithContent(status, reason, content); - } -} - -//virtual -void LLCurl::Responder::completedHeader(U32 status, const std::string& reason, const LLSD& content) -{ - -} - -//namespace boost -//{ - void intrusive_ptr_add_ref(LLCurl::Responder* p) - { - ++p->mReferenceCount; - } - - void intrusive_ptr_release(LLCurl::Responder* p) - { - if (p && 0 == --p->mReferenceCount) - { - delete p; - } - } -//}; - - -////////////////////////////////////////////////////////////////////////////// - -std::set LLCurl::Easy::sFreeHandles; -std::set LLCurl::Easy::sActiveHandles; -LLMutex* LLCurl::Easy::sHandleMutexp = NULL ; +AIThreadSafeSimpleDC LLCurl::Easy::sHandles; //static CURL* LLCurl::Easy::allocEasyHandle() { - llassert(LLCurl::getCurlThread()) ; + llassert(*AIAccess(LLCurl::getCurlThread())) ; CURL* ret = NULL; - LLMutexLock lock(sHandleMutexp) ; + //*** Multi-threaded. + AIAccess handles_w(sHandles); - if (sFreeHandles.empty()) + if (handles_w->free.empty()) { ret = LLCurl::newEasyHandle(); } else { - ret = *(sFreeHandles.begin()); - sFreeHandles.erase(ret); - curl_easy_reset(ret); + ret = *(handles_w->free.begin()); + handles_w->free.erase(ret); } if (ret) { - sActiveHandles.insert(ret); + handles_w->active.insert(ret); } return ret; @@ -256,6 +88,9 @@ CURL* LLCurl::Easy::allocEasyHandle() //static void LLCurl::Easy::releaseEasyHandle(CURL* handle) { + DoutEntering(dc::curl, "LLCurl::Easy::releaseEasyHandle(" << (void*)handle << ")"); + BACKTRACE; + static const S32 MAX_NUM_FREE_HANDLES = 32 ; if (!handle) @@ -264,14 +99,17 @@ void LLCurl::Easy::releaseEasyHandle(CURL* handle) //llerrs << "handle cannot be NULL!" << llendl; } - LLMutexLock lock(sHandleMutexp) ; - if (sActiveHandles.find(handle) != sActiveHandles.end()) - { - sActiveHandles.erase(handle); + //*** Multi-Threaded (logout only?) + AIAccess handles_w(sHandles); - if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES) + if (handles_w->active.find(handle) != handles_w->active.end()) + { + handles_w->active.erase(handle); + + if (handles_w->free.size() < MAX_NUM_FREE_HANDLES) { - sFreeHandles.insert(handle); + curl_easy_reset(handle); + handles_w->free.insert(handle); } else { @@ -284,780 +122,16 @@ void LLCurl::Easy::releaseEasyHandle(CURL* handle) } } -LLCurl::Easy::Easy() - : mHeaders(NULL), - mCurlEasyHandle(NULL) -{ - mErrorBuffer[0] = 0; -} - -LLCurl::Easy* LLCurl::Easy::getEasy() -{ - Easy* easy = new Easy(); - easy->mCurlEasyHandle = allocEasyHandle(); - - if (!easy->mCurlEasyHandle) - { - // this can happen if we have too many open files (fails in c-ares/ares_init.c) - llwarns << "allocEasyHandle() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; - delete easy; - return NULL; - } - - // set no DNS caching as default for all easy handles. This prevents them adopting a - // multi handles cache if they are added to one. - CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); - check_curl_code(result); - - ++gCurlEasyCount; - return easy; -} - LLCurl::Easy::~Easy() { - releaseEasyHandle(mCurlEasyHandle); - --gCurlEasyCount; - curl_slist_free_all(mHeaders); - for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); - - if (mResponder && LLCurl::sNotQuitting) //aborted + AISTAccess responder_w(mResponder); + if (*responder_w && LLCurl::getNotQuitting()) //aborted { std::string reason("Request timeout, aborted.") ; - mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort + (*responder_w)->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort reason, mChannels, mOutput); } - mResponder = NULL; -} - -void LLCurl::Easy::resetState() -{ - curl_easy_reset(mCurlEasyHandle); - - if (mHeaders) - { - curl_slist_free_all(mHeaders); - mHeaders = NULL; - } - - mRequest.str(""); - mRequest.clear(); - - mOutput.reset(); - - mInput.str(""); - mInput.clear(); - - mErrorBuffer[0] = 0; - - mHeaderOutput.str(""); - mHeaderOutput.clear(); -} - -void LLCurl::Easy::setErrorBuffer() -{ - setopt(CURLOPT_ERRORBUFFER, &mErrorBuffer); -} - -const char* LLCurl::Easy::getErrorBuffer() -{ - return mErrorBuffer; -} - -void LLCurl::Easy::setCA() -{ - if (!sCAPath.empty()) - { - setoptString(CURLOPT_CAPATH, sCAPath); - } - if (!sCAFile.empty()) - { - setoptString(CURLOPT_CAINFO, sCAFile); - } -} - -void LLCurl::Easy::setHeaders() -{ - setopt(CURLOPT_HTTPHEADER, mHeaders); -} - -void LLCurl::Easy::getTransferInfo(LLCurl::TransferInfo* info) -{ - check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SIZE_DOWNLOAD, &info->mSizeDownload)); - check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_TOTAL_TIME, &info->mTotalTime)); - check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_SPEED_DOWNLOAD, &info->mSpeedDownload)); -} - -U32 LLCurl::Easy::report(CURLcode code) -{ - U32 responseCode = 0; - std::string responseReason; - - if (code == CURLE_OK) - { - check_curl_code(curl_easy_getinfo(mCurlEasyHandle, CURLINFO_RESPONSE_CODE, &responseCode)); - //*TODO: get reason from first line of mHeaderOutput - } - else - { - responseCode = 499; - responseReason = strerror(code) + " : " + mErrorBuffer; - setopt(CURLOPT_FRESH_CONNECT, TRUE); - } - - if (mResponder) - { - mResponder->completedRaw(responseCode, responseReason, mChannels, mOutput); - mResponder = NULL; - } - - resetState(); - return responseCode; -} - -// Note: these all assume the caller tracks the value (i.e. keeps it persistant) -void LLCurl::Easy::setopt(CURLoption option, S32 value) -{ - CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); - check_curl_code(result); -} - -void LLCurl::Easy::setopt(CURLoption option, void* value) -{ - CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); - check_curl_code(result); -} - -void LLCurl::Easy::setopt(CURLoption option, char* value) -{ - CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, value); - check_curl_code(result); -} - -// Note: this copies the string so that the caller does not have to keep it around -void LLCurl::Easy::setoptString(CURLoption option, const std::string& value) -{ - char* tstring = new char[value.length()+1]; - strcpy(tstring, value.c_str()); - mStrings.push_back(tstring); - CURLcode result = curl_easy_setopt(mCurlEasyHandle, option, tstring); - check_curl_code(result); -} - -void LLCurl::Easy::slist_append(const char* str) -{ - mHeaders = curl_slist_append(mHeaders, str); -} - -size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) -{ - LLCurl::Easy* easy = (LLCurl::Easy*)user_data; - - S32 n = size * nmemb; - S32 startpos = easy->getInput().tellg(); - easy->getInput().seekg(0, std::ios::end); - S32 endpos = easy->getInput().tellg(); - easy->getInput().seekg(startpos, std::ios::beg); - S32 maxn = endpos - startpos; - n = llmin(n, maxn); - easy->getInput().read((char*)data, n); - - return n; -} - -size_t curlWriteCallback(char* data, size_t size, size_t nmemb, void* user_data) -{ - LLCurl::Easy* easy = (LLCurl::Easy*)user_data; - - S32 n = size * nmemb; - easy->getOutput()->append(easy->getChannels().in(), (const U8*)data, n); - - return n; -} - -size_t curlHeaderCallback(void* data, size_t size, size_t nmemb, void* user_data) -{ - LLCurl::Easy* easy = (LLCurl::Easy*)user_data; - - size_t n = size * nmemb; - easy->getHeaderOutput().write((const char*)data, n); - - return n; -} - -void LLCurl::Easy::prepRequest(const std::string& url, - const std::vector& headers, - ResponderPtr responder, S32 time_out, bool post) -{ - resetState(); - - if (post) setoptString(CURLOPT_ENCODING, ""); - - //setopt(CURLOPT_VERBOSE, 1); // useful for debugging - setopt(CURLOPT_NOSIGNAL, 1); - - // Set the CURL options for either Socks or HTTP proxy - LLProxy::getInstance()->applyProxySettings(this); - - mOutput.reset(new LLBufferArray); - mOutput->setThreaded(true); - setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); - setopt(CURLOPT_WRITEDATA, (void*)this); - - setopt(CURLOPT_READFUNCTION, (void*)&curlReadCallback); - setopt(CURLOPT_READDATA, (void*)this); - - setopt(CURLOPT_HEADERFUNCTION, (void*)&curlHeaderCallback); - setopt(CURLOPT_HEADERDATA, (void*)this); - - // Allow up to five redirects - if (responder && responder->followRedir()) - { - setopt(CURLOPT_FOLLOWLOCATION, 1); - setopt(CURLOPT_MAXREDIRS, MAX_REDIRECTS); - } - - setErrorBuffer(); - setCA(); - - setopt(CURLOPT_SSL_VERIFYPEER, true); - - //don't verify host name so urls with scrubbed host names will work (improves DNS performance) - setopt(CURLOPT_SSL_VERIFYHOST, 0); - setopt(CURLOPT_TIMEOUT, llmax(time_out, CURL_REQUEST_TIMEOUT)); - - setoptString(CURLOPT_URL, url); - - mResponder = responder; - - if (!post) - { - slist_append("Connection: keep-alive"); - slist_append("Keep-alive: 300"); - // Accept and other headers - for (std::vector::const_iterator iter = headers.begin(); - iter != headers.end(); ++iter) - { - slist_append((*iter).c_str()); - } - } -} - -//////////////////////////////////////////////////////////////////////////// -LLCurl::Multi::Multi(F32 idle_time_out) - : mQueued(0), - mErrorCount(0), - mState(STATE_READY), - mDead(FALSE), - mValid(TRUE), - mMutexp(NULL), - mDeletionMutexp(NULL), - mEasyMutexp(NULL) -{ - mCurlMultiHandle = LLCurl::newMultiHandle(); - if (!mCurlMultiHandle) - { - llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; - mCurlMultiHandle = LLCurl::newMultiHandle(); - } - - //llassert_always(mCurlMultiHandle); - - if(mCurlMultiHandle) - { - if(LLCurl::getCurlThread()->getThreaded()) - { - mMutexp = new LLMutex; - mDeletionMutexp = new LLMutex; - mEasyMutexp = new LLMutex; - } - LLCurl::getCurlThread()->addMulti(this) ; - - mIdleTimeOut = idle_time_out ; - if(mIdleTimeOut < LLCurl::sCurlRequestTimeOut) - { - mIdleTimeOut = LLCurl::sCurlRequestTimeOut ; - } - - ++gCurlMultiCount; - } -} - -LLCurl::Multi::~Multi() -{ - cleanup(true); - - delete mDeletionMutexp ; - mDeletionMutexp = NULL ; -} - -void LLCurl::Multi::cleanup(bool deleted) -{ - if(!mCurlMultiHandle) - { - return ; //nothing to clean. - } - llassert_always(deleted || !mValid) ; - - LLMutexLock lock(mDeletionMutexp); - - // Clean up active - for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); - iter != mEasyActiveList.end(); ++iter) - { - Easy* easy = *iter; - check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); - - if(deleted) - { - easy->mResponder = NULL ; //avoid triggering mResponder. - } - delete easy; - } - mEasyActiveList.clear(); - mEasyActiveMap.clear(); - - // Clean up freed - for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); - mEasyFreeList.clear(); - - check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle)); - mCurlMultiHandle = NULL ; - - delete mMutexp ; - mMutexp = NULL ; - delete mEasyMutexp ; - mEasyMutexp = NULL ; - - mQueued = 0 ; - mState = STATE_COMPLETED; - - --gCurlMultiCount; - - return ; -} - -void LLCurl::Multi::lock() -{ - if(mMutexp) - { - mMutexp->lock() ; - } -} - -void LLCurl::Multi::unlock() -{ - if(mMutexp) - { - mMutexp->unlock() ; - } -} - -void LLCurl::Multi::markDead() -{ - { - LLMutexLock lock(mDeletionMutexp) ; - - if(mCurlMultiHandle != NULL) - { - mDead = TRUE ; - LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; - - return; - } - } - - //not valid, delete it. - delete this; -} - -void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state) -{ - lock() ; - mState = state ; - unlock() ; - - if(mState == STATE_READY) - { - LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_NORMAL) ; - } -} - -LLCurl::Multi::ePerformState LLCurl::Multi::getState() -{ - return mState; -} - -bool LLCurl::Multi::isCompleted() -{ - return STATE_COMPLETED == getState() ; -} - -bool LLCurl::Multi::waitToComplete() -{ - if(!isValid()) - { - return true ; - } - - if(!mMutexp) //not threaded - { - doPerform() ; - return true ; - } - - bool completed = (STATE_COMPLETED == mState) ; - if(!completed) - { - LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ; - } - - return completed; -} - -CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) -{ - LLMutexLock lock(mMutexp) ; - - CURLMsg* curlmsg = curl_multi_info_read(mCurlMultiHandle, msgs_in_queue); - return curlmsg; -} - -//return true if dead -bool LLCurl::Multi::doPerform() -{ - LLMutexLock lock(mDeletionMutexp) ; - - bool dead = mDead ; - - if(mDead) - { - setState(STATE_COMPLETED); - mQueued = 0 ; - } - else if(getState() != STATE_COMPLETED) - { - setState(STATE_PERFORMING); - - S32 q = 0; - for (S32 call_count = 0; - call_count < MULTI_PERFORM_CALL_REPEAT; - call_count++) - { - LLMutexLock lock(mMutexp) ; - - //WARNING: curl_multi_perform will block for many hundreds of milliseconds - // NEVER call this from the main thread, and NEVER allow the main thread to - // wait on a mutex held by this thread while curl_multi_perform is executing - CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); - if (CURLM_CALL_MULTI_PERFORM != code || q == 0) - { - check_curl_multi_code(code); - - break; - } - } - - mQueued = q; - setState(STATE_COMPLETED) ; - mIdleTimer.reset() ; - } - else if(!mValid && mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it. - { - dead = true ; - } - else if(mValid && mIdleTimer.getElapsedTimeF32() > mIdleTimeOut - 1.f) //idle for too long, mark it invalid. - { - mValid = FALSE ; - } - - return dead ; -} - -S32 LLCurl::Multi::process() -{ - if(!isValid()) - { - return 0 ; - } - - waitToComplete() ; - - if (getState() != STATE_COMPLETED) - { - return 0; - } - - CURLMsg* msg; - int msgs_in_queue; - - S32 processed = 0; - while ((msg = info_read(&msgs_in_queue))) - { - ++processed; - if (msg->msg == CURLMSG_DONE) - { - U32 response = 0; - Easy* easy = NULL ; - - { - LLMutexLock lock(mEasyMutexp) ; - easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); - if (iter != mEasyActiveMap.end()) - { - easy = iter->second; - } - } - - if(easy) - { - response = easy->report(msg->data.result); - removeEasy(easy); - } - else - { - response = 499; - //*TODO: change to llwarns - llerrs << "cleaned up curl request completed!" << llendl; - } - if (response >= 400) - { - // failure of some sort, inc mErrorCount for debugging and flagging multi for destruction - ++mErrorCount; - } - } - } - - setState(STATE_READY); - - return processed; -} - -LLCurl::Easy* LLCurl::Multi::allocEasy() -{ - Easy* easy = 0; - - if (mEasyFreeList.empty()) - { - easy = Easy::getEasy(); - } - else - { - LLMutexLock lock(mEasyMutexp) ; - easy = *(mEasyFreeList.begin()); - mEasyFreeList.erase(easy); - } - if (easy) - { - LLMutexLock lock(mEasyMutexp) ; - mEasyActiveList.insert(easy); - mEasyActiveMap[easy->getCurlHandle()] = easy; - } - return easy; -} - -bool LLCurl::Multi::addEasy(Easy* easy) -{ - LLMutexLock lock(mMutexp) ; - CURLMcode mcode = curl_multi_add_handle(mCurlMultiHandle, easy->getCurlHandle()); - check_curl_multi_code(mcode); - //if (mcode != CURLM_OK) - //{ - // llwarns << "Curl Error: " << curl_multi_strerror(mcode) << llendl; - // return false; - //} - return true; -} - -void LLCurl::Multi::easyFree(Easy* easy) -{ - if(mEasyMutexp) - { - mEasyMutexp->lock() ; - } - - mEasyActiveList.erase(easy); - mEasyActiveMap.erase(easy->getCurlHandle()); - - if (mEasyFreeList.size() < EASY_HANDLE_POOL_SIZE) - { - mEasyFreeList.insert(easy); - - if(mEasyMutexp) - { - mEasyMutexp->unlock() ; - } - - easy->resetState(); - } - else - { - if(mEasyMutexp) - { - mEasyMutexp->unlock() ; - } - delete easy; - } -} - -void LLCurl::Multi::removeEasy(Easy* easy) -{ - { - LLMutexLock lock(mMutexp) ; - check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); - } - easyFree(easy); -} - -//------------------------------------------------------------ -//LLCurlThread -LLCurlThread::CurlRequest::CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread) : - LLQueuedThread::QueuedRequest(handle, LLQueuedThread::PRIORITY_NORMAL, FLAG_AUTO_COMPLETE), - mMulti(multi), - mCurlThread(curl_thread) -{ -} - -LLCurlThread::CurlRequest::~CurlRequest() -{ - if(mMulti) - { - mCurlThread->deleteMulti(mMulti) ; - mMulti = NULL ; - } -} - -bool LLCurlThread::CurlRequest::processRequest() -{ - bool completed = true ; - if(mMulti) - { - completed = mCurlThread->doMultiPerform(mMulti) ; - - if(!completed) - { - setPriority(LLQueuedThread::PRIORITY_LOW) ; - } - } - - return completed ; -} - -void LLCurlThread::CurlRequest::finishRequest(bool completed) -{ - if(mMulti->isDead()) - { - mCurlThread->deleteMulti(mMulti) ; - } - else - { - mCurlThread->cleanupMulti(mMulti) ; //being idle too long, remove the request. - } - - mMulti = NULL ; -} - -LLCurlThread::LLCurlThread(bool threaded) : - LLQueuedThread("curlthread", threaded) -{ -} - -//virtual -LLCurlThread::~LLCurlThread() -{ -} - -S32 LLCurlThread::update(F32 max_time_ms) -{ - return LLQueuedThread::update(max_time_ms); -} - -void LLCurlThread::addMulti(LLCurl::Multi* multi) -{ - multi->mHandle = generateHandle() ; - - CurlRequest* req = new CurlRequest(multi->mHandle, multi, this) ; - - if (!addRequest(req)) - { - llwarns << "curl request added when the thread is quitted" << llendl; - } -} - -void LLCurlThread::killMulti(LLCurl::Multi* multi) -{ - if(!multi) - { - return ; - } - - multi->markDead() ; -} - -//private -bool LLCurlThread::doMultiPerform(LLCurl::Multi* multi) -{ - return multi->doPerform() ; -} - -//private -void LLCurlThread::deleteMulti(LLCurl::Multi* multi) -{ - delete multi ; -} - -//private -void LLCurlThread::cleanupMulti(LLCurl::Multi* multi) -{ - multi->cleanup() ; - if(multi->isDead()) //check if marked dead during cleaning up. - { - deleteMulti(multi) ; - } -} - -//------------------------------------------------------------ - -//static -std::string LLCurl::strerror(CURLcode errorcode) -{ - return std::string(curl_easy_strerror(errorcode)); -} - -//////////////////////////////////////////////////////////////////////////// -// For generating a simple request for data -// using one multi and one easy per request - -LLCurlRequest::LLCurlRequest() : - mActiveMulti(NULL), - mActiveRequestCount(0) -{ - mProcessing = FALSE; -} - -LLCurlRequest::~LLCurlRequest() -{ - //stop all Multi handle background threads - for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ++iter) - { - LLCurl::getCurlThread()->killMulti(*iter) ; - } - mMultiSet.clear() ; -} - -void LLCurlRequest::addMulti() -{ - LLCurl::Multi* multi = new LLCurl::Multi(); - if(!multi->isValid()) - { - LLCurl::getCurlThread()->killMulti(multi) ; - mActiveMulti = NULL ; - mActiveRequestCount = 0 ; - return; - } - - mMultiSet.insert(multi); - mActiveMulti = multi; - mActiveRequestCount = 0; + *responder_w = NULL; } LLCurl::Easy* LLCurlRequest::allocEasy() @@ -1078,531 +152,3 @@ LLCurl::Easy* LLCurlRequest::allocEasy() LLCurl::Easy* easy = mActiveMulti->allocEasy(); return easy; } - -bool LLCurlRequest::addEasy(LLCurl::Easy* easy) -{ - llassert_always(mActiveMulti); - - if (mProcessing) - { - llerrs << "Posting to a LLCurlRequest instance from within a responder is not allowed (causes DNS timeouts)." << llendl; - } - bool res = mActiveMulti->addEasy(easy); - return res; -} - -void LLCurlRequest::get(const std::string& url, LLCurl::ResponderPtr responder) -{ - getByteRange(url, headers_t(), 0, -1, responder); -} - -bool LLCurlRequest::getByteRange(const std::string& url, - const headers_t& headers, - S32 offset, S32 length, - LLCurl::ResponderPtr responder) -{ - LLCurl::Easy* easy = allocEasy(); - if (!easy) - { - return false; - } - easy->prepRequest(url, headers, responder); - easy->setopt(CURLOPT_HTTPGET, 1); - if (length > 0) - { - std::string range = llformat("Range: bytes=%d-%d", offset,offset+length-1); - easy->slist_append(range.c_str()); - } - easy->setHeaders(); - bool res = addEasy(easy); - return res; -} - -bool LLCurlRequest::post(const std::string& url, - const headers_t& headers, - const LLSD& data, - LLCurl::ResponderPtr responder, S32 time_out) -{ - LLCurl::Easy* easy = allocEasy(); - if (!easy) - { - return false; - } - easy->prepRequest(url, headers, responder, time_out); - - LLSDSerialize::toXML(data, easy->getInput()); - S32 bytes = easy->getInput().str().length(); - - easy->setopt(CURLOPT_POST, 1); - easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); - easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); - - easy->slist_append("Content-Type: application/llsd+xml"); - easy->setHeaders(); - - lldebugs << "POSTING: " << bytes << " bytes." << llendl; - bool res = addEasy(easy); - return res; -} - -bool LLCurlRequest::post(const std::string& url, - const headers_t& headers, - const std::string& data, - LLCurl::ResponderPtr responder, S32 time_out) -{ - LLCurl::Easy* easy = allocEasy(); - if (!easy) - { - return false; - } - easy->prepRequest(url, headers, responder, time_out); - - easy->getInput().write(data.data(), data.size()); - S32 bytes = easy->getInput().str().length(); - - easy->setopt(CURLOPT_POST, 1); - easy->setopt(CURLOPT_POSTFIELDS, (void*)NULL); - easy->setopt(CURLOPT_POSTFIELDSIZE, bytes); - - easy->slist_append("Content-Type: application/octet-stream"); - easy->setHeaders(); - - lldebugs << "POSTING: " << bytes << " bytes." << llendl; - bool res = addEasy(easy); - return res; -} - -// Note: call once per frame -S32 LLCurlRequest::process() -{ - S32 res = 0; - - mProcessing = TRUE; - for (curlmulti_set_t::iterator iter = mMultiSet.begin(); - iter != mMultiSet.end(); ) - { - curlmulti_set_t::iterator curiter = iter++; - LLCurl::Multi* multi = *curiter; - - if(!multi->isValid()) - { - if(multi == mActiveMulti) - { - mActiveMulti = NULL ; - mActiveRequestCount = 0 ; - } - mMultiSet.erase(curiter) ; - LLCurl::getCurlThread()->killMulti(multi) ; - continue ; - } - - S32 tres = multi->process(); - res += tres; - if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) - { - mMultiSet.erase(curiter); - LLCurl::getCurlThread()->killMulti(multi); - } - } - mProcessing = FALSE; - return res; -} - -S32 LLCurlRequest::getQueued() -{ - S32 queued = 0; - for (curlmulti_set_t::iterator iter = mMultiSet.begin(); - iter != mMultiSet.end(); ) - { - curlmulti_set_t::iterator curiter = iter++; - LLCurl::Multi* multi = *curiter; - - if(!multi->isValid()) - { - if(multi == mActiveMulti) - { - mActiveMulti = NULL ; - mActiveRequestCount = 0 ; - } - LLCurl::getCurlThread()->killMulti(multi); - mMultiSet.erase(curiter) ; - continue ; - } - - queued += multi->mQueued; - if (multi->getState() != LLCurl::Multi::STATE_READY) - { - ++queued; - } - } - return queued; -} - -//////////////////////////////////////////////////////////////////////////// -// For generating one easy request -// associated with a single multi request - -LLCurlEasyRequest::LLCurlEasyRequest() - : mRequestSent(false), - mResultReturned(false) -{ - mMulti = new LLCurl::Multi(); - - if(mMulti->isValid()) - { - mEasy = mMulti->allocEasy(); - if (mEasy) - { - mEasy->setErrorBuffer(); - mEasy->setCA(); - // Set proxy settings if configured to do so. - LLProxy::getInstance()->applyProxySettings(mEasy); - } - } - else - { - LLCurl::getCurlThread()->killMulti(mMulti) ; - mEasy = NULL ; - mMulti = NULL ; - } -} - -LLCurlEasyRequest::~LLCurlEasyRequest() -{ - LLCurl::getCurlThread()->killMulti(mMulti) ; -} - -void LLCurlEasyRequest::setopt(CURLoption option, S32 value) -{ - if (isValid() && mEasy) - { - mEasy->setopt(option, value); - } -} - -void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value) -{ - if (isValid() && mEasy) - { - mEasy->setoptString(option, value); - } -} - -void LLCurlEasyRequest::setPost(char* postdata, S32 size) -{ - if (isValid() && mEasy) - { - mEasy->setopt(CURLOPT_POST, 1); - mEasy->setopt(CURLOPT_POSTFIELDS, postdata); - mEasy->setopt(CURLOPT_POSTFIELDSIZE, size); - } -} - -void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata) -{ - if (isValid() && mEasy) - { - mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback); - mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER - } -} - -void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) -{ - if (isValid() && mEasy) - { - mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback); - mEasy->setopt(CURLOPT_WRITEDATA, userdata); - } -} - -void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) -{ - if (isValid() && mEasy) - { - mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback); - mEasy->setopt(CURLOPT_READDATA, userdata); - } -} - -void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata) -{ - if (isValid() && mEasy) - { - mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback); - mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata); - } -} - -void LLCurlEasyRequest::slist_append(const char* str) -{ - if (isValid() && mEasy) - { - mEasy->slist_append(str); - } -} - -void LLCurlEasyRequest::sendRequest(const std::string& url) -{ - llassert_always(!mRequestSent); - mRequestSent = true; - lldebugs << url << llendl; - if (isValid() && mEasy) - { - mEasy->setHeaders(); - mEasy->setoptString(CURLOPT_URL, url); - mMulti->addEasy(mEasy); - } -} - -void LLCurlEasyRequest::requestComplete() -{ - llassert_always(mRequestSent); - mRequestSent = false; - if (isValid() && mEasy) - { - mMulti->removeEasy(mEasy); - } -} - -// Usage: Call getRestult until it returns false (no more messages) -bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) -{ - if(!isValid()) - { - return false ; - } - if (!mMulti->isCompleted()) - { //we're busy, try again later - return false; - } - mMulti->setState(LLCurl::Multi::STATE_READY) ; - - if (!mEasy) - { - // Special case - we failed to initialize a curl_easy (can happen if too many open files) - // Act as though the request failed to connect - if (mResultReturned) - { - return false; - } - else - { - *result = CURLE_FAILED_INIT; - mResultReturned = true; - return true; - } - } - // In theory, info_read might return a message with a status other than CURLMSG_DONE - // In practice for all messages returned, msg == CURLMSG_DONE - // Ignore other messages just in case - while(1) - { - S32 q; - CURLMsg* curlmsg = info_read(&q, info); - if (curlmsg) - { - if (curlmsg->msg == CURLMSG_DONE) - { - *result = curlmsg->data.result; - return true; - } - // else continue - } - else - { - return false; - } - } -} - -// private -CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info) -{ - if (mEasy) - { - CURLMsg* curlmsg = mMulti->info_read(q); - if (curlmsg && curlmsg->msg == CURLMSG_DONE) - { - if (info) - { - mEasy->getTransferInfo(info); - } - } - return curlmsg; - } - return NULL; -} - -std::string LLCurlEasyRequest::getErrorString() -{ - return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); -} - -//////////////////////////////////////////////////////////////////////////// - -#if SAFE_SSL -//static -void LLCurl::ssl_locking_callback(int mode, int type, const char *file, int line) -{ - if (mode & CRYPTO_LOCK) - { - LLCurl::sSSLMutex[type]->lock(); - } - else - { - LLCurl::sSSLMutex[type]->unlock(); - } -} - -//static -unsigned long LLCurl::ssl_thread_id(void) -{ - return LLThread::currentID(); -} -#endif - -void LLCurl::initClass(F32 curl_reuest_timeout, S32 max_number_handles, bool multi_threaded) -{ - sCurlRequestTimeOut = curl_reuest_timeout ; //seconds - sMaxHandles = max_number_handles ; //max number of handles, (multi handles and easy handles combined). - - // Do not change this "unless you are familiar with and mean to control - // internal operations of libcurl" - // - http://curl.haxx.se/libcurl/c/curl_global_init.html - CURLcode code = curl_global_init(CURL_GLOBAL_ALL); - - check_curl_code(code); - -#if SAFE_SSL - S32 mutex_count = CRYPTO_num_locks(); - for (S32 i=0; iupdate(1)) //finish all tasks - { - break ; - } - } - sCurlThread->shutdown() ; - delete sCurlThread ; - sCurlThread = NULL ; - -#if SAFE_SSL - CRYPTO_set_locking_callback(NULL); - for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); -#endif - - for (std::set::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) - { - CURL* curl = *iter; - LLCurl::deleteEasyHandle(curl); - } - - Easy::sFreeHandles.clear(); - - delete Easy::sHandleMutexp ; - Easy::sHandleMutexp = NULL ; - - delete sHandleMutexp ; - sHandleMutexp = NULL ; - - llassert(Easy::sActiveHandles.empty()); -} - -//static -CURLM* LLCurl::newMultiHandle() -{ - LLMutexLock lock(sHandleMutexp) ; - - if(sTotalHandles + 1 > sMaxHandles) - { - llwarns << "no more handles available." << llendl ; - return NULL ; //failed - } - sTotalHandles++; - - CURLM* ret = curl_multi_init() ; - if(!ret) - { - llwarns << "curl_multi_init failed." << llendl ; - } - - return ret ; -} - -//static -CURLMcode LLCurl::deleteMultiHandle(CURLM* handle) -{ - if(handle) - { - LLMutexLock lock(sHandleMutexp) ; - sTotalHandles-- ; - return curl_multi_cleanup(handle) ; - } - return CURLM_OK ; -} - -//static -CURL* LLCurl::newEasyHandle() -{ - LLMutexLock lock(sHandleMutexp) ; - - if(sTotalHandles + 1 > sMaxHandles) - { - llwarns << "no more handles available." << llendl ; - return NULL ; //failed - } - sTotalHandles++; - - CURL* ret = curl_easy_init() ; - if(!ret) - { - llwarns << "curl_easy_init failed." << llendl ; - } - - return ret ; -} - -//static -void LLCurl::deleteEasyHandle(CURL* handle) -{ - if(handle) - { - LLMutexLock lock(sHandleMutexp) ; - curl_easy_cleanup(handle) ; - sTotalHandles-- ; - } -} - -const unsigned int LLCurl::MAX_REDIRECTS = 5; - -// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. -void LLCurlFF::check_easy_code(CURLcode code) -{ - check_curl_code(code); -} -void LLCurlFF::check_multi_code(CURLMcode code) -{ - check_curl_multi_code(code); -} diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h index fa8465bfe..32bb6e45f 100644 --- a/indra/llmessage/llcurl.h +++ b/indra/llmessage/llcurl.h @@ -1,457 +1,39 @@ -/** +/** * @file llcurl.h - * @author Zero / Donovan - * @date 2006-10-15 - * @brief A wrapper around libcurl. + * @brief Drop in replacement for old llcurl.h. * - * $LicenseInfo:firstyear=2006&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 22/06/2012 + * Initial version, written by Aleric Inglewood @ SL */ - + #ifndef LL_LLCURL_H #define LL_LLCURL_H -#include "linden_common.h" +#include "aicurl.h" -#include -#include -#include - -#include -#include // TODO: remove dependency - -#include "llbuffer.h" -#include "lliopipe.h" -#include "llsd.h" -#include "llthread.h" -#include "llqueuedthread.h" -#include "llframetimer.h" - -class LLMutex; -class LLCurlThread; - -// For whatever reason, this is not typedef'd in curl.h -typedef size_t (*curl_header_callback)(void *ptr, size_t size, size_t nmemb, void *stream); - -class LLCurl -{ - LOG_CLASS(LLCurl); - -public: - class Easy; - class Multi; - - struct TransferInfo - { - TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) {} - F64 mSizeDownload; - F64 mTotalTime; - F64 mSpeedDownload; - }; - - class Responder - { - //LOG_CLASS(Responder); - public: - - Responder(); - virtual ~Responder(); - - /** - * @brief return true if the status code indicates success. - */ - static bool isGoodStatus(U32 status) - { - return((200 <= status) && (status < 300)); - } - - virtual void errorWithContent( - U32 status, - const std::string& reason, - const LLSD& content); - //< called by completed() on bad status - - virtual void error(U32 status, const std::string& reason); - //< called by default error(status, reason, content) - - virtual void result(const LLSD& content); - //< called by completed for good status codes. - - virtual void completedRaw( - U32 status, - const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer); - /**< Override point for clients that may want to use this - class when the response is some other format besides LLSD - */ - - virtual void completed( - U32 status, - const std::string& reason, - const LLSD& content); - /**< The default implemetnation calls - either: - * result(), or - * error() - */ - - // Override to handle parsing of the header only. Note: this is the only place where the contents - // of the header can be parsed. In the ::completed call above only the body is contained in the LLSD. - virtual void completedHeader(U32 status, const std::string& reason, const LLSD& content); - - // Used internally to set the url for debugging later. - void setURL(const std::string& url); - - virtual bool followRedir() - { - return false; - } - - public: /* but not really -- don't touch this */ - U32 mReferenceCount; - - private: - std::string mURL; - }; - typedef boost::intrusive_ptr ResponderPtr; - - - /** - * @ brief Set certificate authority file used to verify HTTPS certs. - */ - static void setCAFile(const std::string& file); - - /** - * @ brief Set certificate authority path used to verify HTTPS certs. - */ - static void setCAPath(const std::string& path); - - /** - * @ brief Return human-readable string describing libcurl version. - */ - static std::string getVersionString(); - - /** - * @ brief Get certificate authority file used to verify HTTPS certs. - */ - static const std::string& getCAFile() { return sCAFile; } - - /** - * @ brief Get certificate authority path used to verify HTTPS certs. - */ - static const std::string& getCAPath() { return sCAPath; } - - /** - * @ brief Initialize LLCurl class - */ - static void initClass(F32 curl_reuest_timeout = 120.f, S32 max_number_handles = 256, bool multi_threaded = false); - - /** - * @ brief Cleanup LLCurl class - */ - static void cleanupClass(); - - /** - * @ brief curl error code -> string - */ - static std::string strerror(CURLcode errorcode); - - // For OpenSSL callbacks - static std::vector sSSLMutex; - - // OpenSSL callbacks - static void ssl_locking_callback(int mode, int type, const char *file, int line); - static unsigned long ssl_thread_id(void); - - static LLCurlThread* getCurlThread() { return sCurlThread ;} - - static CURLM* newMultiHandle() ; - static CURLMcode deleteMultiHandle(CURLM* handle) ; - static CURL* newEasyHandle() ; - static void deleteEasyHandle(CURL* handle) ; - -private: - static std::string sCAPath; - static std::string sCAFile; - static const unsigned int MAX_REDIRECTS; - static LLCurlThread* sCurlThread; - - static LLMutex* sHandleMutexp ; - static S32 sTotalHandles ; - static S32 sMaxHandles; -public: - static bool sNotQuitting; - static F32 sCurlRequestTimeOut; -}; - -class LLCurl::Easy -{ - LOG_CLASS(Easy); - -private: - Easy(); - -public: - static Easy* getEasy(); - ~Easy(); - - CURL* getCurlHandle() const { return mCurlEasyHandle; } - - void setErrorBuffer(); - void setCA(); - - void setopt(CURLoption option, S32 value); - // These assume the setter does not free value! - void setopt(CURLoption option, void* value); - void setopt(CURLoption option, char* value); - // Copies the string so that it is guaranteed to stick around - void setoptString(CURLoption option, const std::string& value); - - void slist_append(const char* str); - void setHeaders(); - - U32 report(CURLcode); - void getTransferInfo(LLCurl::TransferInfo* info); - - void prepRequest(const std::string& url, const std::vector& headers, LLCurl::ResponderPtr, S32 time_out = 0, bool post = false); - - const char* getErrorBuffer(); - - std::stringstream& getInput() { return mInput; } - std::stringstream& getHeaderOutput() { return mHeaderOutput; } - LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; } - const LLChannelDescriptors& getChannels() { return mChannels; } - - void resetState(); - - static CURL* allocEasyHandle(); - static void releaseEasyHandle(CURL* handle); - -private: - friend class LLCurl; - friend class LLCurl::Multi; - - CURL* mCurlEasyHandle; - struct curl_slist* mHeaders; - - std::stringstream mRequest; - LLChannelDescriptors mChannels; - LLIOPipe::buffer_ptr_t mOutput; - std::stringstream mInput; - std::stringstream mHeaderOutput; - char mErrorBuffer[CURL_ERROR_SIZE]; - - // Note: char*'s not strings since we pass pointers to curl - std::vector mStrings; - - LLCurl::ResponderPtr mResponder; - - static std::set sFreeHandles; - static std::set sActiveHandles; - static LLMutex* sHandleMutexp ; -}; - -class LLCurl::Multi -{ - LOG_CLASS(Multi); - - friend class LLCurlThread ; - -private: - ~Multi(); - - void markDead() ; - bool doPerform(); - -public: - - typedef enum - { - STATE_READY=0, - STATE_PERFORMING=1, - STATE_COMPLETED=2 - } ePerformState; - - Multi(F32 idle_time_out = 0.f); - - LLCurl::Easy* allocEasy(); - bool addEasy(LLCurl::Easy* easy); - void removeEasy(LLCurl::Easy* easy); - - void lock() ; - void unlock() ; - - void setState(ePerformState state) ; - ePerformState getState() ; - - bool isCompleted() ; - bool isValid() {return mCurlMultiHandle != NULL && mValid;} - bool isDead() {return mDead;} - - bool waitToComplete() ; - - S32 process(); - - CURLMsg* info_read(S32* msgs_in_queue); - - S32 mQueued; - S32 mErrorCount; - -private: - void easyFree(LLCurl::Easy*); - void cleanup(bool deleted = false); - - CURLM* mCurlMultiHandle; - - typedef std::set easy_active_list_t; - easy_active_list_t mEasyActiveList; - typedef std::map easy_active_map_t; - easy_active_map_t mEasyActiveMap; - typedef std::set easy_free_list_t; - easy_free_list_t mEasyFreeList; - - LLQueuedThread::handle_t mHandle ; - ePerformState mState; - - BOOL mDead ; - BOOL mValid ; - LLMutex* mMutexp ; - LLMutex* mDeletionMutexp ; - LLMutex* mEasyMutexp ; - LLFrameTimer mIdleTimer ; - F32 mIdleTimeOut; -}; - -class LLCurlThread : public LLQueuedThread -{ -public: - - class CurlRequest : public LLQueuedThread::QueuedRequest - { - protected: - virtual ~CurlRequest(); // use deleteRequest() - - public: - CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread); - - /*virtual*/ bool processRequest(); - /*virtual*/ void finishRequest(bool completed); - - private: - // input - LLCurl::Multi* mMulti; - LLCurlThread* mCurlThread; - }; - friend class CurlRequest; - -public: - LLCurlThread(bool threaded = true) ; - virtual ~LLCurlThread() ; - - S32 update(F32 max_time_ms); - - void addMulti(LLCurl::Multi* multi) ; - void killMulti(LLCurl::Multi* multi) ; - -private: - bool doMultiPerform(LLCurl::Multi* multi) ; - void deleteMulti(LLCurl::Multi* multi) ; - void cleanupMulti(LLCurl::Multi* multi) ; -} ; - -//namespace boost -//{ - void intrusive_ptr_add_ref(LLCurl::Responder* p); - void intrusive_ptr_release(LLCurl::Responder* p); -//}; - - -class LLCurlRequest -{ -public: - typedef std::vector headers_t; - - LLCurlRequest(); - ~LLCurlRequest(); - - void get(const std::string& url, LLCurl::ResponderPtr responder); - bool getByteRange(const std::string& url, const headers_t& headers, S32 offset, S32 length, LLCurl::ResponderPtr responder); - bool post(const std::string& url, const headers_t& headers, const LLSD& data, LLCurl::ResponderPtr responder, S32 time_out = 0); - bool post(const std::string& url, const headers_t& headers, const std::string& data, LLCurl::ResponderPtr responder, S32 time_out = 0); - - S32 process(); - S32 getQueued(); - -private: - void addMulti(); - LLCurl::Easy* allocEasy(); - bool addEasy(LLCurl::Easy* easy); - -private: - typedef std::set curlmulti_set_t; - curlmulti_set_t mMultiSet; - LLCurl::Multi* mActiveMulti; - S32 mActiveRequestCount; - BOOL mProcessing; -}; - -class LLCurlEasyRequest -{ -public: - LLCurlEasyRequest(); - ~LLCurlEasyRequest(); - void setopt(CURLoption option, S32 value); - void setoptString(CURLoption option, const std::string& value); - void setPost(char* postdata, S32 size); - void setHeaderCallback(curl_header_callback callback, void* userdata); - void setWriteCallback(curl_write_callback callback, void* userdata); - void setReadCallback(curl_read_callback callback, void* userdata); - void setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata); - void slist_append(const char* str); - void sendRequest(const std::string& url); - void requestComplete(); - bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL); - std::string getErrorString(); - bool isCompleted() {return mMulti->isCompleted() ;} - bool wait() { return mMulti->waitToComplete(); } - bool isValid() {return mMulti && mMulti->isValid(); } - - LLCurl::Easy* getEasy() const { return mEasy; } - -private: - CURLMsg* info_read(S32* queue, LLCurl::TransferInfo* info); - -private: - LLCurl::Multi* mMulti; - LLCurl::Easy* mEasy; - bool mRequestSent; - bool mResultReturned; -}; - -// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace. -namespace LLCurlFF -{ - void check_easy_code(CURLcode code); - void check_multi_code(CURLMcode code); -} +// Map interface to old LLCurl names so this can be used as a drop-in replacement. +namespace LLCurl = AICurlInterface; #endif // LL_LLCURL_H diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index 037688906..355eebb77 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -220,17 +220,28 @@ static void request( const LLSD& headers = LLSD() ) { + if (responder) + { + // For possible debug output from within the responder. + responder->setURL(url); + } + if (!LLHTTPClient::hasPump()) { - responder->completed(U32_MAX, "No pump", LLSD()); + responder->fatalError("No pump"); return; } LLPumpIO::chain_t chain; - LLURLRequest* req = new LLURLRequest(method, url); - if(!req->isValid())//failed + LLURLRequest* req; + try { - delete req ; + req = new LLURLRequest(method, url); + } + catch(AICurlNoEasyHandle& error) + { + llwarns << "Failed to create LLURLRequest: " << error.what() << llendl; + // This is what the old LL code did: no recovery whatsoever (but also no leaks or crash). return ; } @@ -282,11 +293,6 @@ static void request( } } - if (responder) - { - responder->setURL(url); - } - req->setCallback(new LLHTTPClientURLAdaptor(responder)); if (method == LLURLRequest::HTTP_POST && gMessageSystem) @@ -333,7 +339,7 @@ void LLHTTPClient::getByteRange( std::string range = llformat("bytes=%d-%d", offset, offset+bytes-1); headers["Range"] = range; } - request(url,LLURLRequest::HTTP_GET, NULL, responder, timeout, headers); + request(url, LLURLRequest::HTTP_GET, NULL, responder, timeout, headers); } void LLHTTPClient::head( @@ -372,12 +378,12 @@ class LLHTTPBuffer public: LLHTTPBuffer() { } - static size_t curl_write( void *ptr, size_t size, size_t nmemb, void *user_data) + static size_t curl_write(char* ptr, size_t size, size_t nmemb, void* user_data) { LLHTTPBuffer* self = (LLHTTPBuffer*)user_data; size_t bytes = (size * nmemb); - self->mBuffer.append((char*)ptr,bytes); + self->mBuffer.append(ptr,bytes); return nmemb; } @@ -428,104 +434,91 @@ static LLSD blocking_request( ) { lldebugs << "blockingRequest of " << url << llendl; - char curl_error_buffer[CURL_ERROR_SIZE] = "\0"; - CURL* curlp = LLCurl::newEasyHandle(); - llassert_always(curlp != NULL) ; - LLHTTPBuffer http_buffer; - std::string body_str; - - // other request method checks root cert first, we skip? + S32 http_status = 499; + LLSD response = LLSD::emptyMap(); - // Apply configured proxy settings - LLProxy::getInstance()->applyProxySettings(curlp); - - // * Set curl handle options - curl_easy_setopt(curlp, CURLOPT_NOSIGNAL, 1); // don't use SIGALRM for timeouts - curl_easy_setopt(curlp, CURLOPT_TIMEOUT, timeout); // seconds, see warning at top of function. - curl_easy_setopt(curlp, CURLOPT_WRITEFUNCTION, LLHTTPBuffer::curl_write); - curl_easy_setopt(curlp, CURLOPT_WRITEDATA, &http_buffer); - curl_easy_setopt(curlp, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curlp, CURLOPT_ERRORBUFFER, curl_error_buffer); - - // * Setup headers (don't forget to free them after the call!) - curl_slist* headers_list = NULL; - if (headers.isMap()) + try { - LLSD::map_const_iterator iter = headers.beginMap(); - LLSD::map_const_iterator end = headers.endMap(); - for (; iter != end; ++iter) + AICurlEasyRequest easy_request(false); + AICurlEasyRequest_wat curlEasyRequest_w(*easy_request); + + LLHTTPBuffer http_buffer; + std::string body_str; + + // * Set curl handle options + curlEasyRequest_w->setopt(CURLOPT_TIMEOUT, timeout); // seconds, see warning at top of function. + curlEasyRequest_w->setWriteCallback(&LLHTTPBuffer::curl_write, &http_buffer); + + // * Setup headers. + if (headers.isMap()) { - std::ostringstream header; - header << iter->first << ": " << iter->second.asString() ; - lldebugs << "header = " << header.str() << llendl; - headers_list = curl_slist_append(headers_list, header.str().c_str()); + LLSD::map_const_iterator iter = headers.beginMap(); + LLSD::map_const_iterator end = headers.endMap(); + for (; iter != end; ++iter) + { + std::ostringstream header; + header << iter->first << ": " << iter->second.asString() ; + lldebugs << "header = " << header.str() << llendl; + curlEasyRequest_w->addHeader(header.str().c_str()); + } + } + + // * Setup specific method / "verb" for the URI (currently only GET and POST supported + poppy) + if (method == LLURLRequest::HTTP_GET) + { + curlEasyRequest_w->setopt(CURLOPT_HTTPGET, 1); + } + else if (method == LLURLRequest::HTTP_POST) + { + curlEasyRequest_w->setopt(CURLOPT_POST, 1); + //serialize to ostr then copy to str - need to because ostr ptr is unstable :( + std::ostringstream ostr; + LLSDSerialize::toXML(body, ostr); + body_str = ostr.str(); + curlEasyRequest_w->setopt(CURLOPT_POSTFIELDS, body_str.c_str()); + //copied from PHP libs, correct? + curlEasyRequest_w->addHeader("Content-Type: application/llsd+xml"); + + // copied from llurlrequest.cpp + // it appears that apache2.2.3 or django in etch is busted. If + // we do not clear the expect header, we get a 500. May be + // limited to django/mod_wsgi. + curlEasyRequest_w->addHeader("Expect:"); + } + + // * Do the action using curl, handle results + lldebugs << "HTTP body: " << body_str << llendl; + curlEasyRequest_w->addHeader("Accept: application/llsd+xml"); + curlEasyRequest_w->finalizeRequest(url); + + S32 curl_success = curlEasyRequest_w->perform(); + curlEasyRequest_w->getinfo(CURLINFO_RESPONSE_CODE, &http_status); + // if we get a non-404 and it's not a 200 OR maybe it is but you have error bits, + if ( http_status != 404 && (http_status != 200 || curl_success != 0) ) + { + // We expect 404s, don't spam for them. + llwarns << "CURL REQ URL: " << url << llendl; + llwarns << "CURL REQ METHOD TYPE: " << method << llendl; + llwarns << "CURL REQ HEADERS: " << headers.asString() << llendl; + llwarns << "CURL REQ BODY: " << body_str << llendl; + llwarns << "CURL HTTP_STATUS: " << http_status << llendl; + llwarns << "CURL ERROR: " << curlEasyRequest_w->getErrorString() << llendl; + llwarns << "CURL ERROR BODY: " << http_buffer.asString() << llendl; + response["body"] = http_buffer.asString(); + } + else + { + response["body"] = http_buffer.asLLSD(); + lldebugs << "CURL response: " << http_buffer.asString() << llendl; } } - - // * Setup specific method / "verb" for the URI (currently only GET and POST supported + poppy) - if (method == LLURLRequest::HTTP_GET) + catch(AICurlNoEasyHandle const& error) { - curl_easy_setopt(curlp, CURLOPT_HTTPGET, 1); - } - else if (method == LLURLRequest::HTTP_POST) - { - curl_easy_setopt(curlp, CURLOPT_POST, 1); - //serialize to ostr then copy to str - need to because ostr ptr is unstable :( - std::ostringstream ostr; - LLSDSerialize::toXML(body, ostr); - body_str = ostr.str(); - curl_easy_setopt(curlp, CURLOPT_POSTFIELDS, body_str.c_str()); - //copied from PHP libs, correct? - headers_list = curl_slist_append(headers_list, "Content-Type: application/llsd+xml"); - - // copied from llurlrequest.cpp - // it appears that apache2.2.3 or django in etch is busted. If - // we do not clear the expect header, we get a 500. May be - // limited to django/mod_wsgi. - headers_list = curl_slist_append(headers_list, "Expect:"); - } - - // * Do the action using curl, handle results - lldebugs << "HTTP body: " << body_str << llendl; - headers_list = curl_slist_append(headers_list, "Accept: application/llsd+xml"); - CURLcode curl_result = curl_easy_setopt(curlp, CURLOPT_HTTPHEADER, headers_list); - if ( curl_result != CURLE_OK ) - { - llinfos << "Curl is hosed - can't add headers" << llendl; + response["body"] = error.what(); } - LLSD response = LLSD::emptyMap(); - S32 curl_success = curl_easy_perform(curlp); - S32 http_status = 499; - curl_easy_getinfo(curlp, CURLINFO_RESPONSE_CODE, &http_status); response["status"] = http_status; - // if we get a non-404 and it's not a 200 OR maybe it is but you have error bits, - if ( http_status != 404 && (http_status != 200 || curl_success != 0) ) - { - // We expect 404s, don't spam for them. - llwarns << "CURL REQ URL: " << url << llendl; - llwarns << "CURL REQ METHOD TYPE: " << method << llendl; - llwarns << "CURL REQ HEADERS: " << headers.asString() << llendl; - llwarns << "CURL REQ BODY: " << body_str << llendl; - llwarns << "CURL HTTP_STATUS: " << http_status << llendl; - llwarns << "CURL ERROR: " << curl_error_buffer << llendl; - llwarns << "CURL ERROR BODY: " << http_buffer.asString() << llendl; - response["body"] = http_buffer.asString(); - } - else - { - response["body"] = http_buffer.asLLSD(); - lldebugs << "CURL response: " << http_buffer.asString() << llendl; - } - - if(headers_list) - { // free the header list - curl_slist_free_all(headers_list); - } - - // * Cleanup - LLCurl::deleteEasyHandle(curlp); return response; } diff --git a/indra/llmessage/llhttpclient.h b/indra/llmessage/llhttpclient.h index dda7d550f..39a6498d8 100644 --- a/indra/llmessage/llhttpclient.h +++ b/indra/llmessage/llhttpclient.h @@ -55,7 +55,9 @@ public: typedef LLCurl::Responder Responder; typedef LLCurl::ResponderPtr ResponderPtr; - + // The default actually already ignores responses. + class ResponderIgnore : public Responder { }; + /** @name non-blocking API */ //@{ static void head( diff --git a/indra/llmessage/llproxy.cpp b/indra/llmessage/llproxy.cpp index 4a7d326c0..3db851001 100644 --- a/indra/llmessage/llproxy.cpp +++ b/indra/llmessage/llproxy.cpp @@ -406,16 +406,6 @@ void LLProxy::cleanupClass() deleteSingleton(); } -void LLProxy::applyProxySettings(LLCurlEasyRequest* handle) -{ - applyProxySettings(handle->getEasy()); -} - -void LLProxy::applyProxySettings(LLCurl::Easy* handle) -{ - applyProxySettings(handle->getCurlHandle()); -} - /** * @brief Apply proxy settings to a CuRL request if an HTTP proxy is enabled. * @@ -425,9 +415,9 @@ void LLProxy::applyProxySettings(LLCurl::Easy* handle) * When the HTTP proxy is enabled, the proxy mutex will * be locked every time this method is called. * - * @param handle A pointer to a valid CURL request, before it has been performed. + * @param curlEasyRequest_w An already locked curl easy handle, before it has been performed. */ -void LLProxy::applyProxySettings(CURL* handle) +void LLProxy::applyProxySettings(AICurlEasyRequest_wat const& curlEasyRequest_w) { // Do a faster unlocked check to see if we are supposed to proxy. if (mHTTPProxyEnabled) @@ -437,21 +427,21 @@ void LLProxy::applyProxySettings(CURL* handle) // Now test again to verify that the proxy wasn't disabled between the first check and the lock. if (mHTTPProxyEnabled) { - LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXY, mHTTPProxy.getIPString().c_str())); - LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYPORT, mHTTPProxy.getPort())); + curlEasyRequest_w->setopt(CURLOPT_PROXY, mHTTPProxy.getIPString().c_str()); + curlEasyRequest_w->setopt(CURLOPT_PROXYPORT, mHTTPProxy.getPort()); if (mProxyType == LLPROXY_SOCKS) { - LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5)); + curlEasyRequest_w->setopt(CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); if (mAuthMethodSelected == METHOD_PASSWORD) { std::string auth_string = mSocksUsername + ":" + mSocksPassword; - LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, auth_string.c_str())); + curlEasyRequest_w->setopt(CURLOPT_PROXYUSERPWD, auth_string.c_str()); } } else { - LLCurlFF::check_easy_code(curl_easy_setopt(handle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP)); + curlEasyRequest_w->setopt(CURLOPT_PROXYTYPE, CURLPROXY_HTTP); } } } diff --git a/indra/llmessage/llproxy.h b/indra/llmessage/llproxy.h index a91937054..94f031fc3 100644 --- a/indra/llmessage/llproxy.h +++ b/indra/llmessage/llproxy.h @@ -206,15 +206,7 @@ enum LLSocks5AuthType * The implementation of HTTP proxying is handled by libcurl. LLProxy * is responsible for managing the HTTP proxy options and provides a * thread-safe method to apply those options to a curl request - * (LLProxy::applyProxySettings()). This method is overloaded - * to accommodate the various abstraction libcurl layers that exist - * throughout the viewer (LLCurlEasyRequest, LLCurl::Easy, and CURL). - * - * If you are working with LLCurl or LLCurlEasyRequest objects, - * the configured proxy settings will be applied in the constructors - * of those request handles. If you are working with CURL objects - * directly, you will need to pass the handle of the request to - * applyProxySettings() before issuing the request. + * (LLProxy::applyProxySettings()). * * To ensure thread safety, all LLProxy members that relate to the HTTP * proxy require the LLProxyMutex to be locked before accessing. @@ -251,9 +243,7 @@ public: // Apply the current proxy settings to a curl request. Doesn't do anything if mHTTPProxyEnabled is false. // Safe to call from any thread. - void applyProxySettings(CURL* handle); - void applyProxySettings(LLCurl::Easy* handle); - void applyProxySettings(LLCurlEasyRequest* handle); + void applyProxySettings(AICurlEasyRequest_wat const& curlEasyRequest_w); // Start a connection to the SOCKS 5 proxy. Call from main thread only. S32 startSOCKSProxy(LLHost host); diff --git a/indra/llmessage/llsdmessage.cpp b/indra/llmessage/llsdmessage.cpp index 9148c9dd1..95ed03e66 100644 --- a/indra/llmessage/llsdmessage.cpp +++ b/indra/llmessage/llsdmessage.cpp @@ -151,11 +151,11 @@ bool LLSDMessage::ResponderAdapter::listener(const LLSD& payload, bool success) { if (success) { - mResponder->result(payload); + mResponder->pubResult(payload); } else { - mResponder->errorWithContent(payload["status"].asInteger(), payload["reason"], payload["content"]); + mResponder->pubErrorWithContent(payload["status"].asInteger(), payload["reason"], payload["content"]); } /*---------------- MUST BE LAST STATEMENT BEFORE RETURN ----------------*/ diff --git a/indra/llmessage/llsdrpcclient.h b/indra/llmessage/llsdrpcclient.h index 0cecf4f68..80fa82dcc 100644 --- a/indra/llmessage/llsdrpcclient.h +++ b/indra/llmessage/llsdrpcclient.h @@ -240,11 +240,14 @@ public: virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const { lldebugs << "LLSDRPCClientFactory::build" << llendl; - LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); - if(!http->isValid()) + LLURLRequest* http; + try { - llwarns << "Creating LLURLRequest failed." << llendl ; - delete http; + http = new LLURLRequest(LLURLRequest::HTTP_POST); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "Creating LLURLRequest failed: " << error.what() << llendl ; return false; } @@ -291,11 +294,14 @@ public: { lldebugs << "LLXMLSDRPCClientFactory::build" << llendl; - LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); - if(!http->isValid()) + LLURLRequest* http; + try { - llwarns << "Creating LLURLRequest failed." << llendl ; - delete http; + http = new LLURLRequest(LLURLRequest::HTTP_POST); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "Creating LLURLRequest failed: " << error.what() << llendl ; return false ; } LLIOPipe::ptr_t service(new Client); diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index f5e08f393..97a8e48d9 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -29,6 +29,10 @@ #include "linden_common.h" #include "llurlrequest.h" +#ifdef CWDEBUG +#include +#endif + #include #include #include @@ -52,9 +56,7 @@ const std::string CONTEXT_DEST_URI_SD_LABEL("dest_uri"); const std::string CONTEXT_TRANSFERED_BYTES("transfered_bytes"); -static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user); - - +static size_t headerCallback(char* data, size_t size, size_t nmemb, void* user); /** * class LLURLRequestDetail @@ -65,7 +67,7 @@ public: LLURLRequestDetail(); ~LLURLRequestDetail(); std::string mURL; - LLCurlEasyRequest* mCurlRequest; + AICurlEasyRequest mCurlEasyRequest; LLIOPipe::buffer_ptr_t mResponseBuffer; LLChannelDescriptors mChannels; U8* mLastRead; @@ -76,36 +78,28 @@ public: }; LLURLRequestDetail::LLURLRequestDetail() : - mCurlRequest(NULL), + mCurlEasyRequest(false), mLastRead(NULL), mBodyLimit(0), mByteAccumulator(0), mIsBodyLimitSet(false), mSSLVerifyCallback(NULL) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - mCurlRequest = new LLCurlEasyRequest(); - - if(!mCurlRequest->isValid()) //failed. - { - delete mCurlRequest ; - mCurlRequest = NULL ; - } } LLURLRequestDetail::~LLURLRequestDetail() { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - delete mCurlRequest; mLastRead = NULL; } void LLURLRequest::setSSLVerifyCallback(SSLCertVerifyCallback callback, void *param) { + LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mDetail->mSSLVerifyCallback = callback; - mDetail->mCurlRequest->setSSLCtxCallback(LLURLRequest::_sslCtxCallback, (void *)this); - mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYPEER, true); - mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYHOST, 2); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setSSLCtxCallback(LLURLRequest::_sslCtxCallback, (void *)this); + curlEasyRequest_w->setopt(CURLOPT_SSL_VERIFYPEER, true); + curlEasyRequest_w->setopt(CURLOPT_SSL_VERIFYHOST, 2); } @@ -159,6 +153,7 @@ LLURLRequest::LLURLRequest(LLURLRequest::ERequestAction action) : mAction(action) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); + // This might throw AICurlNoEasyHandle. initialize(); } @@ -168,6 +163,7 @@ LLURLRequest::LLURLRequest( mAction(action) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); + // This might throw AICurlNoEasyHandle. initialize(); setURL(url); } @@ -175,13 +171,16 @@ LLURLRequest::LLURLRequest( LLURLRequest::~LLURLRequest() { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); + { + AICurlEasyRequest_wat curl_easy_request_w(*mDetail->mCurlEasyRequest); + curl_easy_request_w->revokeCallbacks(); + curl_easy_request_w->send_events_to(NULL); + } delete mDetail; - mDetail = NULL ; } void LLURLRequest::setURL(const std::string& url) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mDetail->mURL = url; } @@ -193,7 +192,8 @@ std::string LLURLRequest::getURL() const void LLURLRequest::addHeader(const char* header) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - mDetail->mCurlRequest->slist_append(header); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->addHeader(header); } void LLURLRequest::setBodyLimit(U32 size) @@ -206,7 +206,8 @@ void LLURLRequest::setCallback(LLURLRequestComplete* callback) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mCompletionCallback = callback; - mDetail->mCurlRequest->setHeaderCallback(&headerCallback, (void*)callback); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setHeaderCallback(&headerCallback, (void*)callback); } // Added to mitigate the effect of libcurl looking @@ -242,30 +243,28 @@ void LLURLRequest::useProxy(bool use_proxy) lldebugs << "use_proxy = " << (use_proxy?'Y':'N') << ", env_proxy = \"" << env_proxy << "\"" << llendl; - if (use_proxy) - { - mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, env_proxy); - } - else - { - mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, ""); - } + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setoptString(CURLOPT_PROXY, use_proxy ? env_proxy : std::string("")); } void LLURLRequest::useProxy(const std::string &proxy) { - mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, proxy); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setoptString(CURLOPT_PROXY, proxy); } void LLURLRequest::allowCookies() { - mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, ""); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setoptString(CURLOPT_COOKIEFILE, ""); } //virtual bool LLURLRequest::isValid() { - return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid(); + //FIXME - wtf is with this isValid? + //return mDetail->mCurlRequest->isValid(); + return true; } // virtual @@ -294,6 +293,19 @@ LLIOPipe::EStatus LLURLRequest::handleError( return status; } +void LLURLRequest::added_to_multi_handle(AICurlEasyRequest_wat&) +{ +} + +void LLURLRequest::finished(AICurlEasyRequest_wat&) +{ +} + +void LLURLRequest::removed_from_multi_handle(AICurlEasyRequest_wat&) +{ + mRemoved = true; +} + static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST("URL Request"); // virtual @@ -310,7 +322,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( //llinfos << "LLURLRequest::process_impl()" << llendl; if (!buffer) return STATUS_ERROR; - // we're still waiting or prcessing, check how many + // we're still waiting or processing, check how many // bytes we have accumulated. const S32 MIN_ACCUMULATION = 100000; if(pump && (mDetail->mByteAccumulator > MIN_ACCUMULATION)) @@ -354,44 +366,36 @@ LLIOPipe::EStatus LLURLRequest::process_impl( { return STATUS_ERROR; } + mRemoved = false; mState = STATE_WAITING_FOR_RESPONSE; + mDetail->mCurlEasyRequest.addRequest(); // Add easy handle to multi handle. - // *FIX: Maybe we should just go to the next state now... return STATUS_BREAK; } case STATE_WAITING_FOR_RESPONSE: case STATE_PROCESSING_RESPONSE: { - PUMP_DEBUG; - LLIOPipe::EStatus status = STATUS_BREAK; - static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform"); + if (!mRemoved) // Not removed from multi handle yet? { - LLFastTimer t(FTM_URL_PERFORM); - if(!mDetail->mCurlRequest->wait()) - { - return status ; - } + // Easy handle is still being processed. + return STATUS_BREAK; } + // Curl thread finished with this easy handle. + mState = STATE_CURL_FINISHED; + } + case STATE_CURL_FINISHED: + { + PUMP_DEBUG; + LLIOPipe::EStatus status = STATUS_NO_CONNECTION; // Catch-all failure code. - while(1) + // Left braces in order not to change indentation. { CURLcode result; static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST_GET_RESULT("Get Result"); - bool newmsg = false; - { - LLFastTimer t(FTM_PROCESS_URL_REQUEST_GET_RESULT); - newmsg = mDetail->mCurlRequest->getResult(&result); - } + AICurlEasyRequest_wat(*mDetail->mCurlEasyRequest)->getResult(&result); - if(!newmsg) - { - // keep processing - break; - } - - mState = STATE_HAVE_RESPONSE; context[CONTEXT_REQUEST][CONTEXT_TRANSFERED_BYTES] = mRequestTransferedBytes; context[CONTEXT_RESPONSE][CONTEXT_TRANSFERED_BYTES] = mResponseTransferedBytes; @@ -423,6 +427,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( } mCompletionCallback = NULL; } + status = STATUS_BREAK; // This is what the old code returned. Does it make sense? break; case CURLE_FAILED_INIT: case CURLE_COULDNT_CONNECT: @@ -464,16 +469,15 @@ void LLURLRequest::initialize() { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mState = STATE_INITIALIZED; + // This might throw AICurlNoEasyHandle. mDetail = new LLURLRequestDetail; - if(!isValid()) { - return ; + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + curlEasyRequest_w->setWriteCallback(&downCallback, (void*)this); + curlEasyRequest_w->setReadCallback(&upCallback, (void*)this); } - mDetail->mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); - mDetail->mCurlRequest->setWriteCallback(&downCallback, (void*)this); - mDetail->mCurlRequest->setReadCallback(&upCallback, (void*)this); mRequestTransferedBytes = 0; mResponseTransferedBytes = 0; } @@ -488,70 +492,74 @@ bool LLURLRequest::configure() S32 bytes = mDetail->mResponseBuffer->countAfter( mDetail->mChannels.in(), NULL); - switch(mAction) { - case HTTP_HEAD: - mDetail->mCurlRequest->setopt(CURLOPT_HEADER, 1); - mDetail->mCurlRequest->setopt(CURLOPT_NOBODY, 1); - mDetail->mCurlRequest->setopt(CURLOPT_FOLLOWLOCATION, 1); - rv = true; - break; - case HTTP_GET: - mDetail->mCurlRequest->setopt(CURLOPT_HTTPGET, 1); - mDetail->mCurlRequest->setopt(CURLOPT_FOLLOWLOCATION, 1); + AICurlEasyRequest_wat curlEasyRequest_w(*mDetail->mCurlEasyRequest); + switch(mAction) + { + case HTTP_HEAD: + curlEasyRequest_w->setopt(CURLOPT_HEADER, 1); + curlEasyRequest_w->setopt(CURLOPT_NOBODY, 1); + curlEasyRequest_w->setopt(CURLOPT_FOLLOWLOCATION, 1); + rv = true; + break; + case HTTP_GET: + curlEasyRequest_w->setopt(CURLOPT_HTTPGET, 1); + curlEasyRequest_w->setopt(CURLOPT_FOLLOWLOCATION, 1); - // Set Accept-Encoding to allow response compression - mDetail->mCurlRequest->setoptString(CURLOPT_ENCODING, ""); - rv = true; - break; + // Set Accept-Encoding to allow response compression + curlEasyRequest_w->setoptString(CURLOPT_ENCODING, ""); + rv = true; + break; - case HTTP_PUT: - // Disable the expect http 1.1 extension. POST and PUT default - // to turning this on, and I am not too sure what it means. - addHeader("Expect:"); + case HTTP_PUT: + // Disable the expect http 1.1 extension. POST and PUT default + // to turning this on, and I am not too sure what it means. + addHeader("Expect:"); - mDetail->mCurlRequest->setopt(CURLOPT_UPLOAD, 1); - mDetail->mCurlRequest->setopt(CURLOPT_INFILESIZE, bytes); - rv = true; - break; + curlEasyRequest_w->setopt(CURLOPT_UPLOAD, 1); + curlEasyRequest_w->setopt(CURLOPT_INFILESIZE, bytes); + rv = true; + break; - case HTTP_POST: - // Disable the expect http 1.1 extension. POST and PUT default - // to turning this on, and I am not too sure what it means. - addHeader("Expect:"); + case HTTP_POST: + // Disable the expect http 1.1 extension. POST and PUT default + // to turning this on, and I am not too sure what it means. + addHeader("Expect:"); - // Disable the content type http header. - // *FIX: what should it be? - addHeader("Content-Type:"); + // Disable the content type http header. + // *FIX: what should it be? + addHeader("Content-Type:"); - // Set the handle for an http post - mDetail->mCurlRequest->setPost(NULL, bytes); + // Set the handle for an http post + curlEasyRequest_w->setPost(NULL, bytes); - // Set Accept-Encoding to allow response compression - mDetail->mCurlRequest->setoptString(CURLOPT_ENCODING, ""); - rv = true; - break; + // Set Accept-Encoding to allow response compression + curlEasyRequest_w->setoptString(CURLOPT_ENCODING, ""); + rv = true; + break; - case HTTP_DELETE: - // Set the handle for an http post - mDetail->mCurlRequest->setoptString(CURLOPT_CUSTOMREQUEST, "DELETE"); - rv = true; - break; + case HTTP_DELETE: + // Set the handle for an http post + curlEasyRequest_w->setoptString(CURLOPT_CUSTOMREQUEST, "DELETE"); + rv = true; + break; - case HTTP_MOVE: - // Set the handle for an http post - mDetail->mCurlRequest->setoptString(CURLOPT_CUSTOMREQUEST, "MOVE"); - // *NOTE: should we check for the Destination header? - rv = true; - break; + case HTTP_MOVE: + // Set the handle for an http post + curlEasyRequest_w->setoptString(CURLOPT_CUSTOMREQUEST, "MOVE"); + // *NOTE: should we check for the Destination header? + rv = true; + break; - default: - llwarns << "Unhandled URLRequest action: " << mAction << llendl; - break; - } - if(rv) - { - mDetail->mCurlRequest->sendRequest(mDetail->mURL); + default: + llwarns << "Unhandled URLRequest action: " << mAction << llendl; + break; + } + if(rv) + { + curlEasyRequest_w->finalizeRequest(mDetail->mURL); + curlEasyRequest_w->send_events_to(this); + } } return rv; } @@ -615,9 +623,8 @@ size_t LLURLRequest::upCallback( return bytes; } -static size_t headerCallback(void* data, size_t size, size_t nmemb, void* user) +static size_t headerCallback(char* header_line, size_t size, size_t nmemb, void* user) { - const char* header_line = (const char*)data; size_t header_len = size * nmemb; LLURLRequestComplete* complete = (LLURLRequestComplete*)user; diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h index 1ba51a5ac..6cfe83a5e 100644 --- a/indra/llmessage/llurlrequest.h +++ b/indra/llmessage/llurlrequest.h @@ -66,7 +66,7 @@ typedef struct x509_store_ctx_st X509_STORE_CTX; * worth the time and effort to eventually port this to a raw client * socket. */ -class LLURLRequest : public LLIOPipe +class LLURLRequest : public LLIOPipe, protected AICurlEasyHandleEvents { LOG_CLASS(LLURLRequest); public: @@ -217,6 +217,7 @@ protected: STATE_INITIALIZED, STATE_WAITING_FOR_RESPONSE, STATE_PROCESSING_RESPONSE, + STATE_CURL_FINISHED, STATE_HAVE_RESPONSE, }; EState mState; @@ -228,6 +229,14 @@ protected: static CURLcode _sslCtxCallback(CURL * curl, void *sslctx, void *param); + // mRemoved is used instead of changing mState directly, because I'm not convinced the latter is atomic. + // Set to false before adding curl request and then only tested. + // Reset in removed_from_multi_handle (by another thread), this is thread-safe. + bool mRemoved; + /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat&); + /*virtual*/ void finished(AICurlEasyRequest_wat&); + /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat&); + private: /** * @brief Initialize the object. Called during construction. diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index 83125c7fd..bbb8b5eb4 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -151,6 +151,7 @@ set(viewer_SOURCE_FILES llconfirmationmanager.cpp llconsole.cpp llcontainerview.cpp + llcurlrequest.cpp llcurrencyuimanager.cpp llcylinder.cpp lldebugmessagebox.cpp @@ -646,6 +647,7 @@ set(viewer_HEADER_FILES llconfirmationmanager.h llconsole.h llcontainerview.h + llcurlrequest.h llcurrencyuimanager.h llcylinder.h lldebugmessagebox.h diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 3af73986a..cdabf2221 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -4016,17 +4016,6 @@ Value 120.0 - CurlUseMultipleThreads - - Comment - Use background threads for executing curl_multi_perform (requires restart) - Persist - 1 - Type - Boolean - Value - 1 - Cursor3D Comment diff --git a/indra/newview/hipporestrequest.cpp b/indra/newview/hipporestrequest.cpp index 7b64b35d8..17875f27e 100644 --- a/indra/newview/hipporestrequest.cpp +++ b/indra/newview/hipporestrequest.cpp @@ -255,7 +255,16 @@ static void request(const std::string &url, } LLPumpIO::chain_t chain; - LLURLRequest *req = new LLURLRequest(method, url); + LLURLRequest *req; + try + { + req = new LLURLRequest(method, url); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "Failed to create LLURLRequest: " << error.what() << llendl; + return; + } req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req); /* @@ -324,10 +333,11 @@ int HippoRestRequest::getBlocking(const std::string &url, std::string *result) char curlErrorBuffer[CURL_ERROR_SIZE]; CURL* curlp = curl_easy_init(); + llassert_always(curlp); curl_easy_setopt(curlp, CURLOPT_NOSIGNAL, 1); // don't use SIGALRM for timeouts curl_easy_setopt(curlp, CURLOPT_TIMEOUT, 5); // seconds - curl_easy_setopt(curlp, CURLOPT_CAINFO, gDirUtilp->getCAFile().c_str()); + curl_easy_setopt(curlp, CURLOPT_CAINFO, gDirUtilp->getCAFile().c_str()); curl_easy_setopt(curlp, CURLOPT_WRITEFUNCTION, curlWrite); curl_easy_setopt(curlp, CURLOPT_WRITEDATA, result); @@ -337,7 +347,7 @@ int HippoRestRequest::getBlocking(const std::string &url, std::string *result) *result = ""; S32 curlSuccess = curl_easy_perform(curlp); - S32 httpStatus = 499; + long httpStatus = 499L; // curl_easy_getinfo demands pointer to long. curl_easy_getinfo(curlp, CURLINFO_RESPONSE_CODE, &httpStatus); if (curlSuccess != 0) { diff --git a/indra/newview/llagent.cpp b/indra/newview/llagent.cpp index d0ba0a140..d00b1f979 100644 --- a/indra/newview/llagent.cpp +++ b/indra/newview/llagent.cpp @@ -2370,7 +2370,7 @@ bool LLAgent::sendMaturityPreferenceToServer(int preferredMaturity) body["access_prefs"] = access_prefs; llinfos << "Sending access prefs update to " << (access_prefs["max"].asString()) << " via capability to: " << url << llendl; - LLHTTPClient::post(url, body, new LLHTTPClient::Responder()); // Ignore response + LLHTTPClient::post(url, body, new LLHTTPClient::ResponderIgnore); // Ignore response return true; } return false; diff --git a/indra/newview/llagentlanguage.cpp b/indra/newview/llagentlanguage.cpp index 04e1228c7..4dbc7eb70 100644 --- a/indra/newview/llagentlanguage.cpp +++ b/indra/newview/llagentlanguage.cpp @@ -61,7 +61,7 @@ bool LLAgentLanguage::update() body["language"] = language; body["language_is_public"] = gSavedSettings.getBOOL("LanguageIsPublic"); - LLHTTPClient::post(url, body, new LLHTTPClient::Responder); + LLHTTPClient::post(url, body, new LLHTTPClient::ResponderIgnore); } return true; } diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index ad9b7d8ed..41aba1942 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -611,6 +611,9 @@ bool LLAppViewer::init() initLogging(); + // Curl must be initialized before any thread is running. + AICurlInterface::initCurl(); + // Logging is initialized. Now it's safe to start the error thread. startErrorThread(); @@ -635,12 +638,6 @@ bool LLAppViewer::init() LLPrivateMemoryPoolManager::initClass((BOOL)gSavedSettings.getBOOL("MemoryPrivatePoolEnabled"), (U32)gSavedSettings.getU32("MemoryPrivatePoolSize")) ; mAlloc.setProfilingEnabled(gSavedSettings.getBOOL("MemProfiling")); - // *NOTE:Mani - LLCurl::initClass is not thread safe. - // Called before threads are created. - LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"), - gSavedSettings.getS32("CurlMaximumNumberOfHandles"), - gSavedSettings.getBOOL("CurlUseMultipleThreads")); - LL_INFOS("InitInfo") << "LLCurl initialized." << LL_ENDL ; initThreads(); LL_INFOS("InitInfo") << "Threads initialized." << LL_ENDL ; @@ -1767,10 +1764,9 @@ bool LLAppViewer::cleanup() end_messaging_system(); llinfos << "Message system deleted." << llendflush; - LLUserAuth::getInstance()->reset(); //reset before LLCurl::cleanupClass, else LLCURL::sHandleMutex == NULL - // *NOTE:Mani - The following call is not thread safe. - LLCurl::cleanupClass(); - llinfos << "LLCurl cleaned up." << llendflush; + LLUserAuth::getInstance()->reset(); // Reset before AICurlInterface::cleanupCurl, else LLCURL::sHandleMutex == NULL + LLApp::stopErrorThread(); // The following call is not thread-safe. Have to stop all threads. + AICurlInterface::cleanupCurl(); // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. @@ -1839,6 +1835,8 @@ bool LLAppViewer::initThreads() LLWatchdog::getInstance()->init(watchdog_killer_callback); } + AICurlInterface::startCurlThread(); + LLImage::initClass(); LLVFSThread::initClass(enable_threads && false); diff --git a/indra/newview/llcurlrequest.cpp b/indra/newview/llcurlrequest.cpp new file mode 100644 index 000000000..9192de127 --- /dev/null +++ b/indra/newview/llcurlrequest.cpp @@ -0,0 +1,147 @@ +/** + * @file llcurlrequest.cpp + * @brief Implementation of Request. + * + * Copyright (c) 2012, Aleric Inglewood. + * Copyright (C) 2010, Linden Research, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 17/03/2012 + * Initial version, written by Aleric Inglewood @ SL + * + * 20/03/2012 + * Added copyright notice for Linden Lab for those parts that were + * copied or derived from llcurl.cpp. The code of those parts are + * already in their own llcurl.cpp, so they do not ever need to + * even look at this file; the reason I added the copyright notice + * is to make clear that I am not the author of 100% of this code + * and hence I cannot change the license of it. + */ + +#include "linden_common.h" + +#include "llsdserialize.h" +#include "llcurlrequest.h" +#include "statemachine/aicurleasyrequeststatemachine.h" + +//----------------------------------------------------------------------------- +// class Request +// + +namespace AICurlInterface { + +bool Request::get(std::string const& url, ResponderPtr responder) +{ + return getByteRange(url, headers_t(), 0, -1, responder); +} + +bool Request::getByteRange(std::string const& url, headers_t const& headers, S32 offset, S32 length, ResponderPtr responder) +{ + DoutEntering(dc::curl, "Request::getByteRange(" << url << ", ...)"); + + // This might throw AICurlNoEasyHandle. + AICurlEasyRequestStateMachine* buffered_easy_request = new AICurlEasyRequestStateMachine(true); + + { + AICurlEasyRequest_wat buffered_easy_request_w(*buffered_easy_request->mCurlEasyRequest); + + AICurlResponderBuffer_wat(*buffered_easy_request->mCurlEasyRequest)->prepRequest(buffered_easy_request_w, headers, responder); + + buffered_easy_request_w->setopt(CURLOPT_HTTPGET, 1); + if (length > 0) + { + std::string range = llformat("Range: bytes=%d-%d", offset, offset + length - 1); + buffered_easy_request_w->addHeader(range.c_str()); + } + + buffered_easy_request_w->finalizeRequest(url); + } + + buffered_easy_request->run(); + + return true; // We throw in case of problems. +} + +bool Request::post(std::string const& url, headers_t const& headers, std::string const& data, ResponderPtr responder, S32 time_out) +{ + DoutEntering(dc::curl, "Request::post(" << url << ", ...)"); + + // This might throw AICurlNoEasyHandle. + AICurlEasyRequestStateMachine* buffered_easy_request = new AICurlEasyRequestStateMachine(true); + + { + AICurlEasyRequest_wat buffered_easy_request_w(*buffered_easy_request->mCurlEasyRequest); + AICurlResponderBuffer_wat buffer_w(*buffered_easy_request->mCurlEasyRequest); + + buffer_w->prepRequest(buffered_easy_request_w, headers, responder); + + buffer_w->getInput().write(data.data(), data.size()); + S32 bytes = buffer_w->getInput().str().length(); + buffered_easy_request_w->setPost(NULL, bytes); + buffered_easy_request_w->addHeader("Content-Type: application/octet-stream"); + buffered_easy_request_w->finalizeRequest(url); + + lldebugs << "POSTING: " << bytes << " bytes." << llendl; + } + + buffered_easy_request->run(); + + return true; // We throw in case of problems. +} + +bool Request::post(std::string const& url, headers_t const& headers, LLSD const& data, ResponderPtr responder, S32 time_out) +{ + DoutEntering(dc::curl, "Request::post(" << url << ", ...)"); + + // This might throw AICurlNoEasyHandle. + AICurlEasyRequestStateMachine* buffered_easy_request = new AICurlEasyRequestStateMachine(true); + + { + AICurlEasyRequest_wat buffered_easy_request_w(*buffered_easy_request->mCurlEasyRequest); + AICurlResponderBuffer_wat buffer_w(*buffered_easy_request->mCurlEasyRequest); + + buffer_w->prepRequest(buffered_easy_request_w, headers, responder); + + LLSDSerialize::toXML(data, buffer_w->getInput()); + S32 bytes = buffer_w->getInput().str().length(); + buffered_easy_request_w->setPost(NULL, bytes); + buffered_easy_request_w->addHeader("Content-Type: application/llsd+xml"); + buffered_easy_request_w->finalizeRequest(url); + + lldebugs << "POSTING: " << bytes << " bytes." << llendl; + } + + buffered_easy_request->run(); + + return true; // We throw in case of problems. +} + +S32 Request::process(void) +{ + //FIXME: needs implementation + //DoutEntering(dc::warning, "Request::process()"); + return 0; +} + +} // namespace AICurlInterface +//================================================================================== + diff --git a/indra/newview/llcurlrequest.h b/indra/newview/llcurlrequest.h new file mode 100644 index 000000000..59f3da981 --- /dev/null +++ b/indra/newview/llcurlrequest.h @@ -0,0 +1,59 @@ +/** + * @file llcurlrequest.h + * @brief Declaration of class Request + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 17/03/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AICURLREQUEST_H +#define AICURLREQUEST_H + +#include +#include +#include + +// Things defined in this namespace are called from elsewhere in the viewer code. +namespace AICurlInterface { + +// Forward declaration. +class Responder; +typedef boost::intrusive_ptr ResponderPtr; + +class Request { + public: + typedef std::vector headers_t; + + bool get(std::string const& url, ResponderPtr responder); + bool getByteRange(std::string const& url, headers_t const& headers, S32 offset, S32 length, ResponderPtr responder); + bool post(std::string const& url, headers_t const& headers, std::string const& data, ResponderPtr responder, S32 time_out = 0); + bool post(std::string const& url, headers_t const& headers, LLSD const& data, ResponderPtr responder, S32 time_out = 0); + + S32 process(void); +}; + +} // namespace AICurlInterface + +#endif diff --git a/indra/newview/llfloaterregioninfo.cpp b/indra/newview/llfloaterregioninfo.cpp index 794e774fe..162414041 100644 --- a/indra/newview/llfloaterregioninfo.cpp +++ b/indra/newview/llfloaterregioninfo.cpp @@ -760,7 +760,7 @@ BOOL LLPanelRegionGeneralInfo::sendUpdate() body["allow_parcel_changes"] = childGetValue("allow_parcel_changes_check"); body["block_parcel_search"] = childGetValue("block_parcel_search_check"); - LLHTTPClient::post(url, body, new LLHTTPClient::Responder()); + LLHTTPClient::post(url, body, new LLHTTPClient::ResponderIgnore); } else { diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index caf2ce7da..cd44d3db9 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -35,7 +35,7 @@ #include "llappviewer.h" #include "llbufferstream.h" #include "llcallbacklist.h" -#include "llcurl.h" +#include "llcurlrequest.h" #include "lldatapacker.h" #include "llfasttimer.h" #if MESH_IMPORT @@ -465,7 +465,6 @@ public: LLMeshRepoThread::LLMeshRepoThread() : LLThread("mesh repo") { - mWaiting = false; mMutex = new LLMutex(); mHeaderMutex = new LLMutex(); mSignal = new LLCondition(); @@ -483,7 +482,7 @@ LLMeshRepoThread::~LLMeshRepoThread() void LLMeshRepoThread::run() { - mCurlRequest = new LLCurlRequest(); + mCurlRequest = new AICurlInterface::Request; #if MESH_IMPORT LLCDResult res = LLConvexDecomposition::initThread(); if (res != LLCD_OK) @@ -492,13 +491,10 @@ void LLMeshRepoThread::run() } #endif //MESH_IMPORT + mSignal->lock(); while (!LLApp::isQuitting()) { - mWaiting = true; - mSignal->wait(); - mWaiting = false; - - if (!LLApp::isQuitting()) + // Left braces in order not to change the indentation. { static U32 count = 0; @@ -511,38 +507,53 @@ void LLMeshRepoThread::run() } // NOTE: throttling intentionally favors LOD requests over header requests - + while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < (S32)sMaxConcurrentRequests) { - if (mMutex) { mMutex->lock(); LODRequest req = mLODReqQ.front(); mLODReqQ.pop(); LLMeshRepository::sLODProcessing--; mMutex->unlock(); - if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit + try { + fetchMeshLOD(req.mMeshParams, req.mLOD, count); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "fetchMeshLOD() failed: " << error.what() << llendl; mMutex->lock(); - mLODReqQ.push(req) ; + LLMeshRepository::sLODProcessing++; + mLODReqQ.push(req); mMutex->unlock(); + break; } } } while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < (S32)sMaxConcurrentRequests) { - if (mMutex) { mMutex->lock(); HeaderRequest req = mHeaderReqQ.front(); mHeaderReqQ.pop(); mMutex->unlock(); - if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit + bool success = false; + try + { + success = fetchMeshHeader(req.mMeshParams, count); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "fetchMeshHeader() failed: " << error.what() << llendl; + } + if (!success) { mMutex->lock(); mHeaderReqQ.push(req) ; mMutex->unlock(); + break; } } } @@ -552,7 +563,16 @@ void LLMeshRepoThread::run() for (std::set::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshSkinInfo(mesh_id)) + bool success = false; + try + { + success = fetchMeshSkinInfo(mesh_id); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "fetchMeshSkinInfo(" << mesh_id << ") failed: " << error.what() << llendl; + } + if (!success) { incomplete.insert(mesh_id); } @@ -565,7 +585,16 @@ void LLMeshRepoThread::run() for (std::set::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshDecomposition(mesh_id)) + bool success = false; + try + { + success = fetchMeshDecomposition(mesh_id); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "fetchMeshDecomposition(" << mesh_id << ") failed: " << error.what() << llendl; + } + if (!success) { incomplete.insert(mesh_id); } @@ -578,7 +607,16 @@ void LLMeshRepoThread::run() for (std::set::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshPhysicsShape(mesh_id)) + bool success = false; + try + { + success = fetchMeshPhysicsShape(mesh_id); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "fetchMeshPhysicsShape(" << mesh_id << ") failed: " << error.what() << llendl; + } + if (!success) { incomplete.insert(mesh_id); } @@ -586,14 +624,11 @@ void LLMeshRepoThread::run() mPhysicsShapeRequests = incomplete; } - mCurlRequest->process(); } + + mSignal->wait(); } - - if (mSignal->isLocked()) - { //make sure to let go of the mutex associated with the given signal before shutting down - mSignal->unlock(); - } + mSignal->unlock(); #if MESH_IMPORT res = LLConvexDecomposition::quitThread(); @@ -681,21 +716,15 @@ std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { //protected by mMutex - - if (!mHeaderMutex) - { - return false; - } - mHeaderMutex->lock(); if (mMeshHeader.find(mesh_id) == mMeshHeader.end()) - { //we have no header info for this mesh, do nothing + { + // We have no header info for this mesh, try again later. mHeaderMutex->unlock(); return false; } - bool ret = true ; U32 header_size = mMeshHeaderSize[mesh_id]; if (header_size > 0) @@ -743,12 +772,10 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshSkinInfoResponder(mesh_id, offset, size)); - if(ret) - { - LLMeshRepository::sHTTPRequestCount++; - } + // This might throw AICurlNoEasyHandle. + mCurlRequest->getByteRange(http_url, headers, offset, size, + new LLMeshSkinInfoResponder(mesh_id, offset, size)); + LLMeshRepository::sHTTPRequestCount++; } } } @@ -758,26 +785,22 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } //early out was not hit, effectively fetched - return ret; + return true; } +//return false if failed to get header bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) { //protected by mMutex - if (!mHeaderMutex) - { - return false; - } - mHeaderMutex->lock(); if (mMeshHeader.find(mesh_id) == mMeshHeader.end()) - { //we have no header info for this mesh, do nothing + { + // We have no header info for this mesh, try again later. mHeaderMutex->unlock(); return false; } U32 header_size = mMeshHeaderSize[mesh_id]; - bool ret = true ; if (header_size > 0) { @@ -824,12 +847,10 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshDecompositionResponder(mesh_id, offset, size)); - if(ret) - { - LLMeshRepository::sHTTPRequestCount++; - } + // This might throw AICurlNoEasyHandle. + mCurlRequest->getByteRange(http_url, headers, offset, size, + new LLMeshDecompositionResponder(mesh_id, offset, size)); + LLMeshRepository::sHTTPRequestCount++; } } } @@ -839,26 +860,22 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } //early out was not hit, effectively fetched - return ret; + return true; } +//return false if failed to get header bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) { //protected by mMutex - if (!mHeaderMutex) - { - return false; - } - mHeaderMutex->lock(); if (mMeshHeader.find(mesh_id) == mMeshHeader.end()) - { //we have no header info for this mesh, do nothing + { + // We have no header info for this mesh, retry later. mHeaderMutex->unlock(); return false; } U32 header_size = mMeshHeaderSize[mesh_id]; - bool ret = true ; if (header_size > 0) { @@ -905,13 +922,10 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - ret = mCurlRequest->getByteRange(http_url, headers, offset, size, - new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); - - if(ret) - { - LLMeshRepository::sHTTPRequestCount++; - } + // This might throw AICurlNoEasyHandle. + mCurlRequest->getByteRange(http_url, headers, offset, size, + new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); + LLMeshRepository::sHTTPRequestCount++; } } else @@ -925,7 +939,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) } //early out was not hit, effectively fetched - return ret; + return true; } //return false if failed to get header @@ -944,14 +958,14 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c LLMeshRepository::sCacheBytesRead += bytes; file.read(buffer, bytes); if (headerReceived(mesh_params, buffer, bytes)) - { //did not do an HTTP request, return false + { + // Already have header, no need to retry. return true; } } } //either cache entry doesn't exist or is corrupt, request header from simulator - bool retval = true ; std::vector headers; headers.push_back("Accept: application/octet-stream"); @@ -961,29 +975,19 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits //within the first 4KB //NOTE -- this will break of headers ever exceed 4KB - retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); - if(retval) - { - LLMeshRepository::sHTTPRequestCount++; - } + // This might throw AICurlNoEasyHandle. + mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); + LLMeshRepository::sHTTPRequestCount++; count++; } - return retval; + return true; } -//return false if failed to get mesh lod. -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) +void LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) { //protected by mMutex - if (!mHeaderMutex) - { - return false; - } - mHeaderMutex->lock(); - bool retval = true; - LLUUID mesh_id = mesh_params.getSculptID(); U32 header_size = mMeshHeaderSize[mesh_id]; @@ -1019,7 +1023,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, if (lodReceived(mesh_params, lod, buffer, size)) { delete[] buffer; - return true; + return; } } @@ -1033,13 +1037,10 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, std::string http_url = constructUrl(mesh_id); if (!http_url.empty()) { - retval = mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size, + // This might throw AICurlNoEasyHandle. + mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size, new LLMeshLODResponder(mesh_params, lod, offset, size)); - - if(retval) - { - LLMeshRepository::sHTTPRequestCount++; - } + LLMeshRepository::sHTTPRequestCount++; count++; } else @@ -1056,8 +1057,6 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, { mHeaderMutex->unlock(); } - - return retval; } bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size) @@ -1102,7 +1101,7 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat LLMutexLock lock(mHeaderMutex); mMeshHeaderSize[mesh_id] = header_size; mMeshHeader[mesh_id] = header; - } + } //check for pending requests pending_lod_map::iterator iter = mPendingLOD.find(mesh_params); @@ -1599,7 +1598,7 @@ void LLMeshUploadThread::generateHulls() void LLMeshUploadThread::doWholeModelUpload() { - mCurlRequest = new LLCurlRequest(); + mCurlRequest = new AICurlInterface::Request(); if (mWholeModelUploadURL.empty()) { @@ -1614,6 +1613,7 @@ void LLMeshUploadThread::doWholeModelUpload() LLSD body = full_model_data["asset_resources"]; dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); LLCurlRequest::headers_t headers; + //FIXME: this might throw AICurlNoEasyHandle mCurlRequest->post(mWholeModelUploadURL, headers, body, new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle), mMeshUploadTimeOut); do @@ -1635,7 +1635,7 @@ void LLMeshUploadThread::requestWholeModelFee() { dump_num++; - mCurlRequest = new LLCurlRequest(); + mCurlRequest = new AICurlInterface::Request; generateHulls(); @@ -1645,6 +1645,7 @@ void LLMeshUploadThread::requestWholeModelFee() mPendingUploads++; LLCurlRequest::headers_t headers; + //FIXME: this might throw AICurlNoEasyHandle mCurlRequest->post(mWholeModelFeeCapability, headers, model_data, new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle), mMeshUploadTimeOut); @@ -1665,11 +1666,6 @@ void LLMeshUploadThread::requestWholeModelFee() void LLMeshRepoThread::notifyLoadedMeshes() {//called via gMeshRepo.notifyLoadedMeshes(). mMutex already locked - if (!mMutex) - { - return; - } - while (!mLoadedQ.empty()) { mMutex->lock(); @@ -2373,17 +2369,17 @@ void LLMeshRepository::notifyLoadedMeshes() mInventoryQ.pop(); } - } - #endif //MESH_IMPORT //call completed callbacks on finished decompositions mDecompThread->notifyCompleted(); - if (!mThread->mWaiting) - { //curl thread is churning, wait for it to go idle + if (!mThread->mSignal->tryLock()) + { + // Curl thread is churning, wait for it to go idle. return; } + mThread->mSignal->unlock(); static std::string region_name("never name a region this"); @@ -3329,6 +3325,7 @@ void LLPhysicsDecomp::run() mStageID[stages[i].mName] = i; } + mSignal->lock(); while (!mQuitting) { mSignal->wait(); @@ -3357,14 +3354,10 @@ void LLPhysicsDecomp::run() } } } + mSignal->unlock(); decomp->quitThread(); - if (mSignal->isLocked()) - { //let go of mSignal's associated mutex - mSignal->unlock(); - } - mDone = true; #endif //MESH_IMPORT } diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index c106f810f..5d225cfe4 100644 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -32,6 +32,7 @@ #include "lluuid.h" #include "llviewertexture.h" #include "llvolume.h" +#include "llcurlrequest.h" #if MESH_IMPORT #define LLCONVEXDECOMPINTER_STATIC 1 @@ -67,7 +68,6 @@ struct LLCDHull class LLVOVolume; class LLMeshResponder; -class LLCurlRequest; class LLMutex; class LLCondition; class LLVFS; @@ -246,13 +246,11 @@ public: static S32 sActiveLODRequests; static U32 sMaxConcurrentRequests; - LLCurlRequest* mCurlRequest; + AICurlInterface::Request* mCurlRequest; LLMutex* mMutex; LLMutex* mHeaderMutex; LLCondition* mSignal; - bool mWaiting; - //map of known mesh headers typedef std::map mesh_header_map; mesh_header_map mMeshHeader; @@ -351,7 +349,7 @@ public: void loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod); bool fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count); - bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count); + void fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count); bool headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size); bool lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size); bool skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size); diff --git a/indra/newview/llpanelclassified.cpp b/indra/newview/llpanelclassified.cpp index 5fe634eed..5421cd886 100644 --- a/indra/newview/llpanelclassified.cpp +++ b/indra/newview/llpanelclassified.cpp @@ -999,7 +999,7 @@ void LLPanelClassified::sendClassifiedClickMessage(const std::string& type) std::string url = gAgent.getRegion()->getCapability("SearchStatTracking"); llinfos << "LLPanelClassified::sendClassifiedClickMessage via capability" << llendl; - LLHTTPClient::post(url, body, new LLHTTPClient::Responder()); + LLHTTPClient::post(url, body, new LLHTTPClient::ResponderIgnore); } //////////////////////////////////////////////////////////////////////////////////////////// diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp index a1fad9902..af0b25add 100644 --- a/indra/newview/llstartup.cpp +++ b/indra/newview/llstartup.cpp @@ -3589,6 +3589,7 @@ std::string LLStartUp::startupStateToString(EStartupState state) #define RTNENUM(E) case E: return #E switch(state){ RTNENUM( STATE_FIRST ); + RTNENUM( STATE_BROWSER_INIT ); RTNENUM( STATE_LOGIN_SHOW ); RTNENUM( STATE_LOGIN_WAIT ); RTNENUM( STATE_LOGIN_CLEANUP ); @@ -3596,10 +3597,13 @@ std::string LLStartUp::startupStateToString(EStartupState state) RTNENUM( STATE_UPDATE_CHECK ); RTNENUM( STATE_LOGIN_AUTH_INIT ); RTNENUM( STATE_LOGIN_AUTHENTICATE ); + RTNENUM( STATE_WAIT_LEGACY_LOGIN ); + RTNENUM( STATE_XMLRPC_LEGACY_LOGIN ); RTNENUM( STATE_LOGIN_NO_DATA_YET ); RTNENUM( STATE_LOGIN_DOWNLOADING ); RTNENUM( STATE_LOGIN_PROCESS_RESPONSE ); RTNENUM( STATE_WORLD_INIT ); + RTNENUM( STATE_MULTIMEDIA_INIT ); RTNENUM( STATE_FONT_INIT ); RTNENUM( STATE_SEED_GRANTED_WAIT ); RTNENUM( STATE_SEED_CAP_GRANTED ); @@ -3612,10 +3616,10 @@ std::string LLStartUp::startupStateToString(EStartupState state) RTNENUM( STATE_WEARABLES_WAIT ); RTNENUM( STATE_CLEANUP ); RTNENUM( STATE_STARTED ); - default: - return llformat("(state #%d)", state); } #undef RTNENUM + // Never reached. + return llformat("(state #%d)", state); } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 72132496e..a638e39fd 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1319,8 +1319,15 @@ bool LLTextureFetchWorker::doWork(S32 param) // Will call callbackHttpGet when curl request completes std::vector headers; headers.push_back("Accept: image/x-j2c"); - res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); + try + { + res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, + new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << error.what() << llendl; + } } if (!res) { @@ -2435,7 +2442,7 @@ void LLTextureFetch::shutDownImageDecodeThread() void LLTextureFetch::startThread() { // Construct mCurlGetRequest from Worker Thread - mCurlGetRequest = new LLCurlRequest(); + mCurlGetRequest = new AICurlInterface::Request; } // WORKER THREAD diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index ee3cd2c75..03073d906 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -37,7 +37,7 @@ #include "llimage.h" #include "lluuid.h" #include "llworkerthread.h" -#include "llcurl.h" +#include "llcurlrequest.h" #include "lltextureinfo.h" #include "llapr.h" @@ -107,7 +107,7 @@ public: LLViewerAssetStats * main_stats); void commandDataBreak(); - LLCurlRequest & getCurlRequest() { return *mCurlGetRequest; } + AICurlInterface::Request& getCurlRequest() { return *mCurlGetRequest; } bool isQAMode() const { return mQAMode; } @@ -178,7 +178,7 @@ private: LLTextureCache* mTextureCache; LLImageDecodeThread* mImageDecodeThread; - LLCurlRequest* mCurlGetRequest; + AICurlInterface::Request* mCurlGetRequest; // Map of all requests by UUID typedef std::map map_t; diff --git a/indra/newview/llviewerparcelmedia.cpp b/indra/newview/llviewerparcelmedia.cpp index 1493f4abb..fb78b29fa 100644 --- a/indra/newview/llviewerparcelmedia.cpp +++ b/indra/newview/llviewerparcelmedia.cpp @@ -462,7 +462,7 @@ void LLViewerParcelMedia::sendMediaNavigateMessage(const std::string& url) body["agent-id"] = gAgent.getID(); body["local-id"] = LLViewerParcelMgr::getInstance()->getAgentParcel()->getLocalID(); body["url"] = url; - LLHTTPClient::post(region_url, body, new LLHTTPClient::Responder); + LLHTTPClient::post(region_url, body, new LLHTTPClient::ResponderIgnore); } else { diff --git a/indra/newview/llviewerparcelmgr.cpp b/indra/newview/llviewerparcelmgr.cpp index 64f20ff50..529d0e034 100644 --- a/indra/newview/llviewerparcelmgr.cpp +++ b/indra/newview/llviewerparcelmgr.cpp @@ -1310,7 +1310,7 @@ void LLViewerParcelMgr::sendParcelPropertiesUpdate(LLParcel* parcel, bool use_ag parcel->packMessage(body); llinfos << "Sending parcel properties update via capability to: " << url << llendl; - LLHTTPClient::post(url, body, new LLHTTPClient::Responder()); + LLHTTPClient::post(url, body, new LLHTTPClient::ResponderIgnore); } else { diff --git a/indra/newview/llxmlrpctransaction.cpp b/indra/newview/llxmlrpctransaction.cpp index af5e977b2..5abe2df2e 100644 --- a/indra/newview/llxmlrpctransaction.cpp +++ b/indra/newview/llxmlrpctransaction.cpp @@ -43,6 +43,11 @@ #include "llappviewer.h" #include "hippogridmanager.h" +#include "statemachine/aicurleasyrequeststatemachine.h" + +#ifdef CWDEBUG +#include +#endif LLXMLRPCValue LLXMLRPCValue::operator[](const char* id) const { @@ -154,7 +159,7 @@ class LLXMLRPCTransaction::Impl public: typedef LLXMLRPCTransaction::Status Status; - LLCurlEasyRequest* mCurlRequest; + AICurlEasyRequestStateMachine* mCurlEasyRequestStateMachinePtr; Status mStatus; CURLcode mCurlCode; @@ -176,7 +181,8 @@ public: const std::string& method, LLXMLRPCValue params, bool useGzip); ~Impl(); - bool process(); + bool is_finished(void) const; + void curlEasyRequestCallback(bool success); void setStatus(Status code, const std::string& message = "", const std::string& uri = ""); @@ -191,7 +197,7 @@ private: LLXMLRPCTransaction::Impl::Impl(const std::string& uri, XMLRPC_REQUEST request, bool useGzip) - : mCurlRequest(0), + : mCurlEasyRequestStateMachinePtr(NULL), mStatus(LLXMLRPCTransaction::StatusNotStarted), mURI(uri), mRequestText(0), @@ -203,7 +209,7 @@ LLXMLRPCTransaction::Impl::Impl(const std::string& uri, LLXMLRPCTransaction::Impl::Impl(const std::string& uri, const std::string& method, LLXMLRPCValue params, bool useGzip) - : mCurlRequest(0), + : mCurlEasyRequestStateMachinePtr(NULL), mStatus(LLXMLRPCTransaction::StatusNotStarted), mURI(uri), mRequestText(0), @@ -222,60 +228,66 @@ LLXMLRPCTransaction::Impl::Impl(const std::string& uri, void LLXMLRPCTransaction::Impl::init(XMLRPC_REQUEST request, bool useGzip) { - if (!mCurlRequest) { - mCurlRequest = new LLCurlEasyRequest(); - } + try + { + mCurlEasyRequestStateMachinePtr = new AICurlEasyRequestStateMachine(false); + } + catch(AICurlNoEasyHandle const& error) + { + llwarns << "Failed to initialize LLXMLRPCTransaction: " << error.what() << llendl; + setStatus(StatusOtherError, "No curl easy handle"); + return; + } + AICurlEasyRequest_wat curlEasyRequest_w(*mCurlEasyRequestStateMachinePtr->mCurlEasyRequest); - if(!mCurlRequest->isValid()) + curlEasyRequest_w->setWriteCallback(&curlDownloadCallback, (void*)this); + BOOL vefifySSLCert = !gSavedSettings.getBOOL("NoVerifySSLCert"); + curlEasyRequest_w->setopt(CURLOPT_SSL_VERIFYPEER, vefifySSLCert); + curlEasyRequest_w->setopt(CURLOPT_SSL_VERIFYHOST, vefifySSLCert ? 2 : 0); + // Be a little impatient about establishing connections. + curlEasyRequest_w->setopt(CURLOPT_CONNECTTIMEOUT, 40L); + + /* Setting the DNS cache timeout to -1 disables it completely. + This might help with bug #503 */ + curlEasyRequest_w->setopt(CURLOPT_DNS_CACHE_TIMEOUT, -1); + + curlEasyRequest_w->addHeader("Content-Type: text/xml"); + + if (useGzip) + { + curlEasyRequest_w->setoptString(CURLOPT_ENCODING, ""); + } + + mRequestText = XMLRPC_REQUEST_ToXML(request, &mRequestTextSize); + if (mRequestText) + { + Dout(dc::curl, "Writing " << mRequestTextSize << " bytes: \"" << libcwd::buf2str(mRequestText, mRequestTextSize) << "\".");; + curlEasyRequest_w->setopt(CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)mRequestTextSize); + curlEasyRequest_w->setoptString(CURLOPT_COPYPOSTFIELDS, mRequestText); + } + else + { + setStatus(StatusOtherError); + } + + curlEasyRequest_w->finalizeRequest(mURI); + } + if (mStatus == LLXMLRPCTransaction::StatusNotStarted) // It could be LLXMLRPCTransaction::StatusOtherError. { - llwarns << "mCurlRequest is invalid." << llendl ; - - delete mCurlRequest ; - mCurlRequest = NULL ; - return ; + mCurlEasyRequestStateMachinePtr->run(boost::bind(&LLXMLRPCTransaction::Impl::curlEasyRequestCallback, this, _1)); + setStatus(LLXMLRPCTransaction::StatusStarted); } - - - LLProxy::getInstance()->applyProxySettings(mCurlRequest); - -// mCurlRequest->setopt(CURLOPT_VERBOSE, 1); // usefull for debugging - mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); - mCurlRequest->setWriteCallback(&curlDownloadCallback, (void*)this); - BOOL vefifySSLCert = !gSavedSettings.getBOOL("NoVerifySSLCert"); - mCurlRequest->setopt(CURLOPT_SSL_VERIFYPEER, vefifySSLCert); - mCurlRequest->setopt(CURLOPT_SSL_VERIFYHOST, vefifySSLCert ? 2 : 0); - // Be a little impatient about establishing connections. - mCurlRequest->setopt(CURLOPT_CONNECTTIMEOUT, 40L); - - /* Setting the DNS cache timeout to -1 disables it completely. - This might help with bug #503 */ - mCurlRequest->setopt(CURLOPT_DNS_CACHE_TIMEOUT, -1); - - mCurlRequest->slist_append("Content-Type: text/xml"); - - if (useGzip) - { - mCurlRequest->setoptString(CURLOPT_ENCODING, ""); - } - - mRequestText = XMLRPC_REQUEST_ToXML(request, &mRequestTextSize); - if (mRequestText) - { - mCurlRequest->setoptString(CURLOPT_POSTFIELDS, mRequestText); - mCurlRequest->setopt(CURLOPT_POSTFIELDSIZE, mRequestTextSize); - } - else - { - setStatus(StatusOtherError); - } - - mCurlRequest->sendRequest(mURI); } - LLXMLRPCTransaction::Impl::~Impl() { + if (mCurlEasyRequestStateMachinePtr && mCurlEasyRequestStateMachinePtr->running()) + { + llwarns << "Calling LLXMLRPCTransaction::Impl::~Impl while mCurlEasyRequestStateMachinePtr is still running" << llendl; + mCurlEasyRequestStateMachinePtr->abort(); + } + if (mResponse) { XMLRPC_RequestFree(mResponse, 1); @@ -285,119 +297,76 @@ LLXMLRPCTransaction::Impl::~Impl() { XMLRPC_Free(mRequestText); } - - delete mCurlRequest; - mCurlRequest = NULL ; } -bool LLXMLRPCTransaction::Impl::process() +bool LLXMLRPCTransaction::Impl::is_finished(void) const { - if(!mCurlRequest || !mCurlRequest->isValid()) - { - llwarns << "transaction failed." << llendl ; + // Nothing to process anymore. Just wait till the statemachine finished. + return mStatus != LLXMLRPCTransaction::StatusNotStarted && + mStatus != LLXMLRPCTransaction::StatusStarted && + mStatus != LLXMLRPCTransaction::StatusDownloading; +} - delete mCurlRequest ; - mCurlRequest = NULL ; - return true ; //failed, quit. +void LLXMLRPCTransaction::Impl::curlEasyRequestCallback(bool success) +{ + llassert(mStatus == LLXMLRPCTransaction::StatusStarted || mStatus == LLXMLRPCTransaction::StatusDownloading); + + if (!success) + { + setStatus(LLXMLRPCTransaction::StatusOtherError, "Statemachine failed"); + return; } - switch(mStatus) + AICurlEasyRequest_wat curlEasyRequest_w(*mCurlEasyRequestStateMachinePtr->mCurlEasyRequest); + CURLcode result; + curlEasyRequest_w->getResult(&result, &mTransferInfo); + + if (result != CURLE_OK) { - case LLXMLRPCTransaction::StatusComplete: - case LLXMLRPCTransaction::StatusCURLError: - case LLXMLRPCTransaction::StatusXMLRPCError: - case LLXMLRPCTransaction::StatusOtherError: - { - return true; - } - - case LLXMLRPCTransaction::StatusNotStarted: - { - setStatus(LLXMLRPCTransaction::StatusStarted); - break; - } - - default: - { - // continue onward - } + setCurlStatus(result); + llwarns << "LLXMLRPCTransaction CURL error " + << mCurlCode << ": " << curlEasyRequest_w->getErrorString() << llendl; + llwarns << "LLXMLRPCTransaction request URI: " + << mURI << llendl; + + return; } - //const F32 MAX_PROCESSING_TIME = 0.05f; - //LLTimer timer; + setStatus(LLXMLRPCTransaction::StatusComplete); - mCurlRequest->wait(); + mResponse = XMLRPC_REQUEST_FromXML( + mResponseText.data(), mResponseText.size(), NULL); - /*while (mCurlRequest->perform() > 0) + bool hasError = false; + bool hasFault = false; + int faultCode = 0; + std::string faultString; + + LLXMLRPCValue error(XMLRPC_RequestGetError(mResponse)); + if (error.isValid()) { - if (timer.getElapsedTimeF32() >= MAX_PROCESSING_TIME) - { - return false; - } - }*/ - - while(1) - { - CURLcode result; - bool newmsg = mCurlRequest->getResult(&result, &mTransferInfo); - if (newmsg) - { - if (result != CURLE_OK) - { - setCurlStatus(result); - llwarns << "LLXMLRPCTransaction CURL error " - << mCurlCode << ": " << mCurlRequest->getErrorString() << llendl; - llwarns << "LLXMLRPCTransaction request URI: " - << mURI << llendl; - - return true; - } - - setStatus(LLXMLRPCTransaction::StatusComplete); - - mResponse = XMLRPC_REQUEST_FromXML( - mResponseText.data(), mResponseText.size(), NULL); - - bool hasError = false; - bool hasFault = false; - int faultCode = 0; - std::string faultString; - - LLXMLRPCValue error(XMLRPC_RequestGetError(mResponse)); - if (error.isValid()) - { - hasError = true; - faultCode = error["faultCode"].asInt(); - faultString = error["faultString"].asString(); - } - else if (XMLRPC_ResponseIsFault(mResponse)) - { - hasFault = true; - faultCode = XMLRPC_GetResponseFaultCode(mResponse); - faultString = XMLRPC_GetResponseFaultString(mResponse); - } - - if (hasError || hasFault) - { - setStatus(LLXMLRPCTransaction::StatusXMLRPCError); - - llwarns << "LLXMLRPCTransaction XMLRPC " - << (hasError ? "error " : "fault ") - << faultCode << ": " - << faultString << llendl; - llwarns << "LLXMLRPCTransaction request URI: " - << mURI << llendl; - } - - return true; - } - else - { - break; // done - } + hasError = true; + faultCode = error["faultCode"].asInt(); + faultString = error["faultString"].asString(); + } + else if (XMLRPC_ResponseIsFault(mResponse)) + { + hasFault = true; + faultCode = XMLRPC_GetResponseFaultCode(mResponse); + faultString = XMLRPC_GetResponseFaultString(mResponse); + } + + if (hasError || hasFault) + { + setStatus(LLXMLRPCTransaction::StatusXMLRPCError); + + llwarns << "LLXMLRPCTransaction XMLRPC " + << (hasError ? "error " : "fault ") + << faultCode << ": " + << faultString << llendl; + llwarns << "LLXMLRPCTransaction request URI: " + << mURI << llendl; } - - return false; } void LLXMLRPCTransaction::Impl::setStatus(Status status, @@ -489,6 +458,8 @@ void LLXMLRPCTransaction::Impl::setCurlStatus(CURLcode code) size_t LLXMLRPCTransaction::Impl::curlDownloadCallback( char* data, size_t size, size_t nmemb, void* user_data) { + DoutEntering(dc::curl, "LLXMLRPCTransaction::Impl::curlDownloadCallback(\"" << buf2str(data, size * nmemb) << "\", " << size << ", " << nmemb << ", " << user_data << ")"); + Impl& impl(*(Impl*)user_data); size_t n = size * nmemb; @@ -523,7 +494,7 @@ LLXMLRPCTransaction::~LLXMLRPCTransaction() bool LLXMLRPCTransaction::process() { - return impl.process(); + return impl.is_finished(); } LLXMLRPCTransaction::Status LLXMLRPCTransaction::status(int* curlCode) diff --git a/indra/newview/llxmlrpctransaction.h b/indra/newview/llxmlrpctransaction.h index 528451fcb..42c236265 100644 --- a/indra/newview/llxmlrpctransaction.h +++ b/indra/newview/llxmlrpctransaction.h @@ -111,7 +111,7 @@ public: } Status; bool process(); - // run the request a little, returns true when done + // Returns true when done. Status status(int* curlCode); // return status, and extended CURL code, if code isn't null @@ -127,7 +127,7 @@ public: // retains ownership of the result object, don't free it F64 transferRate(); - // only valid if StsatusComplete, otherwise 0.0 + // only valid if StatusComplete, otherwise 0.0 private: class Impl; diff --git a/indra/newview/statemachine/CMakeLists.txt b/indra/newview/statemachine/CMakeLists.txt index d36caca38..08ecccecf 100644 --- a/indra/newview/statemachine/CMakeLists.txt +++ b/indra/newview/statemachine/CMakeLists.txt @@ -5,7 +5,7 @@ project(statemachine) include(00-Common) include(LLCommon) include(LLPlugin) -include(LLMessage) # This is needed by LLPlugin. +include(LLMessage) include(LLMath) include(LLVFS) include(LLXML) @@ -38,6 +38,7 @@ include_directories( set(statemachine_SOURCE_FILES aistatemachine.cpp + aicurleasyrequeststatemachine.cpp aifilepicker.cpp aifetchinventoryfolder.cpp aievent.cpp @@ -47,6 +48,7 @@ set(statemachine_SOURCE_FILES set(statemachine_HEADER_FILES CMakeLists.txt aistatemachine.h + aicurleasyrequeststatemachine.h aifilepicker.h aidirpicker.h aifetchinventoryfolder.h diff --git a/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp b/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp new file mode 100644 index 000000000..a308be093 --- /dev/null +++ b/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp @@ -0,0 +1,152 @@ +/** + * @file aicurleasyrequeststatemachine.cpp + * @brief Implementation of AICurlEasyRequestStateMachine + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 06/05/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#include "linden_common.h" +#include "aicurleasyrequeststatemachine.h" + +enum curleasyrequeststatemachine_state_type { + AICurlEasyRequestStateMachine_addRequest = AIStateMachine::max_state, + AICurlEasyRequestStateMachine_waitAdded, + AICurlEasyRequestStateMachine_waitFinished, + AICurlEasyRequestStateMachine_finished +}; + +char const* AICurlEasyRequestStateMachine::state_str_impl(state_type run_state) const +{ + switch(run_state) + { + AI_CASE_RETURN(AICurlEasyRequestStateMachine_addRequest); + AI_CASE_RETURN(AICurlEasyRequestStateMachine_waitAdded); + AI_CASE_RETURN(AICurlEasyRequestStateMachine_waitFinished); + AI_CASE_RETURN(AICurlEasyRequestStateMachine_finished); + } + return "UNKNOWN STATE"; +} + +void AICurlEasyRequestStateMachine::initialize_impl(void) +{ + { + AICurlEasyRequest_wat curlEasyRequest_w(*mCurlEasyRequest); + llassert(curlEasyRequest_w->is_finalized()); // Call finalizeRequest(url) before calling run(). + curlEasyRequest_w->send_events_to(this); + } + set_state(AICurlEasyRequestStateMachine_addRequest); +} + +// CURL-THREAD +void AICurlEasyRequestStateMachine::added_to_multi_handle(AICurlEasyRequest_wat&) +{ + set_state(AICurlEasyRequestStateMachine_waitFinished); +} + +// CURL-THREAD +void AICurlEasyRequestStateMachine::finished(AICurlEasyRequest_wat&) +{ +} + +// CURL-THREAD +void AICurlEasyRequestStateMachine::removed_from_multi_handle(AICurlEasyRequest_wat&) +{ + set_state(AICurlEasyRequestStateMachine_finished); +} + +void AICurlEasyRequestStateMachine::multiplex_impl(void) +{ + switch (mRunState) + { + case AICurlEasyRequestStateMachine_addRequest: + { + mCurlEasyRequest.addRequest(); + set_state(AICurlEasyRequestStateMachine_waitAdded); + } + case AICurlEasyRequestStateMachine_waitAdded: + { + idle(); // Wait till AICurlEasyRequestStateMachine::added_to_multi_handle() is called. + break; + } + case AICurlEasyRequestStateMachine_waitFinished: + { + idle(); // Wait till AICurlEasyRequestStateMachine::finished() is called. + break; + } + case AICurlEasyRequestStateMachine_finished: + { + if (mBuffered) + { + AICurlEasyRequest_wat easy_request_w(*mCurlEasyRequest); + AICurlResponderBuffer_wat buffered_easy_request_w(*mCurlEasyRequest); + buffered_easy_request_w->processOutput(easy_request_w); + } + finish(); + break; + } + } +} + +void AICurlEasyRequestStateMachine::abort_impl(void) +{ + Dout(dc::curl, "AICurlEasyRequestStateMachine::abort_impl called for = " << (void*)mCurlEasyRequest.get()); + // We must first revoke the events, or the curl thread might change mRunState still. + { + AICurlEasyRequest_wat curl_easy_request_w(*mCurlEasyRequest); + curl_easy_request_w->send_events_to(NULL); + curl_easy_request_w->revokeCallbacks(); + } + if (mRunState >= AICurlEasyRequestStateMachine_waitAdded && mRunState < AICurlEasyRequestStateMachine_finished) + { + // Revert call to addRequest(). + // Note that it's safe to call this even if the curl thread already removed it, or will removes it + // after we called this, before processing the remove command; only the curl thread calls + // MultiHandle::remove_easy_request, which is a no-op when called twice for the same easy request. + mCurlEasyRequest.removeRequest(); + } +} + +void AICurlEasyRequestStateMachine::finish_impl(void) +{ + Dout(dc::curl, "AICurlEasyRequestStateMachine::finish_impl called for = " << (void*)mCurlEasyRequest.get()); + if (!aborted()) + { + AICurlEasyRequest_wat curl_easy_request_w(*mCurlEasyRequest); + curl_easy_request_w->send_events_to(NULL); + curl_easy_request_w->revokeCallbacks(); + } +} + +AICurlEasyRequestStateMachine::AICurlEasyRequestStateMachine(bool buffered) : mBuffered(buffered), mCurlEasyRequest(buffered) +{ + Dout(dc::statemachine, "Calling AICurlEasyRequestStateMachine(" << (buffered ? "true" : "false") << ") [" << (void*)this << "] [" << (void*)mCurlEasyRequest.get() << "]"); +} + +AICurlEasyRequestStateMachine::~AICurlEasyRequestStateMachine() +{ + Dout(dc::statemachine, "Calling ~AICurlEasyRequestStateMachine() [" << (void*)this << "] [" << (void*)mCurlEasyRequest.get() << "]"); +} + diff --git a/indra/newview/statemachine/aicurleasyrequeststatemachine.h b/indra/newview/statemachine/aicurleasyrequeststatemachine.h new file mode 100644 index 000000000..41c3c7256 --- /dev/null +++ b/indra/newview/statemachine/aicurleasyrequeststatemachine.h @@ -0,0 +1,96 @@ +/** + * @file aicurleasyrequest.h + * @brief Perform a curl easy request. + * + * Copyright (c) 2012, Aleric Inglewood. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * There are special exceptions to the terms and conditions of the GPL as + * it is applied to this Source Code. View the full text of the exception + * in the file doc/FLOSS-exception.txt in this software distribution. + * + * CHANGELOG + * and additional copyright holders. + * + * 06/05/2012 + * Initial version, written by Aleric Inglewood @ SL + */ + +#ifndef AICURLEASYREQUEST_H +#define AICURLEASYREQUEST_H + +#include "aistatemachine.h" +#include "aicurl.h" + +// A curl easy request state machine. +// +// Before calling cersm.run() initialize the object (cersm) as follows: +// +// AICurlEasyRequest_wat cersm_w(cersm); +// cersm_w->setopt(...); // etc, see the interface of AICurlPrivate::CurlEasyRequest and it's base class AICurlPrivate::CurlEasyHandle. +// +// When the state machine finishes, call aborted() to check +// whether or not the statemachine succeeded in fetching +// the URL or not. +// +// Objects of this type can be reused multiple times, see +// also the documentation of AIStateMachine. +// +// Construction of a AICurlEasyRequestStateMachine might throw AICurlNoEasyHandle. +class AICurlEasyRequestStateMachine : public AIStateMachine, public AICurlEasyHandleEvents { + public: + AICurlEasyRequestStateMachine(bool buffered); + + // Transparent access. + AICurlEasyRequest mCurlEasyRequest; + + private: + bool mBuffered; // Argument used for construction of mCurlEasyRequest. + + protected: + // AICurlEasyRequest Events. + + // Called when this curl easy handle was added to a multi handle. + /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat&); + + // Called when this curl easy handle finished processing (right before it is removed from the multi handle). + /*virtual*/ void finished(AICurlEasyRequest_wat&); + + // Called after this curl easy handle was removed from a multi handle. + /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat&); + + protected: + // AIStateMachine implementations. + + // Call finish() (or abort()), not delete. + /*virtual*/ ~AICurlEasyRequestStateMachine(); + + // Handle initializing the object. + /*virtual*/ void initialize_impl(void); + + // Handle mRunState. + /*virtual*/ void multiplex_impl(void); + + // Handle aborting from current bs_run state. + /*virtual*/ void abort_impl(void); + + // Handle cleaning up from initialization (or post abort) state. + /*virtual*/ void finish_impl(void); + + // Implemenation of state_str for run states. + /*virtual*/ char const* state_str_impl(state_type run_state) const; +}; + +#endif diff --git a/indra/newview/statemachine/aistatemachine.h b/indra/newview/statemachine/aistatemachine.h index f20fcf048..16eb28cd4 100644 --- a/indra/newview/statemachine/aistatemachine.h +++ b/indra/newview/statemachine/aistatemachine.h @@ -237,7 +237,7 @@ class AIStateMachine { protected: //! The user should call 'kill()', not delete a AIStateMachine (derived) directly. - virtual ~AIStateMachine() { llassert(mState == bs_killed && mActive == as_idle); } + virtual ~AIStateMachine() { llassert((mState == bs_killed && mActive == as_idle) || mState == bs_initialize); } public: //! Halt the state machine until cont() is called. From 1f56645b69b853024e77234bd3d5d13785280970 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Thu, 28 Jun 2012 05:56:21 +0200 Subject: [PATCH 02/11] Always set proxy settings for every HTTP curl connection. Move applyProxySettings to CurlEasyRequest and call it from applyDefaultOptions. Use AIThreadSafe for LLProxy for a more robust threadsafeness. (This forces correct locking, checks that the unshared vars are indeed unshared and made it easy to use read/write locking, which might be important in this case (we do a lot of read-only accesses to it). --- indra/llmessage/aicurl.cpp | 35 +++++- indra/llmessage/aicurlprivate.h | 4 + indra/llmessage/llproxy.cpp | 181 +++++++++----------------------- indra/llmessage/llproxy.h | 152 ++++++++++++++++----------- 4 files changed, 179 insertions(+), 193 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 6278c91c2..3ab86f56f 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -805,6 +805,39 @@ static int curl_debug_callback(CURL*, curl_infotype infotype, char* buf, size_t } #endif +void CurlEasyRequest::applyProxySettings(void) +{ + LLProxy& proxy = *LLProxy::getInstance(); + + // Do a faster unlocked check to see if we are supposed to proxy. + if (proxy.HTTPProxyEnabled()) + { + // We think we should proxy, read lock the shared proxy members. + LLProxy::Shared_crat proxy_r(proxy.shared_lockobj()); + + // Now test again to verify that the proxy wasn't disabled between the first check and the lock. + if (proxy.HTTPProxyEnabled()) + { + setopt(CURLOPT_PROXY, proxy.getHTTPProxy(proxy_r).getIPString().c_str()); + setopt(CURLOPT_PROXYPORT, proxy.getHTTPProxy(proxy_r).getPort()); + + if (proxy.getHTTPProxyType(proxy_r) == LLPROXY_SOCKS) + { + setopt(CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); + if (proxy.getSelectedAuthMethod(proxy_r) == METHOD_PASSWORD) + { + std::string auth_string = proxy.getSocksUser(proxy_r) + ":" + proxy.getSocksPwd(proxy_r); + setopt(CURLOPT_PROXYUSERPWD, auth_string.c_str()); + } + } + else + { + setopt(CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + } + } + } +} + void CurlEasyRequest::applyDefaultOptions(void) { CertificateAuthority_rat CertificateAuthority_r(gCertificateAuthority); @@ -822,7 +855,7 @@ void CurlEasyRequest::applyDefaultOptions(void) // The old code did this for the 'buffered' version, but I think it's nonsense. //setopt(CURLOPT_DNS_CACHE_TIMEOUT, 0); // Set the CURL options for either Socks or HTTP proxy. - LLProxy::getInstance()->applyProxySettings(this); + applyProxySettings(); Debug( if (dc::curl.is_on()) { diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index 9a7f50180..fcf51e927 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -176,6 +176,10 @@ class CurlEasyRequest : public CurlEasyHandle { // Reset everything to the state it was in when this object was just created. void resetState(void); + private: + // Called from applyDefaultOptions. + void applyProxySettings(void); + public: // Set default options that we want applied to all curl easy handles. void applyDefaultOptions(void); diff --git a/indra/llmessage/llproxy.cpp b/indra/llmessage/llproxy.cpp index 3db851001..a82ae3cc7 100644 --- a/indra/llmessage/llproxy.cpp +++ b/indra/llmessage/llproxy.cpp @@ -47,23 +47,22 @@ static apr_status_t tcp_blocking_handshake(LLSocket::ptr_t handle, char * dataou static LLSocket::ptr_t tcp_open_channel(LLHost host); // Open a TCP channel to a given host static void tcp_close_channel(LLSocket::ptr_t* handle_ptr); // Close an open TCP channel -LLProxy::LLProxy(): - mHTTPProxyEnabled(false), - mProxyMutex(), - mUDPProxy(), - mTCPProxy(), - mHTTPProxy(), +ProxyShared::ProxyShared(void): mProxyType(LLPROXY_SOCKS), - mAuthMethodSelected(METHOD_NOAUTH), - mSocksUsername(), - mSocksPassword() + mAuthMethodSelected(METHOD_NOAUTH) +{ +} + +LLProxy::LLProxy(): + mHTTPProxyEnabled(false) { } LLProxy::~LLProxy() { stopSOCKSProxy(); - disableHTTPProxy(); + Shared_wat shared_w(mShared); + disableHTTPProxy(shared_w); } /** @@ -78,15 +77,18 @@ S32 LLProxy::proxyHandshake(LLHost proxy) { S32 result; + Unshared_rat unshared_r(mUnshared); + Shared_rat shared_r(mShared); + /* SOCKS 5 Auth request */ socks_auth_request_t socks_auth_request; socks_auth_response_t socks_auth_response; socks_auth_request.version = SOCKS_VERSION; // SOCKS version 5 socks_auth_request.num_methods = 1; // Sending 1 method. - socks_auth_request.methods = getSelectedAuthMethod(); // Send only the selected method. + socks_auth_request.methods = getSelectedAuthMethod(shared_r); // Send only the selected method. - result = tcp_blocking_handshake(mProxyControlChannel, + result = tcp_blocking_handshake(unshared_r->mProxyControlChannel, static_cast(static_cast(&socks_auth_request)), sizeof(socks_auth_request), static_cast(static_cast(&socks_auth_response)), @@ -109,8 +111,8 @@ S32 LLProxy::proxyHandshake(LLHost proxy) if (socks_auth_response.method == METHOD_PASSWORD) { // The server has requested a username/password combination - std::string socks_username(getSocksUser()); - std::string socks_password(getSocksPwd()); + std::string socks_username(getSocksUser(shared_r)); + std::string socks_password(getSocksPwd(shared_r)); U32 request_size = socks_username.size() + socks_password.size() + 3; char * password_auth = new char[request_size]; password_auth[0] = 0x01; @@ -121,7 +123,7 @@ S32 LLProxy::proxyHandshake(LLHost proxy) authmethod_password_reply_t password_reply; - result = tcp_blocking_handshake(mProxyControlChannel, + result = tcp_blocking_handshake(unshared_r->mProxyControlChannel, password_auth, request_size, static_cast(static_cast(&password_reply)), @@ -157,7 +159,7 @@ S32 LLProxy::proxyHandshake(LLHost proxy) // "If the client is not in possession of the information at the time of the UDP ASSOCIATE, // the client MUST use a port number and address of all zeros. RFC 1928" - result = tcp_blocking_handshake(mProxyControlChannel, + result = tcp_blocking_handshake(unshared_r->mProxyControlChannel, static_cast(static_cast(&connect_request)), sizeof(connect_request), static_cast(static_cast(&connect_reply)), @@ -176,10 +178,14 @@ S32 LLProxy::proxyHandshake(LLHost proxy) return SOCKS_UDP_FWD_NOT_GRANTED; } - mUDPProxy.setPort(ntohs(connect_reply.port)); // reply port is in network byte order - mUDPProxy.setAddress(proxy.getAddress()); + { + // Write acccess type and read access type are really the same, so unshared_w must be simply a reference. + Unshared_wat& unshared_w = unshared_r; + unshared_w->mUDPProxy.setPort(ntohs(connect_reply.port)); // reply port is in network byte order + unshared_w->mUDPProxy.setAddress(proxy.getAddress()); + } // The connection was successful. We now have the UDP port to send requests that need forwarding to. - LL_INFOS("Proxy") << "SOCKS 5 UDP proxy connected on " << mUDPProxy << LL_ENDL; + LL_INFOS("Proxy") << "SOCKS 5 UDP proxy connected on " << unshared_r->mUDPProxy << LL_ENDL; return SOCKS_OK; } @@ -197,9 +203,11 @@ S32 LLProxy::proxyHandshake(LLHost proxy) */ S32 LLProxy::startSOCKSProxy(LLHost host) { + Unshared_wat unshared_w(mUnshared); + if (host.isOk()) { - mTCPProxy = host; + unshared_w->mTCPProxy = host; } else { @@ -209,13 +217,13 @@ S32 LLProxy::startSOCKSProxy(LLHost host) // Close any running SOCKS connection. stopSOCKSProxy(); - mProxyControlChannel = tcp_open_channel(mTCPProxy); - if (!mProxyControlChannel) + unshared_w->mProxyControlChannel = tcp_open_channel(unshared_w->mTCPProxy); + if (!unshared_w->mProxyControlChannel) { return SOCKS_HOST_CONNECT_FAILED; } - S32 status = proxyHandshake(mTCPProxy); + S32 status = proxyHandshake(unshared_w->mTCPProxy); if (status != SOCKS_OK) { @@ -246,14 +254,16 @@ void LLProxy::stopSOCKSProxy() // then we must shut down any HTTP proxy operations. But it is allowable if web // proxy is being used to continue proxying HTTP. - if (LLPROXY_SOCKS == getHTTPProxyType()) + Shared_rat shared_r(mShared); + if (LLPROXY_SOCKS == getHTTPProxyType(shared_r)) { - disableHTTPProxy(); + Shared_wat shared_w(shared_r); + disableHTTPProxy(shared_w); } - - if (mProxyControlChannel) + Unshared_wat unshared_w(mUnshared); + if (unshared_w->mProxyControlChannel) { - tcp_close_channel(&mProxyControlChannel); + tcp_close_channel(&unshared_w->mProxyControlChannel); } } @@ -262,9 +272,7 @@ void LLProxy::stopSOCKSProxy() */ void LLProxy::setAuthNone() { - LLMutexLock lock(&mProxyMutex); - - mAuthMethodSelected = METHOD_NOAUTH; + Shared_wat(mShared)->mAuthMethodSelected = METHOD_NOAUTH; } /** @@ -288,11 +296,10 @@ bool LLProxy::setAuthPassword(const std::string &username, const std::string &pa return false; } - LLMutexLock lock(&mProxyMutex); - - mAuthMethodSelected = METHOD_PASSWORD; - mSocksUsername = username; - mSocksPassword = password; + Shared_wat shared_w(mShared); + shared_w->mAuthMethodSelected = METHOD_PASSWORD; + shared_w->mSocksUsername = username; + shared_w->mSocksPassword = password; return true; } @@ -314,12 +321,10 @@ bool LLProxy::enableHTTPProxy(LLHost httpHost, LLHttpProxyType type) return false; } - LLMutexLock lock(&mProxyMutex); - - mHTTPProxy = httpHost; - mProxyType = type; - + Shared_wat shared_w(mShared); mHTTPProxyEnabled = true; + shared_w->mHTTPProxy = httpHost; + shared_w->mProxyType = type; return true; } @@ -335,9 +340,8 @@ bool LLProxy::enableHTTPProxy() { bool ok; - LLMutexLock lock(&mProxyMutex); - - ok = (mHTTPProxy.isOk()); + Shared_rat shared_r(mShared); + ok = (shared_r->mHTTPProxy.isOk()); if (ok) { mHTTPProxyEnabled = true; @@ -346,54 +350,6 @@ bool LLProxy::enableHTTPProxy() return ok; } -/** - * @brief Disable the HTTP proxy. - */ -void LLProxy::disableHTTPProxy() -{ - LLMutexLock lock(&mProxyMutex); - - mHTTPProxyEnabled = false; -} - -/** - * @brief Get the currently selected HTTP proxy type - */ -LLHttpProxyType LLProxy::getHTTPProxyType() const -{ - LLMutexLock lock(&mProxyMutex); - return mProxyType; -} - -/** - * @brief Get the SOCKS 5 password. - */ -std::string LLProxy::getSocksPwd() const -{ - LLMutexLock lock(&mProxyMutex); - return mSocksPassword; -} - -/** - * @brief Get the SOCKS 5 username. - */ -std::string LLProxy::getSocksUser() const -{ - LLMutexLock lock(&mProxyMutex); - return mSocksUsername; -} - -/** - * @brief Get the currently selected SOCKS 5 authentication method. - * - * @return Returns either none or password. - */ -LLSocks5AuthType LLProxy::getSelectedAuthMethod() const -{ - LLMutexLock lock(&mProxyMutex); - return mAuthMethodSelected; -} - /** * @brief Stop the LLProxy and make certain that any APR pools and classes are deleted before terminating APR. * @@ -406,47 +362,6 @@ void LLProxy::cleanupClass() deleteSingleton(); } -/** - * @brief Apply proxy settings to a CuRL request if an HTTP proxy is enabled. - * - * This method has been designed to be safe to call from - * any thread in the viewer. This allows requests in the - * texture fetch thread to be aware of the proxy settings. - * When the HTTP proxy is enabled, the proxy mutex will - * be locked every time this method is called. - * - * @param curlEasyRequest_w An already locked curl easy handle, before it has been performed. - */ -void LLProxy::applyProxySettings(AICurlEasyRequest_wat const& curlEasyRequest_w) -{ - // Do a faster unlocked check to see if we are supposed to proxy. - if (mHTTPProxyEnabled) - { - // We think we should proxy, lock the proxy mutex. - LLMutexLock lock(&mProxyMutex); - // Now test again to verify that the proxy wasn't disabled between the first check and the lock. - if (mHTTPProxyEnabled) - { - curlEasyRequest_w->setopt(CURLOPT_PROXY, mHTTPProxy.getIPString().c_str()); - curlEasyRequest_w->setopt(CURLOPT_PROXYPORT, mHTTPProxy.getPort()); - - if (mProxyType == LLPROXY_SOCKS) - { - curlEasyRequest_w->setopt(CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); - if (mAuthMethodSelected == METHOD_PASSWORD) - { - std::string auth_string = mSocksUsername + ":" + mSocksPassword; - curlEasyRequest_w->setopt(CURLOPT_PROXYUSERPWD, auth_string.c_str()); - } - } - else - { - curlEasyRequest_w->setopt(CURLOPT_PROXYTYPE, CURLPROXY_HTTP); - } - } - } -} - /** * @brief Send one TCP packet and receive one in return. * diff --git a/indra/llmessage/llproxy.h b/indra/llmessage/llproxy.h index 94f031fc3..c0d6ab183 100644 --- a/indra/llmessage/llproxy.h +++ b/indra/llmessage/llproxy.h @@ -33,6 +33,7 @@ #include "llmemory.h" #include "llsingleton.h" #include "llthread.h" +#include "aithreadsafe.h" #include // SOCKS error codes returned from the StartProxy method @@ -211,28 +212,87 @@ enum LLSocks5AuthType * To ensure thread safety, all LLProxy members that relate to the HTTP * proxy require the LLProxyMutex to be locked before accessing. */ + +struct ProxyUnshared +{ + /*########################################################################################### + MEMBERS READ AND WRITTEN ONLY IN THE MAIN THREAD. + ###########################################################################################*/ + + // UDP proxy address and port + LLHost mUDPProxy; + + // TCP proxy control channel address and port + LLHost mTCPProxy; + + // socket handle to proxy TCP control channel + LLSocket::ptr_t mProxyControlChannel; + + /*########################################################################################### + END OF UNSHARED MEMBERS + ###########################################################################################*/ +}; + +struct ProxyShared +{ + ProxyShared(void); + + /*########################################################################################### + MEMBERS WRITTEN IN MAIN THREAD AND READ IN ANY THREAD. + ###########################################################################################*/ + + // HTTP proxy address and port + LLHost mHTTPProxy; + + // Currently selected HTTP proxy type. Can be web or socks. + LLHttpProxyType mProxyType; + + // SOCKS 5 selected authentication method. + LLSocks5AuthType mAuthMethodSelected; + + // SOCKS 5 username + std::string mSocksUsername; + // SOCKS 5 password + std::string mSocksPassword; + + /*########################################################################################### + END OF SHARED MEMBERS + ###########################################################################################*/ +}; + class LLProxy: public LLSingleton { LOG_CLASS(LLProxy); + public: + typedef AISTAccessConst Unshared_crat; // Constant Read Access Type for Unshared (cannot be converted to write access). + typedef AISTAccess Unshared_rat; // Read Access Type for Unshared (same as write access type, since we don't lock at all). + typedef AISTAccess Unshared_wat; // Write Access Type, for Unshared. + typedef AIReadAccessConst Shared_crat; // Constant Read Access Type for Shared (cannot be converted to write access). + typedef AIReadAccess Shared_rat; // Read Access Type for Shared. + typedef AIWriteAccess Shared_wat; // Write Access Type for Shared. + /*########################################################################################### - METHODS THAT DO NOT LOCK mProxyMutex! + Public methods that only access variables not shared between threads. ###########################################################################################*/ // Constructor, cannot have parameters due to LLSingleton parent class. Call from main thread only. LLProxy(); - // Static check for enabled status for UDP packets. Call from main thread only. - static bool isSOCKSProxyEnabled() { return sUDPProxyEnabled; } + // Static check for enabled status for UDP packets. Called from main thread only. + static bool isSOCKSProxyEnabled(void) { llassert(is_main_thread()); return sUDPProxyEnabled; } - // Get the UDP proxy address and port. Call from main thread only. - LLHost getUDPProxy() const { return mUDPProxy; } + // Get the UDP proxy address and port. Called from main thread only. + LLHost getUDPProxy(void) const { return Unshared_crat(mUnshared)->mUDPProxy; } /*########################################################################################### - END OF NON-LOCKING METHODS + End of methods that only access variables not shared between threads. ###########################################################################################*/ + // Return true if there is a good chance that the HTTP proxy is currently enabled. + bool HTTPProxyEnabled(void) const { return mHTTPProxyEnabled; } + /*########################################################################################### - METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + Public methods that access variables shared between threads. ###########################################################################################*/ // Destructor, closes open connections. Do not call directly, use cleanupClass(). ~LLProxy(); @@ -263,30 +323,37 @@ public: bool enableHTTPProxy(); // Stop proxying HTTP packets. Call from main thread only. - void disableHTTPProxy(); + // Note that this needs shared_w to be passed because we want the shared members to be locked when this is reset to false. + void disableHTTPProxy(Shared_wat const& shared_w) { mHTTPProxyEnabled = false; } + void disableHTTPProxy(void) { disableHTTPProxy(Shared_wat(mShared)); } + + // Get the currently selected HTTP proxy address and port + LLHost const& getHTTPProxy(Shared_crat const& shared_r) const { return shared_r->mHTTPProxy; } + + // Get the currently selected HTTP proxy type + LLHttpProxyType getHTTPProxyType(Shared_crat const& shared_r) const { return shared_r->mProxyType; } + + // Get the currently selected auth method. + LLSocks5AuthType getSelectedAuthMethod(Shared_crat const& shared_r) const { return shared_r->mAuthMethodSelected; } + + // SOCKS 5 username and password accessors. + std::string getSocksUser(Shared_crat const& shared_r) const { return shared_r->mSocksUsername; } + std::string getSocksPwd(Shared_crat const& shared_r) const { return shared_r->mSocksPassword; } /*########################################################################################### - END OF LOCKING METHODS + End of methods that access variables shared between threads. ###########################################################################################*/ + private: /*########################################################################################### - METHODS THAT LOCK mProxyMutex! DO NOT CALL WHILE mProxyMutex IS LOCKED! + Private methods that access variables shared between threads. ###########################################################################################*/ // Perform a SOCKS 5 authentication and UDP association with the proxy server. S32 proxyHandshake(LLHost proxy); - // Get the currently selected auth method. - LLSocks5AuthType getSelectedAuthMethod() const; - - // Get the currently selected HTTP proxy type - LLHttpProxyType getHTTPProxyType() const; - - std::string getSocksPwd() const; - std::string getSocksUser() const; - /*########################################################################################### - END OF LOCKING METHODS + End of methods that access variables shared between threads. ###########################################################################################*/ private: @@ -294,49 +361,16 @@ private: // Instead use enableHTTPProxy() and disableHTTPProxy() instead. mutable LLAtomic32 mHTTPProxyEnabled; - // Mutex to protect shared members in non-main thread calls to applyProxySettings(). - mutable LLMutex mProxyMutex; - - /*########################################################################################### - MEMBERS READ AND WRITTEN ONLY IN THE MAIN THREAD. DO NOT SHARE! - ###########################################################################################*/ - // Is the UDP proxy enabled? static bool sUDPProxyEnabled; - // UDP proxy address and port - LLHost mUDPProxy; - // TCP proxy control channel address and port - LLHost mTCPProxy; + AIThreadSafeSingleThreadDC mUnshared; + AIThreadSafeDC mShared; - // socket handle to proxy TCP control channel - LLSocket::ptr_t mProxyControlChannel; - - /*########################################################################################### - END OF UNSHARED MEMBERS - ###########################################################################################*/ - - /*########################################################################################### - MEMBERS WRITTEN IN MAIN THREAD AND READ IN ANY THREAD. ONLY READ OR WRITE AFTER LOCKING mProxyMutex! - ###########################################################################################*/ - - // HTTP proxy address and port - LLHost mHTTPProxy; - - // Currently selected HTTP proxy type. Can be web or socks. - LLHttpProxyType mProxyType; - - // SOCKS 5 selected authentication method. - LLSocks5AuthType mAuthMethodSelected; - - // SOCKS 5 username - std::string mSocksUsername; - // SOCKS 5 password - std::string mSocksPassword; - - /*########################################################################################### - END OF SHARED MEMBERS - ###########################################################################################*/ +public: + // For thread-safe read access. Use the _crat access types with these. + AIThreadSafeSingleThreadDC const& unshared_lockobj(void) const { return mUnshared; } + AIThreadSafeDC const& shared_lockobj(void) const { return mShared; } }; #endif From fef461fd13cdaae63066bbc0cf5a4695294d520c Mon Sep 17 00:00:00 2001 From: Lirusaito Date: Mon, 25 Jun 2012 15:18:07 -0400 Subject: [PATCH 03/11] Grabbed openSSL-1.0.0d from upstream for linux, necessary for non-standalone compiles. Also brought in linux64 version I had sitting around, collecting dust. --- indra/newview/viewer_manifest.py | 8 ++++---- install.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index 8f49f94a9..fc9086286 100755 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -801,10 +801,10 @@ class Linux_i686Manifest(LinuxManifest): self.path("libapr-1.so.0") self.path("libaprutil-1.so.0") self.path("libdb-4.2.so") - self.path("libcrypto.so.0.9.7") + self.path("libcrypto.so.1.0.0") self.path("libexpat.so.1") self.path("libhunspell-1.2.so.0.0.0", "libhunspell-1.2.so.0") - self.path("libssl.so.0.9.7") + self.path("libssl.so.1.0.0") #self.path("libuuid.so.1") self.path("libSDL-1.2.so.0") self.path("libELFIO.so") @@ -836,10 +836,10 @@ class Linux_x86_64Manifest(LinuxManifest): self.path("libapr-1.so.0") self.path("libaprutil-1.so.0") self.path("libdb-4.2.so") - self.path("libcrypto.so.0.9.8") + self.path("libcrypto.so.1.0.0") self.path("libexpat.so.1") self.path("libhunspell-1.2.so.0.0.0", "libhunspell-1.2.so.0") - self.path("libssl.so.0.9.8") + self.path("libssl.so.1.0.0") self.path("libuuid.so", "libuuid.so.1") self.path("libSDL-1.2.so.0") self.path("libELFIO.so") diff --git a/install.xml b/install.xml index a56aac3ad..fd128ea38 100644 --- a/install.xml +++ b/install.xml @@ -1069,16 +1069,16 @@ anguage Infrstructure (CLI) international standard linux md5sum - f219ef07b02e2abb9282345c3a8f2b39 + 613856e3880c5898f9629f9f7eb3545c url - http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/openSSL-0.9.7c-linux-20080812.tar.bz2 + https://bitbucket.org/Lirusaito/singularityviewer/downloads/openssl-1.0.0d-linux-20110418.tar.bz2 linux64 md5sum - 00b23f28a2457d9dabbaff0b29ee7323 + 5785bec161f1ad3069fa8330ba42f404 url - http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/openSSL-0.9.8g-linux64-20080909.tar.bz2 + https://bitbucket.org/Lirusaito/singularityviewer/downloads/openSSL-1.0.0d-linux64-for-singularity.tar.bz2 windows From 433c7c3f99deea67f1d09f8b75dd88a2e733f90b Mon Sep 17 00:00:00 2001 From: Lirusaito Date: Thu, 28 Jun 2012 04:08:25 -0400 Subject: [PATCH 04/11] Spelling fixes and stuff like that to AICurl* and llproxy.* documentations Also removes a duplicate include from llares.cpp Conflicts: indra/llmessage/aicurl.cpp --- indra/llmessage/aicurl.cpp | 20 ++++++++++---------- indra/llmessage/aicurl.h | 22 +++++++++++----------- indra/llmessage/aicurlprivate.h | 14 +++++++------- indra/llmessage/aicurlthread.cpp | 32 ++++++++++++++++---------------- indra/llmessage/aicurlthread.h | 4 ++-- indra/llmessage/llares.cpp | 1 - indra/llmessage/llproxy.cpp | 2 +- indra/llmessage/llproxy.h | 2 +- 8 files changed, 48 insertions(+), 49 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 3ab86f56f..9a60ee52d 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -187,7 +187,7 @@ void ssl_init(void) CRYPTO_set_dynlock_destroy_callback(&ssl_dyn_destroy_function); } -// Cleanup OpenSLL library thread-safety. +// Cleanup OpenSSL library thread-safety. void ssl_cleanup(void) { // Dynamic locks callbacks. @@ -844,7 +844,7 @@ void CurlEasyRequest::applyDefaultOptions(void) setoptString(CURLOPT_CAINFO, CertificateAuthority_r->file); // This option forces openssl to use TLS version 1. // The Linden Lab servers don't support later TLS versions, and libopenssl-1.0.1c has - // a bug were renegotion fails (see http://rt.openssl.org/Ticket/Display.html?id=2828), + // a bug where renegotiation fails (see http://rt.openssl.org/Ticket/Display.html?id=2828), // causing the connection to fail completely without this hack. // For a commandline test of the same, observe the difference between: // openssl s_client -connect login.agni.lindenlab.com:443 -CAfile packaged/app_settings/CA.pem -debug @@ -854,12 +854,12 @@ void CurlEasyRequest::applyDefaultOptions(void) setopt(CURLOPT_NOSIGNAL, 1); // The old code did this for the 'buffered' version, but I think it's nonsense. //setopt(CURLOPT_DNS_CACHE_TIMEOUT, 0); - // Set the CURL options for either Socks or HTTP proxy. + // Set the CURL options for either SOCKS or HTTP proxy. applyProxySettings(); Debug( if (dc::curl.is_on()) { - setopt(CURLOPT_VERBOSE, 1); // Usefull for debugging. + setopt(CURLOPT_VERBOSE, 1); // Useful for debugging. setopt(CURLOPT_DEBUGFUNCTION, &curl_debug_callback); setopt(CURLOPT_DEBUGDATA, this); } @@ -874,11 +874,11 @@ void CurlEasyRequest::finalizeRequest(std::string const& url) lldebugs << url << llendl; setopt(CURLOPT_HTTPHEADER, mHeaders); setoptString(CURLOPT_URL, url); - // The following line is a bit tricky: we store a pointer to the object without increasing it's reference count. + // The following line is a bit tricky: we store a pointer to the object without increasing its reference count. // Of course we could increment the reference count, but that doesn't help much: if then this pointer would // get "lost" we'd have a memory leak. Either way we must make sure that it is impossible that this pointer // will be used if the object is deleted [In fact, since this is finalizeRequest() and not addRequest(), - // incrementing the reference counter would be wrong: if addRequest is never called then the object is + // incrementing the reference counter would be wrong: if addRequest() is never called then the object is // destroyed shortly after and this pointer is never even used.] // This pointer is used in MultiHandle::check_run_count, which means that addRequest() was called and // the reference counter was increased and the object is being kept alive, see the comments above @@ -953,7 +953,7 @@ CurlResponderBuffer::CurlResponderBuffer() CurlResponderBuffer::~CurlResponderBuffer() { ThreadSafeBufferedCurlEasyRequest* lockobj = get_lockobj(); - AICurlEasyRequest_wat curl_easy_request_w(*lockobj); // Wait till possible callbacks have returned. + AICurlEasyRequest_wat curl_easy_request_w(*lockobj); // Wait 'til possible callbacks have returned. curl_easy_request_w->send_events_to(NULL); curl_easy_request_w->revokeCallbacks(); if (mResponder) @@ -1042,7 +1042,7 @@ size_t CurlResponderBuffer::curlWriteCallback(char* data, size_t size, size_t nm ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); // We need to lock the curl easy request object too, because that lock is used - // to make sure that callbacks and destruction aren't done simulaneously. + // to make sure that callbacks and destruction aren't done simultaneously. AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); AICurlResponderBuffer_wat buffer_w(*lockobj); @@ -1057,7 +1057,7 @@ size_t CurlResponderBuffer::curlReadCallback(char* data, size_t size, size_t nme ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); // We need to lock the curl easy request object too, because that lock is used - // to make sure that callbacks and destruction aren't done simulaneously. + // to make sure that callbacks and destruction aren't done simultaneously. AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); AICurlResponderBuffer_wat buffer_w(*lockobj); @@ -1078,7 +1078,7 @@ size_t CurlResponderBuffer::curlHeaderCallback(char* data, size_t size, size_t n ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); // We need to lock the curl easy request object too, because that lock is used - // to make sure that callbacks and destruction aren't done simulaneously. + // to make sure that callbacks and destruction aren't done simultaneously. AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); AICurlResponderBuffer_wat buffer_w(*lockobj); diff --git a/indra/llmessage/aicurl.h b/indra/llmessage/aicurl.h index ffff2a01e..a0ab05408 100644 --- a/indra/llmessage/aicurl.h +++ b/indra/llmessage/aicurl.h @@ -125,19 +125,19 @@ void setCAPath(std::string const& file); // The life cycle of classes derived from this class is as follows: // They are allocated with new on the line where get(), getByteRange() or post() is called, // and the pointer to the allocated object is then put in a reference counting ResponderPtr. -// This ResponderPtr is passed to CurlResponderBuffer::prepRequest which stores it in it's -// member mResponder. Hence, the life time of a Responder is never longer than it's -// associated CurlResponderBuffer, however, if everything works correct, then normally a +// This ResponderPtr is passed to CurlResponderBuffer::prepRequest which stores it in its +// member mResponder. Hence, the life time of a Responder is never longer than its +// associated CurlResponderBuffer, however, if everything works correctly, then normally a // responder is deleted in CurlResponderBuffer::removed_from_multi_handle by setting // mReponder to NULL. // -// Note that the life time of CurlResponderBuffer is (a bit) shorter than the associated +// Note that the lifetime of CurlResponderBuffer is (a bit) shorter than the associated // CurlEasyRequest (because of the order of base classes of ThreadSafeBufferedCurlEasyRequest) // and the callbacks, as set by prepRequest, only use those two. // A callback locks the CurlEasyRequest before actually making the callback, and the // destruction of CurlResponderBuffer also first locks the CurlEasyRequest, and then revokes // the callbacks. This assures that a Responder is never used when the objects it uses are -// destructed. Also, if any of those are destructed then the responder is automatically +// destructed. Also, if any of those are destructed then the Responder is automatically // destructed too. // class Responder { @@ -150,8 +150,8 @@ class Responder { std::string mURL; public: - // Called to set the url of the current request for this responder, - // used only when printing debug output regarding activity of the responder. + // Called to set the URL of the current request for this Responder, + // used only when printing debug output regarding activity of the Responder. void setURL(std::string const& url); public: @@ -193,7 +193,7 @@ class Responder { public: // Called from LLSDMessage::ResponderAdapter::listener. - // LLSDMessage::ResponderAdapter is a hack, showing among others by fact that these function needs to be public. + // LLSDMessage::ResponderAdapter is a hack, showing among others by fact that these functions need to be public. void pubErrorWithContent(U32 status, std::string const& reason, LLSD const& content) { errorWithContent(status, reason, content); } void pubResult(LLSD const& content) { result(content); } @@ -246,12 +246,12 @@ struct AICurlEasyHandleEvents { // Therefore we use the following trick: we wrap CurlEasyRequestPtr too, and only allow // read accesses on it. -// AICurlEasyRequest: a thread safe, reference counting, auto cleaning curl easy handle. +// AICurlEasyRequest: a thread safe, reference counting, auto-cleaning curl easy handle. class AICurlEasyRequest { public: // Initial construction is allowed (thread-safe). // Note: If ThreadSafeCurlEasyRequest() throws then the memory allocated is still freed. - // 'new' never returned however and the constructor nor destructor of mCurlEasyRequest is called in this case. + // 'new' never returned however and neither the constructor nor destructor of mCurlEasyRequest is called in this case. // This might throw AICurlNoEasyHandle. AICurlEasyRequest(bool buffered) : mCurlEasyRequest(buffered ? new AICurlPrivate::ThreadSafeBufferedCurlEasyRequest : new AICurlPrivate::ThreadSafeCurlEasyRequest) { } @@ -295,7 +295,7 @@ class AICurlEasyRequest { // If we have a correct (with regard to reference counting) AICurlPrivate::CurlEasyRequestPtr, // then it's OK to construct a AICurlEasyRequest from it. - // Note that the external AICurlPrivate::CurlEasyRequestPtr needs it's own locking, because + // Note that the external AICurlPrivate::CurlEasyRequestPtr needs its own locking, because // it's not thread-safe in itself. AICurlEasyRequest(AICurlPrivate::CurlEasyRequestPtr const& ptr) : mCurlEasyRequest(ptr) { } diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index fcf51e927..c5843176d 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -97,7 +97,7 @@ class CurlEasyHandle : public boost::noncopyable, protected AICurlEasyHandleEven CURLMcode remove_handle_from_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi_handle); public: - // Retuns total number of existing CURL* handles (excluding ones created outside this class). + // Returns total number of existing CURL* handles (excluding ones created outside this class). static U32 getTotalEasyHandles(void) { return sTotalEasyHandles; } // Returns true if this easy handle was added to a curl multi handle. @@ -114,7 +114,7 @@ class CurlEasyHandle : public boost::noncopyable, protected AICurlEasyHandleEven bool operator==(CURL* easy_handle) const { return mEasyHandle == easy_handle; } protected: - // Return the underlaying curl easy handle. + // Return the underlying curl easy handle. CURL* getEasyHandle(void) const { return mEasyHandle; } private: @@ -149,7 +149,7 @@ class CurlEasyRequest : public CurlEasyHandle { void addHeader(char const* str); private: - // Call back stubs. + // Callback stubs. static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* userdata); static size_t writeCallback(char* ptr, size_t size, size_t nmemb, void* userdata); static size_t readCallback(char* ptr, size_t size, size_t nmemb, void* userdata); @@ -217,7 +217,7 @@ class CurlEasyRequest : public CurlEasyHandle { ~CurlEasyRequest(); public: - // Post initialization, set the parent to which to pass the events to. + // Post-initialization, set the parent to pass the events to. void send_events_to(AICurlEasyHandleEvents* target) { mEventsTarget = target; } // For debugging purposes @@ -237,7 +237,7 @@ class CurlEasyRequest : public CurlEasyHandle { // Curl callbacks write into and read from these buffers. // The interface with the rest of the code is through AICurlInterface::Responder. // -// The life time of a CurlResponderBuffer is slightly shorter than it's +// The lifetime of a CurlResponderBuffer is slightly shorter than its // associated CurlEasyRequest; this class can only be created as base class // of ThreadSafeBufferedCurlEasyRequest, and is therefore constructed after // the construction of the associated CurlEasyRequest and destructed before it. @@ -311,8 +311,8 @@ class ThreadSafeCurlEasyRequest : public AIThreadSafeSimple { friend void intrusive_ptr_release(ThreadSafeCurlEasyRequest* p); // Called by boost::intrusive_ptr when a boost::intrusive_ptr is destroyed. }; -// Same as the above but adds a CurlResponderBuffer. The latter has it's own locking in order to -// allow casting the underlaying CurlEasyRequest to ThreadSafeCurlEasyRequest, independent of +// Same as the above but adds a CurlResponderBuffer. The latter has its own locking in order to +// allow casting the underlying CurlEasyRequest to ThreadSafeCurlEasyRequest, independent of // what class it is part of: ThreadSafeCurlEasyRequest or ThreadSafeBufferedCurlEasyRequest. // The virtual destructor of ThreadSafeCurlEasyRequest allows to treat each easy handle transparently // as a ThreadSafeCurlEasyRequest object, or optionally dynamic_cast it to a ThreadSafeBufferedCurlEasyRequest. diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 6b2d1d68d..7e51ee5b5 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -115,9 +115,9 @@ namespace curlthread { //----------------------------------------------------------------------------- // PollSet -// A PollSet can store at least 1024 file descriptors, or FD_SETSIZE if that is larger than 1024 [MAXSIZE]. -// The number of stored file descriptors is mNrFds [0 <= mNrFds <= MAXSIZE]. -// The largest file descriptor is stored is mMaxFd, which is -1 iff mNrFds == 0. +// A PollSet can store at least 1024 filedescriptors, or FD_SETSIZE if that is larger than 1024 [MAXSIZE]. +// The number of stored filedescriptors is mNrFds [0 <= mNrFds <= MAXSIZE]. +// The largest filedescriptor is stored is mMaxFd, which is -1 iff mNrFds == 0. // The file descriptors are stored contiguous in mFileDescriptors[i], with 0 <= i < mNrFds. // File descriptors with the highest priority should be stored first (low index). // @@ -126,7 +126,7 @@ namespace curlthread { // // After a call to refresh(): // -// mFdSet has bits set for at most FD_SETSIZE - 1 file descriptors, copied from mFileDescriptors starting +// mFdSet has bits set for at most FD_SETSIZE - 1 filedescriptors, copied from mFileDescriptors starting // at index mNext (wrapping around to 0). If mNrFds < FD_SETSIZE then mNext is reset to 0 before copying starts. // If mNrFds >= FD_SETSIZE then mNext is set to the next filedescriptor that was not copied (otherwise it is left at 0). // @@ -139,7 +139,7 @@ PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [std::max(1024, FD_S FD_ZERO(&mFdSet); } -// Add file descriptor s to the PollSet. +// Add filedescriptor s to the PollSet. void PollSet::add(curl_socket_t s) { llassert_always(mNrFds < mSize); @@ -147,7 +147,7 @@ void PollSet::add(curl_socket_t s) mMaxFd = std::max(mMaxFd, s); } -// Remove file descriptor s from the PollSet. +// Remove filedescriptor s from the PollSet. void PollSet::remove(curl_socket_t s) { // The number of open filedescriptors is relatively small, @@ -220,8 +220,8 @@ refresh_t PollSet::refresh(void) llassert_always(mNext < mNrFds); - // Test if mNrFds is larger or equal FD_SETSIZE; equal, because we reserve one - // file descriptor for the wakeup fd: we copy maximal FD_SETSIZE - 1 file descriptors. + // Test if mNrFds is larger than or equal FD_SETSIZE; equal, because we reserve one + // filedescriptor for the wakeup fd: we copy maximal FD_SETSIZE - 1 filedescriptors. // If not then we're going to copy everything so that we can save on CPU cycles // by not calculating mMaxFdSet here. if (mNrFds >= FD_SETSIZE) @@ -278,8 +278,8 @@ refresh_t PollSet::refresh(void) return complete_not_empty; } -// FIXME: This needs a rewrite on windows, as FD_ISSET is slow there; it would make -// more sense to iterate directly over the fd's in mFdSet on windows. +// FIXME: This needs a rewrite on Windows, as FD_ISSET is slow there; it would make +// more sense to iterate directly over the fd's in mFdSet on Windows. void PollSet::reset(void) { llassert((mNrFds == 0) == mCopiedFileDescriptors.empty()); @@ -298,8 +298,8 @@ inline int PollSet::get(void) const return (mIter == mCopiedFileDescriptors.end()) ? -1 : *mIter; } -// FIXME: This needs a rewrite on windows, as FD_ISSET is slow there; it would make -// more sense to iterate directly over the fd's in mFdSet on windows. +// FIXME: This needs a rewrite on Windows, as FD_ISSET is slow there; it would make +// more sense to iterate directly over the fd's in mFdSet on Windows. void PollSet::next(void) { llassert(mIter != mCopiedFileDescriptors.end()); // Only call next() if the last call to get() didn't return -1. @@ -476,7 +476,7 @@ AICurlThread::~AICurlThread() void AICurlThread::create_wakeup_fds(void) { #ifdef WINDOWS -// Probably need to use sockets here, cause windows select doesn't work for a pipe. +// Probably need to use sockets here, cause Windows select doesn't work for a pipe. #error Missing implementation #else int pipefd[2]; @@ -626,7 +626,7 @@ void AICurlThread::run(void) int nfds = std::max(max_rfd, max_wfd) + 1; llassert(0 <= nfds && nfds <= FD_SETSIZE); llassert((max_rfd == -1) == (read_fd_set == NULL) && - (max_wfd == -1) == (write_fd_set == NULL)); // Needed on windows. + (max_wfd == -1) == (write_fd_set == NULL)); // Needed on Windows. llassert((max_rfd == -1 || multi_handle_w->mReadPollSet.is_set(max_rfd)) && (max_wfd == -1 || multi_handle_w->mWritePollSet.is_set(max_wfd))); int ready = 0; @@ -971,7 +971,7 @@ void AICurlEasyRequest::addRequest(void) command_being_processed_rat command_being_processed_r(command_being_processed); if (*command_being_processed_r == *this) { - // May not be inbetween being removed from the command queue but not added to the multi session handle yet. + // May not be in-between being removed from the command queue but not added to the multi session handle yet. llassert(command_being_processed_r->command() == cmd_remove); } else @@ -1020,7 +1020,7 @@ void AICurlEasyRequest::removeRequest(void) command_being_processed_rat command_being_processed_r(command_being_processed); if (*command_being_processed_r == *this) { - // May not be inbetween being removed from the command queue but not removed from the multi session handle yet. + // May not be in-between being removed from the command queue but not removed from the multi session handle yet. llassert(command_being_processed_r->command() != cmd_remove); } else diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index 448e53ac0..e8b080b94 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -100,7 +100,7 @@ class PollSet fd_set mFdSet; // Output variable for select(). (Re)initialized by calling refresh(). int mMaxFdSet; // The largest filedescriptor set in mFdSet by refresh(), or -1 when it was empty. - std::vector mCopiedFileDescriptors; // File descriptors copied by refresh to mFdSet. + std::vector mCopiedFileDescriptors; // Filedescriptors copied by refresh to mFdSet. std::vector::iterator mIter; // Index into mCopiedFileDescriptors for next(); loop variable. }; @@ -136,7 +136,7 @@ class MultiHandle : public CurlMultiHandle bool mHandleAddedOrRemoved; // Set when an easy handle was added or removed, reset in check_run_count(). int mPrevRunningHandles; // The last value of mRunningHandles that check_run_count() was called with. int mRunningHandles; // The last value returned by curl_multi_socket_action. - long mTimeOut; // The last time out in ms as set by the call back CURLMOPT_TIMERFUNCTION. + long mTimeOut; // The last time out in ms as set by the callback CURLMOPT_TIMERFUNCTION. private: static int socket_callback(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp); diff --git a/indra/llmessage/llares.cpp b/indra/llmessage/llares.cpp index 3328f2fd5..d83cd6a37 100644 --- a/indra/llmessage/llares.cpp +++ b/indra/llmessage/llares.cpp @@ -28,7 +28,6 @@ #include "linden_common.h" #include "llares.h" -#include "llscopedvolatileaprpool.h" #include #include diff --git a/indra/llmessage/llproxy.cpp b/indra/llmessage/llproxy.cpp index a82ae3cc7..d46c70f04 100644 --- a/indra/llmessage/llproxy.cpp +++ b/indra/llmessage/llproxy.cpp @@ -179,7 +179,7 @@ S32 LLProxy::proxyHandshake(LLHost proxy) } { - // Write acccess type and read access type are really the same, so unshared_w must be simply a reference. + // Write access type and read access type are really the same, so unshared_w must be simply a reference. Unshared_wat& unshared_w = unshared_r; unshared_w->mUDPProxy.setPort(ntohs(connect_reply.port)); // reply port is in network byte order unshared_w->mUDPProxy.setAddress(proxy.getAddress()); diff --git a/indra/llmessage/llproxy.h b/indra/llmessage/llproxy.h index c0d6ab183..00bfa1ad4 100644 --- a/indra/llmessage/llproxy.h +++ b/indra/llmessage/llproxy.h @@ -244,7 +244,7 @@ struct ProxyShared // HTTP proxy address and port LLHost mHTTPProxy; - // Currently selected HTTP proxy type. Can be web or socks. + // Currently selected HTTP proxy type. Can be web or SOCKS. LLHttpProxyType mProxyType; // SOCKS 5 selected authentication method. From 2dee921cd544d6c8efa1655ceb8081c850b8bf97 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 29 Jun 2012 01:33:38 +0200 Subject: [PATCH 05/11] Fix libcurl version check. --- indra/llmessage/aicurl.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 3ab86f56f..6dc43cc2c 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -204,6 +204,10 @@ void ssl_cleanup(void) } // namespace openSSL //----------------------------------------------------------------------------------- +static unsigned int encoded_version(int major, int minor, int patch) +{ + return (major << 16) | (minor << 8) | patch; +} //================================================================================== // External API @@ -283,7 +287,8 @@ void initCurl(F32 curl_request_timeout, S32 max_number_handles) } } - gSetoptParamsNeedDup = (version_info->version_num < 0x071700); + // Before version 7.17.0, strings were not copied. Instead the user was forced keep them available until libcurl no longer needed them. + gSetoptParamsNeedDup = (version_info->version_num < encoded_version(7, 17, 0)); if (gSetoptParamsNeedDup) { llwarns << "Your libcurl version is too old." << llendl; From 90493b65712a769276835691af6549ca90b69e63 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Fri, 29 Jun 2012 05:20:24 +0200 Subject: [PATCH 06/11] Add support for libopenSSL older than version 1.0.0. --- indra/llmessage/aicurl.cpp | 50 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 6dc43cc2c..abba39c45 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -83,6 +83,13 @@ bool gSetoptParamsNeedDup; } // namespace +// See http://www.openssl.org/docs/crypto/threads.html: +// CRYPTO_THREADID and associated functions were introduced in OpenSSL 1.0.0 to replace +// (actually, deprecate) the previous CRYPTO_set_id_callback(), CRYPTO_get_id_callback(), +// and CRYPTO_thread_id() functions which assumed thread IDs to always be represented by +// 'unsigned long'. +#define HAVE_CRYPTO_THREADID (OPENSSL_VERSION_NUMBER >= (1 << 28)) + //----------------------------------------------------------------------------------- // Needed for thread-safe openSSL operation. @@ -115,6 +122,7 @@ void ssl_locking_function(int mode, int n, char const* file, int line) } } +#if HAVE_CRYPTO_THREADID // OpenSSL uniq id function. void ssl_id_function(CRYPTO_THREADID* thread_id) { @@ -124,6 +132,7 @@ void ssl_id_function(CRYPTO_THREADID* thread_id) CRYPTO_THREADID_set_pointer(thread_id, apr_os_thread_current()); #endif } +#endif // HAVE_CRYPTO_THREADID // OpenSSL allocate and initialize dynamic crypto lock. CRYPTO_dynlock_value* ssl_dyn_create_function(char const* file, int line) @@ -157,13 +166,21 @@ void ssl_dyn_lock_function(int mode, CRYPTO_dynlock_value* l, char const* file, } typedef void (*ssl_locking_function_type)(int, int, char const*, int); +#if HAVE_CRYPTO_THREADID typedef void (*ssl_id_function_type)(CRYPTO_THREADID*); +#else +typedef unsigned long (*ulong_thread_id_function_type)(void); +#endif typedef CRYPTO_dynlock_value* (*ssl_dyn_create_function_type)(char const*, int); typedef void (*ssl_dyn_destroy_function_type)(CRYPTO_dynlock_value*, char const*, int); typedef void (*ssl_dyn_lock_function_type)(int, CRYPTO_dynlock_value*, char const*, int); ssl_locking_function_type old_ssl_locking_function; +#if HAVE_CRYPTO_THREADID ssl_id_function_type old_ssl_id_function; +#else +ulong_thread_id_function_type old_ulong_thread_id_function; +#endif ssl_dyn_create_function_type old_ssl_dyn_create_function; ssl_dyn_destroy_function_type old_ssl_dyn_destroy_function; ssl_dyn_lock_function_type old_ssl_dyn_lock_function; @@ -171,13 +188,36 @@ ssl_dyn_lock_function_type old_ssl_dyn_lock_function; // Initialize OpenSSL library for thread-safety. void ssl_init(void) { + // The version identifier format is: MMNNFFPPS: major minor fix patch status. + int const compiled_openSLL_major = (OPENSSL_VERSION_NUMBER >> 28) & 0xff; + int const compiled_openSLL_minor = (OPENSSL_VERSION_NUMBER >> 20) & 0xff; + int const linked_openSLL_major = (SSLeay() >> 28) & 0xff; + int const linked_openSLL_minor = (SSLeay() >> 20) & 0xff; + // Check if dynamically loaded version is compatible with the one we compiled against. + // As off version 1.0.0 also minor versions are compatible. + if (linked_openSLL_major != compiled_openSLL_major || + (compiled_openSLL_major == 0 && linked_openSLL_minor != compiled_openSLL_minor)) + { + llerrs << "The viewer was compiled against " << OPENSSL_VERSION_TEXT << + " but linked against " << SSLeay_version(SSLEAY_VERSION) << + ". Those versions are not compatible." << llendl; + } // Static locks vector. ssl_rwlock_array = new AIRWLock[CRYPTO_num_locks()]; // Static locks callbacks. old_ssl_locking_function = CRYPTO_get_locking_callback(); +#if HAVE_CRYPTO_THREADID old_ssl_id_function = CRYPTO_THREADID_get_callback(); +#else + old_ulong_thread_id_function = CRYPTO_get_id_callback(); +#endif CRYPTO_set_locking_callback(&ssl_locking_function); - CRYPTO_THREADID_set_callback(&ssl_id_function); // Setting this avoids the need for a thread-local error number facility, which is hard to check. + // Setting this avoids the need for a thread-local error number facility, which is hard to check. +#if HAVE_CRYPTO_THREADID + CRYPTO_THREADID_set_callback(&ssl_id_function); +#else + CRYPTO_set_id_callback(&apr_os_thread_current); +#endif // Dynamic locks callbacks. old_ssl_dyn_create_function = CRYPTO_get_dynlock_create_callback(); old_ssl_dyn_lock_function = CRYPTO_get_dynlock_lock_callback(); @@ -185,6 +225,8 @@ void ssl_init(void) CRYPTO_set_dynlock_create_callback(&ssl_dyn_create_function); CRYPTO_set_dynlock_lock_callback(&ssl_dyn_lock_function); CRYPTO_set_dynlock_destroy_callback(&ssl_dyn_destroy_function); + llinfos << "Successful initialization of " << + SSLeay_version(SSLEAY_VERSION) << " (0x" << std::hex << SSLeay() << ")." << llendl; } // Cleanup OpenSLL library thread-safety. @@ -195,7 +237,11 @@ void ssl_cleanup(void) CRYPTO_set_dynlock_lock_callback(old_ssl_dyn_lock_function); CRYPTO_set_dynlock_create_callback(old_ssl_dyn_create_function); // Static locks callbacks. +#if HAVE_CRYPTO_THREADID CRYPTO_THREADID_set_callback(old_ssl_id_function); +#else + CRYPTO_set_id_callback(old_ulong_thread_id_function); +#endif CRYPTO_set_locking_callback(old_ssl_locking_function); // Static locks vector. delete [] ssl_rwlock_array; @@ -246,7 +292,7 @@ void initCurl(F32 curl_request_timeout, S32 max_number_handles) } llinfos << "Successful initialization of libcurl " << - version_info->version << " (" << version_info->version_num << "), (" << + version_info->version << " (0x" << std::hex << version_info->version_num << "), (" << version_info->ssl_version << ", libz/" << version_info->libz_version << ")." << llendl; // Detect SSL library used. From cb5efad026c8d97587e8ce358118c877fc579a39 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sat, 30 Jun 2012 15:41:24 +0200 Subject: [PATCH 07/11] Turn llassert[_always] into a (single) statement and print line nr in decimal. --- indra/llcommon/llerrorlegacy.h | 2 +- indra/llcommon/llqueuedthread.cpp | 2 +- indra/llmessage/llassetstorage.cpp | 4 ++-- indra/newview/llmeshrepository.cpp | 2 +- indra/newview/llspatialpartition.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/indra/llcommon/llerrorlegacy.h b/indra/llcommon/llerrorlegacy.h index fa41c5b3d..278f781ae 100644 --- a/indra/llcommon/llerrorlegacy.h +++ b/indra/llcommon/llerrorlegacy.h @@ -114,7 +114,7 @@ const int LL_ERR_PRICE_MISMATCH = -23018; : liru_slashpos2 == std::string::npos ? std::string(__FILE__)/*Apparently, we're in / or perhaps the top of the drive, print as is*/\ : std::string(__FILE__).substr(1+liru_slashpos2))/*print foo/bar.cpp or perhaps foo\bar.cpp*/ -#define llassert_always(func) if (LL_UNLIKELY(!(func))) llerrs <<"\nASSERT(" #func ")\nfile:"<getExists(uuid, type); @@ -441,7 +441,7 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, LLAssetType::EType type, LL if (user_data) { // The *user_data should not be passed without a callback to clean it up. - llassert(callback != NULL) + llassert(callback != NULL); } if (mShutDown) diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index cd44d3db9..e949460c5 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -680,7 +680,7 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) if (pending != mPendingLOD.end()) { //append this lod request to existing header request pending->second.push_back(lod); - llassert(pending->second.size() <= LLModel::NUM_LODS) + llassert(pending->second.size() <= LLModel::NUM_LODS); } else { //if no header request is pending, fetch header diff --git a/indra/newview/llspatialpartition.cpp b/indra/newview/llspatialpartition.cpp index db0e490cf..f4cde0074 100644 --- a/indra/newview/llspatialpartition.cpp +++ b/indra/newview/llspatialpartition.cpp @@ -2007,7 +2007,7 @@ public: virtual void processGroup(LLSpatialGroup* group) { - llassert(!group->isState(LLSpatialGroup::DIRTY) && !group->getData().empty()) + llassert(!group->isState(LLSpatialGroup::DIRTY) && !group->getData().empty()); if (mRes < 2) { From a803507d67bc7ff3d7f137a85d86017a254ab0de Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Sun, 1 Jul 2012 22:15:03 +0200 Subject: [PATCH 08/11] Use correct way to check if we logged in yet or not. Without this fix, we trigger an assert, in debug mode, that was added to Singularity exactly to find out if we called functions like getExpandedFilename(LL_PATH_PER_SL_ACCOUNT ...) before logging in. Checking if THAT function returns empty() is clearly not safe, but very error prone. --- indra/newview/llviewertexturelist.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indra/newview/llviewertexturelist.cpp b/indra/newview/llviewertexturelist.cpp index 4be8df0e9..d685b951e 100644 --- a/indra/newview/llviewertexturelist.cpp +++ b/indra/newview/llviewertexturelist.cpp @@ -277,7 +277,7 @@ void LLViewerTextureList::shutdown() break; } - if (count > 0 && !gDirUtilp->getExpandedFilename(LL_PATH_PER_SL_ACCOUNT, "").empty()) + if (count > 0 && !gDirUtilp->getLindenUserDir(true).empty()) { std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_PER_SL_ACCOUNT, get_texture_list_name()); llofstream file; From 125a10bb447b8ab693d6c526b296ea95467e1ab9 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 4 Jul 2012 00:10:43 +0200 Subject: [PATCH 09/11] Code hardening, review, bug fixes, documentation, curl stats and cleanup. Bug fixes: AICurlEasyRequestStateMachine didn't delete itself. curl_multi_socket_action calls were made for potentional removed sockets. The curl thread wasn't terminated. --- indra/llmessage/aicurl.cpp | 107 ++++--- indra/llmessage/aicurlprivate.h | 42 ++- indra/llmessage/aicurlthread.cpp | 276 ++++++++++++------ indra/llmessage/aicurlthread.h | 2 +- indra/newview/llxmlrpctransaction.cpp | 14 +- .../aicurleasyrequeststatemachine.cpp | 2 + 6 files changed, 300 insertions(+), 143 deletions(-) diff --git a/indra/llmessage/aicurl.cpp b/indra/llmessage/aicurl.cpp index 5a012c0e7..a2ea765f1 100644 --- a/indra/llmessage/aicurl.cpp +++ b/indra/llmessage/aicurl.cpp @@ -259,10 +259,9 @@ static unsigned int encoded_version(int major, int minor, int patch) // External API // -namespace AICurlInterface { - #undef AICurlPrivate -using AICurlPrivate::check_easy_code; + +namespace AICurlInterface { // MAIN-THREAD void initCurl(F32 curl_request_timeout, S32 max_number_handles) @@ -270,7 +269,7 @@ void initCurl(F32 curl_request_timeout, S32 max_number_handles) DoutEntering(dc::curl, "AICurlInterface::initCurl(" << curl_request_timeout << ", " << max_number_handles << ")"); llassert(LLThread::getRunning() == 0); // We must not call curl_global_init unless we are the only thread. - CURLcode res = check_easy_code(curl_global_init(CURL_GLOBAL_ALL)); + CURLcode res = curl_global_init(CURL_GLOBAL_ALL); if (res != CURLE_OK) { llerrs << "curl_global_init(CURL_GLOBAL_ALL) failed." << llendl; @@ -346,10 +345,12 @@ void initCurl(F32 curl_request_timeout, S32 max_number_handles) // MAIN-THREAD void cleanupCurl(void) { + using AICurlPrivate::stopCurlThread; using AICurlPrivate::curlThreadIsRunning; DoutEntering(dc::curl, "AICurlInterface::cleanupCurl()"); + stopCurlThread(); ssl_cleanup(); llassert(LLThread::getRunning() <= (curlThreadIsRunning() ? 1 : 0)); // We must not call curl_global_cleanup unless we are the only thread left. @@ -495,30 +496,32 @@ void intrusive_ptr_release(Responder* responder) namespace AICurlPrivate { -// THREAD-SAFE -CURLcode check_easy_code(CURLcode code) +//static +LLAtomicU32 Stats::easy_calls; +LLAtomicU32 Stats::easy_errors; +LLAtomicU32 Stats::easy_init_calls; +LLAtomicU32 Stats::easy_init_errors; +LLAtomicU32 Stats::easy_cleanup_calls; +LLAtomicU32 Stats::multi_calls; +LLAtomicU32 Stats::multi_errors; + +//static +void Stats::print(void) { - if (code != CURLE_OK) - { - char* error_buffer = LLThreadLocalData::tldata().mCurlErrorBuffer; - llinfos << "curl easy error detected: " << curl_easy_strerror(code); - if (error_buffer && *error_buffer != '\0') - { - llcont << ": " << error_buffer; - } - llcont << llendl; - } - return code; + llinfos << "====== CURL STATS ======" << llendl; + llinfos << " Curl multi errors/calls: " << std::dec << multi_errors << "/" << multi_calls << llendl; + llinfos << " Curl easy errors/calls: " << std::dec << easy_errors << "/" << easy_calls << llendl; + llinfos << " curl_easy_init() errors/calls: " << std::dec << easy_init_errors << "/" << easy_init_calls << llendl; + llinfos << " Current number of curl easy handles: " << std::dec << (easy_init_calls - easy_init_errors - easy_cleanup_calls) << llendl; + llinfos << "=== END OF CURL STATS ===" << llendl; } // THREAD-SAFE -CURLMcode check_multi_code(CURLMcode code) +void handle_multi_error(CURLMcode code) { - if (code != CURLM_OK) - { - llinfos << "curl multi error detected: " << curl_multi_strerror(code) << llendl; - } - return code; + Stats::multi_errors++; + llinfos << "curl multi error detected: " << curl_multi_strerror(code) << + "; (errors/calls = " << Stats::multi_errors << "/" << Stats::multi_calls << ")" << llendl; } //============================================================================= @@ -528,33 +531,46 @@ CURLMcode check_multi_code(CURLMcode code) //----------------------------------------------------------------------------- // CurlEasyHandle -LLAtomicU32 CurlEasyHandle::sTotalEasyHandles; +// THREAD-SAFE +//static +void CurlEasyHandle::handle_easy_error(CURLcode code) +{ + char* error_buffer = LLThreadLocalData::tldata().mCurlErrorBuffer; + llinfos << "curl easy error detected: " << curl_easy_strerror(code); + if (error_buffer && *error_buffer != '\0') + { + llcont << ": " << error_buffer; + } + Stats::easy_errors++; + llcont << "; (errors/calls = " << Stats::easy_errors << "/" << Stats::easy_calls << ")" << llendl; +} // Throws AICurlNoEasyHandle. CurlEasyHandle::CurlEasyHandle(void) : mActiveMultiHandle(NULL), mErrorBuffer(NULL) { mEasyHandle = curl_easy_init(); #if 0 - //FIXME: for debugging, throw once every 10 times. - static int c = 0; - if (++c % 10 == 5) + // Fake curl_easy_init() failures: throw once every 10 times (for debugging purposes). + static int count = 0; + if (mEasyHandle && (++count % 10) == 5) { curl_easy_cleanup(mEasyHandle); mEasyHandle = NULL; } #endif + Stats::easy_init_calls++; if (!mEasyHandle) { + Stats::easy_init_errors++; throw AICurlNoEasyHandle("curl_easy_init() returned NULL"); } - sTotalEasyHandles++; } CurlEasyHandle::~CurlEasyHandle() { llassert(!mActiveMultiHandle); curl_easy_cleanup(mEasyHandle); - --sTotalEasyHandles; + Stats::easy_cleanup_calls++; } //static @@ -573,8 +589,13 @@ void CurlEasyHandle::setErrorBuffer(void) char* error_buffer = getTLErrorBuffer(); if (mErrorBuffer != error_buffer) { - curl_easy_setopt(mEasyHandle, CURLOPT_ERRORBUFFER, error_buffer); mErrorBuffer = error_buffer; + CURLcode res = curl_easy_setopt(mEasyHandle, CURLOPT_ERRORBUFFER, error_buffer); + if (res != CURLE_OK) + { + llwarns << "curl_easy_setopt(" << (void*)mEasyHandle << "CURLOPT_ERRORBUFFER, " << (void*)error_buffer << ") failed with error " << res << llendl; + mErrorBuffer = NULL; + } } } @@ -727,27 +748,29 @@ void CurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* us setopt(CURLOPT_SSL_CTX_DATA, userdata ? this : NULL); } +#define llmaybewarns lllog(LLApp::isExiting() ? LLError::LEVEL_INFO : LLError::LEVEL_WARN, NULL, NULL, false) + static size_t noHeaderCallback(char* ptr, size_t size, size_t nmemb, void* userdata) { - llwarns << "Calling noHeaderCallback(); curl session aborted." << llendl; + llmaybewarns << "Calling noHeaderCallback(); curl session aborted." << llendl; return 0; // Cause a CURL_WRITE_ERROR } static size_t noWriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) { - llwarns << "Calling noWriteCallback(); curl session aborted." << llendl; + llmaybewarns << "Calling noWriteCallback(); curl session aborted." << llendl; return 0; // Cause a CURL_WRITE_ERROR } static size_t noReadCallback(char* ptr, size_t size, size_t nmemb, void* userdata) { - llwarns << "Calling noReadCallback(); curl session aborted." << llendl; + llmaybewarns << "Calling noReadCallback(); curl session aborted." << llendl; return CURL_READFUNC_ABORT; // Cause a CURLE_ABORTED_BY_CALLBACK } static CURLcode noSSLCtxCallback(CURL* curl, void* sslctx, void* parm) { - llwarns << "Calling noSSLCtxCallback(); curl session aborted." << llendl; + llmaybewarns << "Calling noSSLCtxCallback(); curl session aborted." << llendl; return CURLE_ABORTED_BY_CALLBACK; } @@ -765,7 +788,7 @@ void CurlEasyRequest::revokeCallbacks(void) mWriteCallback = &noWriteCallback; mReadCallback = &noReadCallback; mSSLCtxCallback = &noSSLCtxCallback; - if (active()) + if (active() && !LLApp::isExiting()) { llwarns << "Revoking callbacks on a still active CurlEasyRequest object!" << llendl; } @@ -907,14 +930,17 @@ void CurlEasyRequest::applyDefaultOptions(void) //setopt(CURLOPT_DNS_CACHE_TIMEOUT, 0); // Set the CURL options for either SOCKS or HTTP proxy. applyProxySettings(); +#if 0 + // Cause libcurl to print all it's I/O traffic on the debug channel. Debug( if (dc::curl.is_on()) { - setopt(CURLOPT_VERBOSE, 1); // Useful for debugging. + setopt(CURLOPT_VERBOSE, 1); setopt(CURLOPT_DEBUGFUNCTION, &curl_debug_callback); setopt(CURLOPT_DEBUGDATA, this); } ); +#endif } void CurlEasyRequest::finalizeRequest(std::string const& url) @@ -1204,9 +1230,12 @@ LLAtomicU32 CurlMultiHandle::sTotalMultiHandles; CurlMultiHandle::CurlMultiHandle(void) { + DoutEntering(dc::curl, "CurlMultiHandle::CurlMultiHandle() [" << (void*)this << "]."); mMultiHandle = curl_multi_init(); + Stats::multi_calls++; if (!mMultiHandle) { + Stats::multi_errors++; throw AICurlNoMultiHandle("curl_multi_init() returned NULL"); } sTotalMultiHandles++; @@ -1215,7 +1244,11 @@ CurlMultiHandle::CurlMultiHandle(void) CurlMultiHandle::~CurlMultiHandle() { curl_multi_cleanup(mMultiHandle); - --sTotalMultiHandles; + Stats::multi_calls++; + int total = --sTotalMultiHandles; + Dout(dc::curl, "Called CurlMultiHandle::~CurlMultiHandle() [" << (void*)this << "], " << total << " remaining."); + if (total == 0) + Stats::print(); } } // namespace AICurlPrivate diff --git a/indra/llmessage/aicurlprivate.h b/indra/llmessage/aicurlprivate.h index c5843176d..82d4e9c44 100644 --- a/indra/llmessage/aicurlprivate.h +++ b/indra/llmessage/aicurlprivate.h @@ -32,15 +32,29 @@ #define AICURLPRIVATE_H #include +#include "llatomic.h" namespace AICurlPrivate { namespace curlthread { class MultiHandle; } -CURLcode check_easy_code(CURLcode code); -CURLMcode check_multi_code(CURLMcode code); +struct Stats { + static LLAtomicU32 easy_calls; + static LLAtomicU32 easy_errors; + static LLAtomicU32 easy_init_calls; + static LLAtomicU32 easy_init_errors; + static LLAtomicU32 easy_cleanup_calls; + static LLAtomicU32 multi_calls; + static LLAtomicU32 multi_errors; + + static void print(void); +}; + +void handle_multi_error(CURLMcode code); +inline CURLMcode check_multi_code(CURLMcode code) { Stats::multi_calls++; if (code != CURLM_OK) handle_multi_error(code); return code; } bool curlThreadIsRunning(void); void wakeUpCurlThread(void); +void stopCurlThread(void); class ThreadSafeCurlEasyRequest; class ThreadSafeBufferedCurlEasyRequest; @@ -88,7 +102,6 @@ class CurlEasyHandle : public boost::noncopyable, protected AICurlEasyHandleEven CURL* mEasyHandle; CURLM* mActiveMultiHandle; char* mErrorBuffer; - static LLAtomicU32 sTotalEasyHandles; private: // This should only be called from MultiHandle; add/remove an easy handle to/from a multi handle. @@ -97,22 +110,31 @@ class CurlEasyHandle : public boost::noncopyable, protected AICurlEasyHandleEven CURLMcode remove_handle_from_multi(AICurlEasyRequest_wat& curl_easy_request_w, CURLM* multi_handle); public: - // Returns total number of existing CURL* handles (excluding ones created outside this class). - static U32 getTotalEasyHandles(void) { return sTotalEasyHandles; } - // Returns true if this easy handle was added to a curl multi handle. bool active(void) const { return mActiveMultiHandle; } - // Call this prior to every curl_easy function whose return value is passed to check_easy_code. - void setErrorBuffer(void); - // If there was an error code as result, then this returns a human readable error string. // Only valid when setErrorBuffer was called and the curl_easy function returned an error. - std::string getErrorString(void) const { return mErrorBuffer; } + std::string getErrorString(void) const { return mErrorBuffer ? mErrorBuffer : "(null)"; } // Used for debugging purposes. bool operator==(CURL* easy_handle) const { return mEasyHandle == easy_handle; } + private: + // Call this prior to every curl_easy function whose return value is passed to check_easy_code. + void setErrorBuffer(void); + + static void handle_easy_error(CURLcode code); + + // Always first call setErrorBuffer()! + static inline CURLcode check_easy_code(CURLcode code) + { + Stats::easy_calls++; + if (code != CURLE_OK) + handle_easy_error(code); + return code; + } + protected: // Return the underlying curl easy handle. CURL* getEasyHandle(void) const { return mEasyHandle; } diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 7e51ee5b5..005e4c620 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -132,8 +132,10 @@ namespace curlthread { // // mMaxFdSet is the largest filedescriptor in mFdSet or -1 if it is empty. +static size_t const MAXSIZE = std::max(1024, FD_SETSIZE); + // Create an empty PollSet. -PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [std::max(1024, FD_SETSIZE)]), mSize(std::max(1024, FD_SETSIZE)), +PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [MAXSIZE]), mNrFds(0), mMaxFd(-1), mNext(0), mMaxFdSet(-1) { FD_ZERO(&mFdSet); @@ -142,7 +144,7 @@ PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [std::max(1024, FD_S // Add filedescriptor s to the PollSet. void PollSet::add(curl_socket_t s) { - llassert_always(mNrFds < mSize); + llassert_always(mNrFds < MAXSIZE); mFileDescriptors[mNrFds++] = s; mMaxFd = std::max(mMaxFd, s); } @@ -159,17 +161,57 @@ void PollSet::remove(curl_socket_t s) // much clock cycles. Therefore, shift the whole vector // back, keeping it compact and keeping the filedescriptors // in the same order (which is supposedly their priority). + // + // The general case is where mFileDescriptors contains s at an index + // between 0 and mNrFds: + // mNrFds = 6 + // v + // index: 0 1 2 3 4 5 + // a b c s d e + + // This function should never be called unless s is actually in mFileDescriptors, + // as a result of a previous call to PollSet::add(). llassert(mNrFds > 0); + + // Correct mNrFds for when the descriptor is removed. + // Make i 'point' to the last entry. int i = --mNrFds; - int prev = mFileDescriptors[i]; + // i = NrFds = 5 + // v + // index: 0 1 2 3 4 5 + // a b c s d e + int prev = mFileDescriptors[i]; // prev = 'e' int max = -1; for (--i; i >= 0 && prev != s; --i) { - int cur = mFileDescriptors[i]; - mFileDescriptors[i] = prev; - max = std::max(max, prev); - prev = cur; + int cur = mFileDescriptors[i]; // cur = 'd' + mFileDescriptors[i] = prev; // Overwrite 'd' with 'e'. + max = std::max(max, prev); // max is the maximum value in 'i' or higher. + prev = cur; // prev = 'd' + // i NrFds = 5 + // v v + // index: 0 1 2 3 4 + // a b c s e // prev = 'd' + // + // Next loop iteration: cur = 's', overwrite 's' with 'd', prev = 's'; loop terminates. + // i NrFds = 5 + // v v + // index: 0 1 2 3 4 + // a b c d e // prev = 's' } + llassert(prev == s); + // At this point i was decremented once more and points to the element before the old s. + // i NrFds = 5 + // v v + // index: 0 1 2 3 4 + // a b c d e // max = std::max('d', 'e') + + // If mNext pointed to an element before s, it should be left alone. Otherwise, if mNext pointed + // to s it must now point to 'd', or if it pointed beyond 's' it must be decremented by 1. + if (mNext > i + 1) // i + 1 is where s was. + --mNext; + + // If s was the largest file descriptor, we have to update mMaxFd. if (s == mMaxFd) { while (i >= 0) @@ -182,8 +224,10 @@ void PollSet::remove(curl_socket_t s) llassert(mMaxFd < s); llassert((mMaxFd == -1) == (mNrFds == 0)); } - if (mNext == mNrFds) - mNext = 0; + + // ALSO make sure that s is no longer set in mFdSet, or we might confuse libcurl by + // calling curl_multi_socket_action for a socket that it told us to remove. + clr(s); } bool PollSet::contains(curl_socket_t fd) const @@ -220,7 +264,7 @@ refresh_t PollSet::refresh(void) llassert_always(mNext < mNrFds); - // Test if mNrFds is larger than or equal FD_SETSIZE; equal, because we reserve one + // Test if mNrFds is larger than or equal to FD_SETSIZE; equal, because we reserve one // filedescriptor for the wakeup fd: we copy maximal FD_SETSIZE - 1 filedescriptors. // If not then we're going to copy everything so that we can save on CPU cycles // by not calculating mMaxFdSet here. @@ -228,22 +272,9 @@ refresh_t PollSet::refresh(void) { llwarns << "PollSet::reset: More than FD_SETSIZE (" << FD_SETSIZE << ") file descriptors active!" << llendl; // Calculate mMaxFdSet. - int max = -1; - int count = 0; - int end = mNrFds; - int i = mNext; - while (++count < FD_SETSIZE) - { - max = std::max(max, mFileDescriptors[i]); - if (++i == end) - { - if (end == mNext) - break; - end = mNext; - i = 0; - llassert(i < end); // If mNext == 0 then the while loop terminates before we get here. - } - } + // Run over FD_SETSIZE - 1 elements, starting at mNext, wrapping to 0 when we reach the end. + int max = -1, i = mNext, count = 0; + while (++count < FD_SETSIZE) { max = std::max(max, mFileDescriptors[i]); if (++i == mNrFds) i = 0; } mMaxFdSet = max; } else @@ -252,7 +283,6 @@ refresh_t PollSet::refresh(void) mMaxFdSet = mMaxFd; } int count = 0; - int end = mNrFds; int i = mNext; for(;;) { @@ -263,15 +293,13 @@ refresh_t PollSet::refresh(void) } FD_SET(mFileDescriptors[i], &mFdSet); mCopiedFileDescriptors.push_back(mFileDescriptors[i]); - if (++i == end) + if (++i == mNrFds) { + // If we reached the end and start at the beginning, then we copied everything. if (mNext == 0) break; - // If this was true then we got here a second time, which means that we accessed all - // filedescriptors with mNext != 0, but count is still < FD_SETSIZE, which is not - // possible because mNext is only non-zero when mNrFds >= FD_SETSIZE. - llassert(end != mNext); - end = mNext; + // When can only come here if mNrFds >= FD_SETSIZE, hence we can just + // wrap around and terminate on count reaching FD_SETSIZE. i = 0; } } @@ -280,9 +308,29 @@ refresh_t PollSet::refresh(void) // FIXME: This needs a rewrite on Windows, as FD_ISSET is slow there; it would make // more sense to iterate directly over the fd's in mFdSet on Windows. +// +// The API reset(), get() and next() allows one to run over all filedescriptors +// in mFdSet that are set. This works by running only over the filedescriptors +// that were set initially (by the call to refresh()) and then checking if that +// filedescriptor is (still) set in mFdSet. +// +// A call to reset() resets mIter to the beginning, so that get() returns +// the first filedescriptor that is still set. A call to next() progresses +// the iterator to the next set filedescriptor. If get() return -1, then there +// were no more filedescriptors set. +// +// Note that one should never call next() unless get() didn't return -1, so +// the call sequence is: +// refresh(); +// /* reset some or all bits in mFdSet */ +// reset(); +// while (get() != -1) // next(); +// +// Note also that this API is only used by MergeIterator, which wraps it +// and provides a different API to use. + void PollSet::reset(void) { - llassert((mNrFds == 0) == mCopiedFileDescriptors.empty()); if (mCopiedFileDescriptors.empty()) mIter = mCopiedFileDescriptors.end(); else @@ -298,8 +346,6 @@ inline int PollSet::get(void) const return (mIter == mCopiedFileDescriptors.end()) ? -1 : *mIter; } -// FIXME: This needs a rewrite on Windows, as FD_ISSET is slow there; it would make -// more sense to iterate directly over the fd's in mFdSet on Windows. void PollSet::next(void) { llassert(mIter != mCopiedFileDescriptors.end()); // Only call next() if the last call to get() didn't return -1. @@ -308,6 +354,10 @@ void PollSet::next(void) //----------------------------------------------------------------------------- // MergeIterator +// +// This class takes two PollSet's and allows one to run over all filedescriptors +// that are set in one or both poll sets, returning each filedescriptor only +// once, by calling next() until it returns false. class MergeIterator { @@ -343,7 +393,6 @@ bool MergeIterator::next(int& fd_out, int& ev_bitmask_out) fd_out = rfd; ev_bitmask_out = CURL_CSELECT_IN | CURL_CSELECT_OUT; mReadPollSet.next(); - } else if ((unsigned int)rfd < (unsigned int)wfd) // Use and increment smaller one, unless it's -1. { @@ -440,6 +489,9 @@ class AICurlThread : public LLThread // MAIN-THREAD void wakeup_thread(void); + // MAIN-THREAD + void stop_thread(void) { mRunning = false; wakeup_thread(); } + protected: virtual void run(void); void wakeup(AICurlMultiHandle_wat const& multi_handle_w); @@ -453,13 +505,15 @@ class AICurlThread : public LLThread int mWakeUpFd; int mZeroTimeOut; + + volatile bool mRunning; }; // Only the main thread is accessing this. AICurlThread* AICurlThread::sInstance = NULL; // MAIN-THREAD -AICurlThread::AICurlThread(void) : LLThread("AICurlThread"), mWakeUpFd_in(-1), mWakeUpFd(-1), mZeroTimeOut(0) +AICurlThread::AICurlThread(void) : LLThread("AICurlThread"), mWakeUpFd_in(-1), mWakeUpFd(-1), mZeroTimeOut(0), mRunning(true) { create_wakeup_fds(); sInstance = this; @@ -567,7 +621,7 @@ void AICurlThread::wakeup(AICurlMultiHandle_wat const& multi_handle_w) llwarns << "read(3) from mWakeUpFd returned 0, indicating that the pipe on the other end was closed! Shutting down curl thread." << llendl; close(mWakeUpFd); mWakeUpFd = -1; - shutdown(); + mRunning = false; return; } #endif @@ -610,30 +664,29 @@ void AICurlThread::run(void) { DoutEntering(dc::curl, "AICurlThread::run()"); - AICurlMultiHandle_wat multi_handle_w(AICurlMultiHandle::getInstance()); - for(;;) { - refresh_t rres = multi_handle_w->mReadPollSet.refresh(); - refresh_t wres = multi_handle_w->mWritePollSet.refresh(); - fd_set* read_fd_set = multi_handle_w->mReadPollSet.access(); - if (LL_LIKELY(mWakeUpFd != -1)) - FD_SET(mWakeUpFd, read_fd_set); - else if ((rres & empty)) - read_fd_set = NULL; - fd_set* write_fd_set = ((wres & empty)) ? NULL : multi_handle_w->mWritePollSet.access(); - int const max_rfd = std::max(multi_handle_w->mReadPollSet.get_max_fd(), mWakeUpFd); - int const max_wfd = multi_handle_w->mWritePollSet.get_max_fd(); - int nfds = std::max(max_rfd, max_wfd) + 1; - llassert(0 <= nfds && nfds <= FD_SETSIZE); - llassert((max_rfd == -1) == (read_fd_set == NULL) && - (max_wfd == -1) == (write_fd_set == NULL)); // Needed on Windows. - llassert((max_rfd == -1 || multi_handle_w->mReadPollSet.is_set(max_rfd)) && - (max_wfd == -1 || multi_handle_w->mWritePollSet.is_set(max_wfd))); - int ready = 0; - if (LL_UNLIKELY(nfds == 0)) // Only happens when the thread is shutting down. - ms_sleep(1000); - else + AICurlMultiHandle_wat multi_handle_w(AICurlMultiHandle::getInstance()); + while(mRunning) { + // If mRunning is true then we can only get here if mWakeUpFd != -1. + llassert(mWakeUpFd != -1); + // Copy the next batch of file descriptors from the PollSets mFiledescriptors into their mFdSet. + multi_handle_w->mReadPollSet.refresh(); + refresh_t wres = multi_handle_w->mWritePollSet.refresh(); + // Add wake up fd if any, and pass NULL to select() if a set is empty. + fd_set* read_fd_set = multi_handle_w->mReadPollSet.access(); + FD_SET(mWakeUpFd, read_fd_set); + fd_set* write_fd_set = ((wres & empty)) ? NULL : multi_handle_w->mWritePollSet.access(); + // Calculate nfds (ignored on windows). + int const max_rfd = std::max(multi_handle_w->mReadPollSet.get_max_fd(), mWakeUpFd); + int const max_wfd = multi_handle_w->mWritePollSet.get_max_fd(); + int nfds = std::max(max_rfd, max_wfd) + 1; + llassert(0 <= nfds && nfds <= FD_SETSIZE); + llassert((max_rfd == -1) == (read_fd_set == NULL) && + (max_wfd == -1) == (write_fd_set == NULL)); // Needed on Windows. + llassert((max_rfd == -1 || multi_handle_w->mReadPollSet.is_set(max_rfd)) && + (max_wfd == -1 || multi_handle_w->mWritePollSet.is_set(max_wfd))); + int ready = 0; struct timeval timeout; long timeout_ms = multi_handle_w->getTimeOut(); // If no timeout is set, sleep 1 second. @@ -695,37 +748,41 @@ void AICurlThread::run(void) if (ready == -1) last_errno = errno; #endif - } - // Select returns the total number of bits set in each of the fd_set's (upon return), - // or -1 when an error occurred. A value of 0 means that a timeout occurred. - if (ready == -1) - { - llwarns << "select() failed: " << errno << ", " << strerror(errno) << llendl; - continue; - } - else if (ready == 0) - { - multi_handle_w->socket_action(CURL_SOCKET_TIMEOUT, 0); - } - else - { - if (multi_handle_w->mReadPollSet.is_set(mWakeUpFd)) + // Select returns the total number of bits set in each of the fd_set's (upon return), + // or -1 when an error occurred. A value of 0 means that a timeout occurred. + if (ready == -1) { - wakeup(multi_handle_w); - --ready; + llwarns << "select() failed: " << errno << ", " << strerror(errno) << llendl; + continue; } - MergeIterator iter(multi_handle_w->mReadPollSet, multi_handle_w->mWritePollSet); - int fd, ev_bitmask; - while (ready > 0 && iter.next(fd, ev_bitmask)) + else if (ready == 0) { - ready -= (ev_bitmask == (CURL_CSELECT_IN|CURL_CSELECT_OUT)) ? 2 : 1; - multi_handle_w->socket_action(fd, ev_bitmask); - llassert(ready >= 0); + multi_handle_w->socket_action(CURL_SOCKET_TIMEOUT, 0); } - llassert(ready == 0); + else + { + if (multi_handle_w->mReadPollSet.is_set(mWakeUpFd)) + { + // Process commands from main-thread. This can add or remove filedescriptors from the poll sets. + wakeup(multi_handle_w); + --ready; + } + // Handle all active filedescriptors. + MergeIterator iter(multi_handle_w->mReadPollSet, multi_handle_w->mWritePollSet); + int fd, ev_bitmask; + while (ready > 0 && iter.next(fd, ev_bitmask)) + { + ready -= (ev_bitmask == (CURL_CSELECT_IN|CURL_CSELECT_OUT)) ? 2 : 1; + multi_handle_w->socket_action(fd, ev_bitmask); + llassert(ready >= 0); + } + // Should have handled them all. + llassert(ready == 0); + } + multi_handle_w->check_run_count(); } - multi_handle_w->check_run_count(); } + AICurlMultiHandle::destroyInstance(); } //----------------------------------------------------------------------------- @@ -733,10 +790,10 @@ void AICurlThread::run(void) MultiHandle::MultiHandle(void) : mHandleAddedOrRemoved(false), mPrevRunningHandles(0), mRunningHandles(0), mTimeOut(-1) { - curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETFUNCTION, &MultiHandle::socket_callback); - curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETDATA, this); - curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERFUNCTION, &MultiHandle::timer_callback); - curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERDATA, this); + check_multi_code(curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETFUNCTION, &MultiHandle::socket_callback)); + check_multi_code(curl_multi_setopt(mMultiHandle, CURLMOPT_SOCKETDATA, this)); + check_multi_code(curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERFUNCTION, &MultiHandle::timer_callback)); + check_multi_code(curl_multi_setopt(mMultiHandle, CURLMOPT_TIMERDATA, this)); } MultiHandle::~MultiHandle() @@ -818,7 +875,13 @@ CURLMcode MultiHandle::assign(curl_socket_t sockfd, void* sockptr) CURLMsg const* MultiHandle::info_read(int* msgs_in_queue) const { - return curl_multi_info_read(mMultiHandle, msgs_in_queue); + CURLMsg const* ret = curl_multi_info_read(mMultiHandle, msgs_in_queue); + // NULL could be an error, but normally it isn't, so don't print anything and + // never increment Stats::multi_errors. However, lets just increment multi_calls + // when it certainly wasn't an error... + if (ret) + Stats::multi_calls++; + return ret; } CURLMcode MultiHandle::add_easy_request(AICurlEasyRequest const& easy_request) @@ -863,7 +926,8 @@ void MultiHandle::check_run_count(void) { CURL* easy = msg->easy_handle; ThreadSafeCurlEasyRequest* ptr; - curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ptr); + CURLcode rese = curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ptr); + llassert_always(rese == CURLE_OK); AICurlEasyRequest easy_request(ptr); llassert(*AICurlEasyRequest_wat(*easy_request) == easy); // Store the result and transfer info in the easy handle. @@ -904,6 +968,15 @@ void MultiHandle::check_run_count(void) } // namespace curlthread } // namespace AICurlPrivate +//static +void AICurlMultiHandle::destroyInstance(void) +{ + LLThreadLocalData& tldata = LLThreadLocalData::tldata(); + Dout(dc::curl, "Destroying AICurlMultiHandle [" << (void*)tldata.mCurlMultiHandle << "] for thread \"" << tldata.mName << "\"."); + delete tldata.mCurlMultiHandle; + tldata.mCurlMultiHandle = NULL; +} + //============================================================================= // MAIN-THREAD (needing to access the above declarations). @@ -913,8 +986,8 @@ AICurlMultiHandle& AICurlMultiHandle::getInstance(void) LLThreadLocalData& tldata = LLThreadLocalData::tldata(); if (!tldata.mCurlMultiHandle) { - llinfos << "Creating AICurlMultiHandle for thread \"" << tldata.mName << "\"." << llendl; tldata.mCurlMultiHandle = new AICurlMultiHandle; + Dout(dc::curl, "Created AICurlMultiHandle [" << (void*)tldata.mCurlMultiHandle << "] for thread \"" << tldata.mName << "\"."); } return *static_cast(tldata.mCurlMultiHandle); } @@ -934,6 +1007,21 @@ void wakeUpCurlThread(void) AICurlThread::sInstance->wakeup_thread(); } +void stopCurlThread(void) +{ + using curlthread::AICurlThread; + if (AICurlThread::sInstance) + { + AICurlThread::sInstance->stop_thread(); + int count = 101; + while(--count && !AICurlThread::sInstance->isStopped()) + { + ms_sleep(10); + } + Dout(dc::curl, "Curl thread" << (curlThreadIsRunning() ? " not" : "") << " stopped after " << ((100 - count) * 10) << "ms."); + } +} + } // namespace AICurlPrivate //----------------------------------------------------------------------------- @@ -994,7 +1082,7 @@ void AICurlEasyRequest::removeRequest(void) { // Write-lock the command queue. - command_queue_wat command_queue_w(command_queue); + command_queue_wat command_queue_w(command_queue); #ifdef SHOW_ASSERT // This debug code checks if we aren't calling removeRequest() twice for the same object. // That means that the thread calling this function already finished it, following from that diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index e8b080b94..a2262d9a8 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -92,7 +92,6 @@ class PollSet private: curl_socket_t* mFileDescriptors; - size_t mSize; // Size of mFileDescriptors array. int mNrFds; // The number of filedescriptors in the array. int mMaxFd; // The largest filedescriptor in the array, or -1 when it is empty. int mNext; // The index of the first file descriptor to start copying, the next call to refresh(). @@ -172,6 +171,7 @@ class MultiHandle : public CurlMultiHandle class AICurlMultiHandle : public AIThreadSafeSingleThreadDC, public LLThreadLocalDataMember { public: static AICurlMultiHandle& getInstance(void); + static void destroyInstance(void); private: // Use getInstance(). AICurlMultiHandle(void) { } diff --git a/indra/newview/llxmlrpctransaction.cpp b/indra/newview/llxmlrpctransaction.cpp index 5abe2df2e..72ccfb936 100644 --- a/indra/newview/llxmlrpctransaction.cpp +++ b/indra/newview/llxmlrpctransaction.cpp @@ -278,6 +278,12 @@ void LLXMLRPCTransaction::Impl::init(XMLRPC_REQUEST request, bool useGzip) mCurlEasyRequestStateMachinePtr->run(boost::bind(&LLXMLRPCTransaction::Impl::curlEasyRequestCallback, this, _1)); setStatus(LLXMLRPCTransaction::StatusStarted); } + else + { + // This deletes the statemachine immediately. + mCurlEasyRequestStateMachinePtr->kill(); + mCurlEasyRequestStateMachinePtr = NULL; + } } LLXMLRPCTransaction::Impl::~Impl() @@ -311,13 +317,19 @@ void LLXMLRPCTransaction::Impl::curlEasyRequestCallback(bool success) { llassert(mStatus == LLXMLRPCTransaction::StatusStarted || mStatus == LLXMLRPCTransaction::StatusDownloading); + AICurlEasyRequestStateMachine* state_machine = mCurlEasyRequestStateMachinePtr; + // We're done with the statemachine, one way or another. + // Set mCurlEasyRequestStateMachinePtr to NULL so we won't call mCurlEasyRequestStateMachinePtr->running() in the destructor. + // Note that the state machine auto-cleaning: it will be deleted by the main-thread after this function returns. + mCurlEasyRequestStateMachinePtr = NULL; + if (!success) { setStatus(LLXMLRPCTransaction::StatusOtherError, "Statemachine failed"); return; } - AICurlEasyRequest_wat curlEasyRequest_w(*mCurlEasyRequestStateMachinePtr->mCurlEasyRequest); + AICurlEasyRequest_wat curlEasyRequest_w(*state_machine->mCurlEasyRequest); CURLcode result; curlEasyRequest_w->getResult(&result, &mTransferInfo); diff --git a/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp b/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp index a308be093..752fe0827 100644 --- a/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp +++ b/indra/newview/statemachine/aicurleasyrequeststatemachine.cpp @@ -138,6 +138,8 @@ void AICurlEasyRequestStateMachine::finish_impl(void) curl_easy_request_w->send_events_to(NULL); curl_easy_request_w->revokeCallbacks(); } + // Auto clean up. + kill(); } AICurlEasyRequestStateMachine::AICurlEasyRequestStateMachine(bool buffered) : mBuffered(buffered), mCurlEasyRequest(buffered) From 07e7eeedd116e8020492fc303e4a1e0bc1562c18 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 4 Jul 2012 07:32:24 +0200 Subject: [PATCH 10/11] Added some windows code. Iterating directly over the elements of fd_set::fd_array in windows is faster than using FD_ISSET. --- indra/llmessage/aicurlthread.cpp | 144 ++++++++++++++++++++++--------- indra/llmessage/aicurlthread.h | 26 +++--- 2 files changed, 117 insertions(+), 53 deletions(-) diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index 005e4c620..e7b62947d 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -136,7 +136,10 @@ static size_t const MAXSIZE = std::max(1024, FD_SETSIZE); // Create an empty PollSet. PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [MAXSIZE]), - mNrFds(0), mMaxFd(-1), mNext(0), mMaxFdSet(-1) + mNrFds(0), mNext(0) +#if !LL_WINDOWS + , mMaxFd(-1), mMaxFdSet(-1) +#endif { FD_ZERO(&mFdSet); } @@ -146,7 +149,9 @@ void PollSet::add(curl_socket_t s) { llassert_always(mNrFds < MAXSIZE); mFileDescriptors[mNrFds++] = s; +#if !LL_WINDOWS mMaxFd = std::max(mMaxFd, s); +#endif } // Remove filedescriptor s from the PollSet. @@ -180,54 +185,82 @@ void PollSet::remove(curl_socket_t s) // v // index: 0 1 2 3 4 5 // a b c s d e - int prev = mFileDescriptors[i]; // prev = 'e' - int max = -1; - for (--i; i >= 0 && prev != s; --i) + curl_socket_t cur = mFileDescriptors[i]; // cur = 'e' +#if !LL_WINDOWS + curl_socket_t max = -1; +#endif + while (cur != s) { - int cur = mFileDescriptors[i]; // cur = 'd' - mFileDescriptors[i] = prev; // Overwrite 'd' with 'e'. - max = std::max(max, prev); // max is the maximum value in 'i' or higher. - prev = cur; // prev = 'd' + llassert(i > 0); + curl_socket_t next = mFileDescriptors[--i]; // next = 'd' + mFileDescriptors[i] = cur; // Overwrite 'd' with 'e'. +#if !LL_WINDOWS + max = std::max(max, cur); // max is the maximum value in 'i' or higher. +#endif + cur = next; // cur = 'd' // i NrFds = 5 // v v // index: 0 1 2 3 4 - // a b c s e // prev = 'd' + // a b c s e // cur = 'd' // - // Next loop iteration: cur = 's', overwrite 's' with 'd', prev = 's'; loop terminates. + // Next loop iteration: next = 's', overwrite 's' with 'd', cur = 's'; loop terminates. // i NrFds = 5 // v v // index: 0 1 2 3 4 - // a b c d e // prev = 's' + // a b c d e // cur = 's' } - llassert(prev == s); + llassert(cur == s); // At this point i was decremented once more and points to the element before the old s. // i NrFds = 5 // v v // index: 0 1 2 3 4 - // a b c d e // max = std::max('d', 'e') + // a b c d e // max = std::max('d', 'e') // If mNext pointed to an element before s, it should be left alone. Otherwise, if mNext pointed // to s it must now point to 'd', or if it pointed beyond 's' it must be decremented by 1. - if (mNext > i + 1) // i + 1 is where s was. + if (mNext > i) // i is where s was. --mNext; +#if !LL_WINDOWS // If s was the largest file descriptor, we have to update mMaxFd. if (s == mMaxFd) { - while (i >= 0) + while (i > 0) { - int cur = mFileDescriptors[i]; - max = std::max(max, cur); - --i; + curl_socket_t next = mFileDescriptors[--i]; + max = std::max(max, next); } mMaxFd = max; llassert(mMaxFd < s); llassert((mMaxFd == -1) == (mNrFds == 0)); } +#endif // ALSO make sure that s is no longer set in mFdSet, or we might confuse libcurl by // calling curl_multi_socket_action for a socket that it told us to remove. +#if !LL_WINDOWS clr(s); +#else + // We have to use a custom implementation here, because we don't want to invalidate mIter. + // This is the same algorithm as above, but with mFdSet.fd_count instead of mNrFds, + // mFdSet.fd_array instead of mFileDescriptors and mIter instead of mNext. + if (FD_ISSET(s, &mFdSet)) + { + int i = --mFdSet.fd_count; + llassert(i >= 0); + curl_socket_t cur = mFdSet.fd_array[i]; + while (cur != s) + { + llassert(i > 0); + curl_socket_t next = mFileDescriptors[--i]; + mFileDescriptors[i] = cur; + cur = next; + } + if (mIter > i) + --mIter; + llassert(mIter <= mFdSet.fd_count); + } +#endif } bool PollSet::contains(curl_socket_t fd) const @@ -254,11 +287,15 @@ inline void PollSet::clr(curl_socket_t fd) refresh_t PollSet::refresh(void) { FD_ZERO(&mFdSet); +#if !LL_WINDOWS mCopiedFileDescriptors.clear(); +#endif if (mNrFds == 0) { +#if !LL_WINDOWS mMaxFdSet = -1; +#endif return empty_and_complete; } @@ -271,16 +308,20 @@ refresh_t PollSet::refresh(void) if (mNrFds >= FD_SETSIZE) { llwarns << "PollSet::reset: More than FD_SETSIZE (" << FD_SETSIZE << ") file descriptors active!" << llendl; +#if !LL_WINDOWS // Calculate mMaxFdSet. // Run over FD_SETSIZE - 1 elements, starting at mNext, wrapping to 0 when we reach the end. int max = -1, i = mNext, count = 0; while (++count < FD_SETSIZE) { max = std::max(max, mFileDescriptors[i]); if (++i == mNrFds) i = 0; } mMaxFdSet = max; +#endif } else { mNext = 0; // Start at the beginning if we copy everything anyway. +#if !LL_WINDOWS mMaxFdSet = mMaxFd; +#endif } int count = 0; int i = mNext; @@ -292,7 +333,9 @@ refresh_t PollSet::refresh(void) return not_complete_not_empty; } FD_SET(mFileDescriptors[i], &mFdSet); +#if !LL_WINDOWS mCopiedFileDescriptors.push_back(mFileDescriptors[i]); +#endif if (++i == mNrFds) { // If we reached the end and start at the beginning, then we copied everything. @@ -306,9 +349,6 @@ refresh_t PollSet::refresh(void) return complete_not_empty; } -// FIXME: This needs a rewrite on Windows, as FD_ISSET is slow there; it would make -// more sense to iterate directly over the fd's in mFdSet on Windows. -// // The API reset(), get() and next() allows one to run over all filedescriptors // in mFdSet that are set. This works by running only over the filedescriptors // that were set initially (by the call to refresh()) and then checking if that @@ -324,13 +364,16 @@ refresh_t PollSet::refresh(void) // refresh(); // /* reset some or all bits in mFdSet */ // reset(); -// while (get() != -1) // next(); +// while (get() != CURL_SOCKET_BAD) // next(); // // Note also that this API is only used by MergeIterator, which wraps it // and provides a different API to use. void PollSet::reset(void) { +#if LL_WINDOWS + mIter = 0; +#else if (mCopiedFileDescriptors.empty()) mIter = mCopiedFileDescriptors.end(); else @@ -339,17 +382,27 @@ void PollSet::reset(void) if (!FD_ISSET(*mIter, &mFdSet)) next(); } +#endif } -inline int PollSet::get(void) const +inline curl_socket_t PollSet::get(void) const { - return (mIter == mCopiedFileDescriptors.end()) ? -1 : *mIter; +#if LL_WINDOWS + return (mIter >= mFdSet.fd_count) ? CURL_SOCKET_BAD : mFdSet.fd_array[mIter]; +#else + return (mIter == mCopiedFileDescriptors.end()) ? CURL_SOCKET_BAD : *mIter; +#endif } void PollSet::next(void) { +#if LL_WINDOWS + llassert(mIter < mFdSet.fd_count); + ++mIter; +#else llassert(mIter != mCopiedFileDescriptors.end()); // Only call next() if the last call to get() didn't return -1. while (++mIter != mCopiedFileDescriptors.end() && !FD_ISSET(*mIter, &mFdSet)); +#endif } //----------------------------------------------------------------------------- @@ -364,7 +417,7 @@ class MergeIterator public: MergeIterator(PollSet& readPollSet, PollSet& writePollSet); - bool next(int& fd_out, int& ev_bitmask_out); + bool next(curl_socket_t& fd_out, int& ev_bitmask_out); private: PollSet& mReadPollSet; @@ -380,12 +433,12 @@ MergeIterator::MergeIterator(PollSet& readPollSet, PollSet& writePollSet) : mWritePollSet.reset(); } -bool MergeIterator::next(int& fd_out, int& ev_bitmask_out) +bool MergeIterator::next(curl_socket_t& fd_out, int& ev_bitmask_out) { - int rfd = mReadPollSet.get(); - int wfd = mWritePollSet.get(); + curl_socket_t rfd = mReadPollSet.get(); + curl_socket_t wfd = mWritePollSet.get(); - if (rfd == -1 && wfd == -1) + if (rfd == CURL_SOCKET_BAD && wfd == CURL_SOCKET_BAD) return false; if (rfd == wfd) @@ -394,12 +447,12 @@ bool MergeIterator::next(int& fd_out, int& ev_bitmask_out) ev_bitmask_out = CURL_CSELECT_IN | CURL_CSELECT_OUT; mReadPollSet.next(); } - else if ((unsigned int)rfd < (unsigned int)wfd) // Use and increment smaller one, unless it's -1. + else if (wfd == CURL_SOCKET_BAD || rfd < wfd) // Use and increment smaller one, unless it's CURL_SOCKET_BAD. { fd_out = rfd; ev_bitmask_out = CURL_CSELECT_IN; mReadPollSet.next(); - if (wfd != -1 && mWritePollSet.is_set(rfd)) + if (wfd != CURL_SOCKET_BAD && mWritePollSet.is_set(rfd)) { ev_bitmask_out |= CURL_CSELECT_OUT; mWritePollSet.clr(rfd); @@ -410,7 +463,7 @@ bool MergeIterator::next(int& fd_out, int& ev_bitmask_out) fd_out = wfd; ev_bitmask_out = CURL_CSELECT_OUT; mWritePollSet.next(); - if (rfd != -1 && mReadPollSet.is_set(wfd)) + if (rfd != CURL_SOCKET_BAD && mReadPollSet.is_set(wfd)) { ev_bitmask_out |= CURL_CSELECT_IN; mReadPollSet.clr(wfd); @@ -501,8 +554,8 @@ class AICurlThread : public LLThread void create_wakeup_fds(void); void cleanup_wakeup_fds(void); - int mWakeUpFd_in; - int mWakeUpFd; + curl_socket_t mWakeUpFd_in; + curl_socket_t mWakeUpFd; int mZeroTimeOut; @@ -513,7 +566,7 @@ class AICurlThread : public LLThread AICurlThread* AICurlThread::sInstance = NULL; // MAIN-THREAD -AICurlThread::AICurlThread(void) : LLThread("AICurlThread"), mWakeUpFd_in(-1), mWakeUpFd(-1), mZeroTimeOut(0), mRunning(true) +AICurlThread::AICurlThread(void) : LLThread("AICurlThread"), mWakeUpFd_in(CURL_SOCKET_BAD), mWakeUpFd(CURL_SOCKET_BAD), mZeroTimeOut(0), mRunning(true) { create_wakeup_fds(); sInstance = this; @@ -554,9 +607,9 @@ void AICurlThread::create_wakeup_fds(void) // MAIN-THREAD void AICurlThread::cleanup_wakeup_fds(void) { - if (mWakeUpFd_in != -1) + if (mWakeUpFd_in != CURL_SOCKET_BAD) close(mWakeUpFd_in); - if (mWakeUpFd != -1) + if (mWakeUpFd != CURL_SOCKET_BAD) close(mWakeUpFd); } @@ -620,7 +673,7 @@ void AICurlThread::wakeup(AICurlMultiHandle_wat const& multi_handle_w) { llwarns << "read(3) from mWakeUpFd returned 0, indicating that the pipe on the other end was closed! Shutting down curl thread." << llendl; close(mWakeUpFd); - mWakeUpFd = -1; + mWakeUpFd = CURL_SOCKET_BAD; mRunning = false; return; } @@ -668,8 +721,8 @@ void AICurlThread::run(void) AICurlMultiHandle_wat multi_handle_w(AICurlMultiHandle::getInstance()); while(mRunning) { - // If mRunning is true then we can only get here if mWakeUpFd != -1. - llassert(mWakeUpFd != -1); + // If mRunning is true then we can only get here if mWakeUpFd != CURL_SOCKET_BAD. + llassert(mWakeUpFd != CURL_SOCKET_BAD); // Copy the next batch of file descriptors from the PollSets mFiledescriptors into their mFdSet. multi_handle_w->mReadPollSet.refresh(); refresh_t wres = multi_handle_w->mWritePollSet.refresh(); @@ -678,14 +731,18 @@ void AICurlThread::run(void) FD_SET(mWakeUpFd, read_fd_set); fd_set* write_fd_set = ((wres & empty)) ? NULL : multi_handle_w->mWritePollSet.access(); // Calculate nfds (ignored on windows). - int const max_rfd = std::max(multi_handle_w->mReadPollSet.get_max_fd(), mWakeUpFd); - int const max_wfd = multi_handle_w->mWritePollSet.get_max_fd(); +#if !LL_WINDOWS + curl_socket_t const max_rfd = std::max(multi_handle_w->mReadPollSet.get_max_fd(), mWakeUpFd); + curl_socket_t const max_wfd = multi_handle_w->mWritePollSet.get_max_fd(); int nfds = std::max(max_rfd, max_wfd) + 1; llassert(0 <= nfds && nfds <= FD_SETSIZE); llassert((max_rfd == -1) == (read_fd_set == NULL) && (max_wfd == -1) == (write_fd_set == NULL)); // Needed on Windows. llassert((max_rfd == -1 || multi_handle_w->mReadPollSet.is_set(max_rfd)) && (max_wfd == -1 || multi_handle_w->mWritePollSet.is_set(max_wfd))); +#else + int nfds = 64; +#endif int ready = 0; struct timeval timeout; long timeout_ms = multi_handle_w->getTimeOut(); @@ -769,7 +826,8 @@ void AICurlThread::run(void) } // Handle all active filedescriptors. MergeIterator iter(multi_handle_w->mReadPollSet, multi_handle_w->mWritePollSet); - int fd, ev_bitmask; + curl_socket_t fd; + int ev_bitmask; while (ready > 0 && iter.next(fd, ev_bitmask)) { ready -= (ev_bitmask == (CURL_CSELECT_IN|CURL_CSELECT_OUT)) ? 2 : 1; diff --git a/indra/llmessage/aicurlthread.h b/indra/llmessage/aicurlthread.h index a2262d9a8..3980915d0 100644 --- a/indra/llmessage/aicurlthread.h +++ b/indra/llmessage/aicurlthread.h @@ -72,8 +72,10 @@ class PollSet // Return a pointer to the underlaying fd_set. fd_set* access(void) { return &mFdSet; } +#if !LL_WINDOWS // Return the largest fd set in mFdSet by refresh. - int get_max_fd(void) const { return mMaxFdSet; } + curl_socket_t get_max_fd(void) const { return mMaxFdSet; } +#endif // Return true if a filedescriptor is set in mFileDescriptors (used for debugging). bool contains(curl_socket_t s) const; @@ -85,22 +87,26 @@ class PollSet void clr(curl_socket_t fd); // Iterate over all file descriptors that were set by refresh and are still set in mFdSet. - void reset(void); // Reset the iterator. - int get(void) const; // Return next filedescriptor, or -1 when there are no more. - // Only valid if reset() was called after the last call to refresh(). - void next(void); // Advance to next filedescriptor. + void reset(void); // Reset the iterator. + curl_socket_t get(void) const; // Return next filedescriptor, or CURL_SOCKET_BAD when there are no more. + // Only valid if reset() was called after the last call to refresh(). + void next(void); // Advance to next filedescriptor. private: curl_socket_t* mFileDescriptors; - int mNrFds; // The number of filedescriptors in the array. - int mMaxFd; // The largest filedescriptor in the array, or -1 when it is empty. - int mNext; // The index of the first file descriptor to start copying, the next call to refresh(). + int mNrFds; // The number of filedescriptors in the array. + int mNext; // The index of the first file descriptor to start copying, the next call to refresh(). - fd_set mFdSet; // Output variable for select(). (Re)initialized by calling refresh(). - int mMaxFdSet; // The largest filedescriptor set in mFdSet by refresh(), or -1 when it was empty. + fd_set mFdSet; // Output variable for select(). (Re)initialized by calling refresh(). +#if !LL_WINDOWS + curl_socket_t mMaxFd; // The largest filedescriptor in the array, or CURL_SOCKET_BAD when it is empty. + curl_socket_t mMaxFdSet; // The largest filedescriptor set in mFdSet by refresh(), or CURL_SOCKET_BAD when it was empty. std::vector mCopiedFileDescriptors; // Filedescriptors copied by refresh to mFdSet. std::vector::iterator mIter; // Index into mCopiedFileDescriptors for next(); loop variable. +#else + int mIter; // Index into fd_set::fd_array. +#endif }; //----------------------------------------------------------------------------- From 14276b3cf875247be76180b044e333233cf824cb Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Wed, 4 Jul 2012 08:44:22 +0200 Subject: [PATCH 11/11] Bug fix --- indra/llmessage/aicurlthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indra/llmessage/aicurlthread.cpp b/indra/llmessage/aicurlthread.cpp index e7b62947d..f91d49cdb 100644 --- a/indra/llmessage/aicurlthread.cpp +++ b/indra/llmessage/aicurlthread.cpp @@ -447,7 +447,7 @@ bool MergeIterator::next(curl_socket_t& fd_out, int& ev_bitmask_out) ev_bitmask_out = CURL_CSELECT_IN | CURL_CSELECT_OUT; mReadPollSet.next(); } - else if (wfd == CURL_SOCKET_BAD || rfd < wfd) // Use and increment smaller one, unless it's CURL_SOCKET_BAD. + else if (wfd == CURL_SOCKET_BAD || (rfd != CURL_SOCKET_BAD && rfd < wfd)) // Use and increment smaller one, unless it's CURL_SOCKET_BAD. { fd_out = rfd; ev_bitmask_out = CURL_CSELECT_IN;