Singularity-Viewer/SingularityViewer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shyotl's old llcurl

Browse Source
This commit is contained in:
Siana Gearz
2012-08-07 13:12:56 +02:00
parent 6ed9a62501
commit 68ad5411c9
8 changed files with 290 additions and 781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

763
indra/llmessage/llcurl.cpp
View File

File diff suppressed because it is too large Load Diff

165
indra/llmessage/llcurl.h
View File

@@ -42,11 +42,8 @@
#include "lliopipe.h"
#include "llsd.h"
#include "llthread.h"
#include "llqueuedthread.h"
#include "llframetimer.h"
class LLMutex;
class LLCurlThread;
// For whatever reason, this is not typedef'd in curl.h
typedef size_t (*curl_header_callback)(void *ptr, size_t size, size_t nmemb, void *stream);
@@ -59,6 +56,8 @@ public:
class Easy;
class Multi;
static bool sMultiThreaded;
struct TransferInfo
{
TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) {}
@@ -125,7 +124,6 @@ public:
{
return false;
}
public: /* but not really -- don't touch this */
U32 mReferenceCount;
@@ -163,7 +161,7 @@ public:
/**
* @ brief Initialize LLCurl class
*/
static void initClass(F32 curl_reuest_timeout = 120.f, S32 max_number_handles = 256, bool multi_threaded = false);
static void initClass(bool multi_threaded = false);
/**
* @ brief Cleanup LLCurl class
@@ -182,25 +180,10 @@ public:
static void ssl_locking_callback(int mode, int type, const char *file, int line);
static unsigned long ssl_thread_id(void);
static LLCurlThread* getCurlThread() { return sCurlThread ;}
static CURLM* newMultiHandle() ;
static CURLMcode deleteMultiHandle(CURLM* handle) ;
static CURL* newEasyHandle() ;
static void deleteEasyHandle(CURL* handle) ;
private:
static std::string sCAPath;
static std::string sCAFile;
static const unsigned int MAX_REDIRECTS;
static LLCurlThread* sCurlThread;
static LLMutex* sHandleMutexp ;
static S32 sTotalHandles ;
static S32 sMaxHandles;
public:
static bool sNotQuitting;
static F32 sCurlRequestTimeOut;
};
class LLCurl::Easy
@@ -209,7 +192,7 @@ class LLCurl::Easy
private:
Easy();
public:
static Easy* getEasy();
~Easy();
@@ -218,41 +201,41 @@ public:
void setErrorBuffer();
void setCA();
void setopt(CURLoption option, S32 value);
// These assume the setter does not free value!
void setopt(CURLoption option, void* value);
void setopt(CURLoption option, char* value);
// Copies the string so that it is guaranteed to stick around
// Copies the string so that it is gauranteed to stick around
void setoptString(CURLoption option, const std::string& value);
void slist_append(const char* str);
void setHeaders();
U32 report(CURLcode);
void getTransferInfo(LLCurl::TransferInfo* info);
void prepRequest(const std::string& url, const std::vector<std::string>& headers, LLCurl::ResponderPtr, S32 time_out = 0, bool post = false);
void prepRequest(const std::string& url, const std::vector<std::string>& headers, ResponderPtr, S32 time_out = 0, bool post = false);
const char* getErrorBuffer();
std::stringstream& getInput() { return mInput; }
std::stringstream& getHeaderOutput() { return mHeaderOutput; }
LLIOPipe::buffer_ptr_t& getOutput() { return mOutput; }
const LLChannelDescriptors& getChannels() { return mChannels; }
void resetState();
static CURL* allocEasyHandle();
static void releaseEasyHandle(CURL* handle);
private:
private:
friend class LLCurl;
friend class LLCurl::Multi;
CURL* mCurlEasyHandle;
struct curl_slist* mHeaders;
std::stringstream mRequest;
LLChannelDescriptors mChannels;
LLIOPipe::buffer_ptr_t mOutput;
@@ -262,125 +245,70 @@ private:
// Note: char*'s not strings since we pass pointers to curl
std::vector<char*> mStrings;
LLCurl::ResponderPtr mResponder;
ResponderPtr mResponder;
static std::set<CURL*> sFreeHandles;
static std::set<CURL*> sActiveHandles;
static LLMutex* sHandleMutexp ;
static LLMutex* sHandleMutex;
static LLMutex* sMultiMutex;
};
class LLCurl::Multi
class LLCurl::Multi : public LLThread
{
LOG_CLASS(Multi);
friend class LLCurlThread ;
private:
~Multi();
void markDead() ;
bool doPerform();
public:
typedef enum
{
STATE_READY=0,
STATE_PERFORMING=1,
STATE_COMPLETED=2
PERFORM_STATE_READY=0,
PERFORM_STATE_PERFORMING=1,
PERFORM_STATE_COMPLETED=2
} ePerformState;
Multi(F32 idle_time_out = 0.f);
Multi();
~Multi();
LLCurl::Easy* allocEasy();
bool addEasy(LLCurl::Easy* easy);
void removeEasy(LLCurl::Easy* easy);
Easy* allocEasy();
bool addEasy(Easy* easy);
void lock() ;
void unlock() ;
void setState(ePerformState state) ;
ePerformState getState() ;
bool isCompleted() ;
bool isValid() {return mCurlMultiHandle != NULL && mValid;}
bool isDead() {return mDead;}
bool waitToComplete() ;
void removeEasy(Easy* easy);
S32 process();
void perform();
void doPerform();
virtual void run();
CURLMsg* info_read(S32* msgs_in_queue);
S32 mQueued;
S32 mErrorCount;
S32 mPerformState;
LLCondition* mSignal;
bool mQuitting;
bool mThreaded;
private:
void easyFree(LLCurl::Easy*);
void cleanup(bool deleted = false) ;
void easyFree(Easy*);
CURLM* mCurlMultiHandle;
typedef std::set<LLCurl::Easy*> easy_active_list_t;
typedef std::set<Easy*> easy_active_list_t;
easy_active_list_t mEasyActiveList;
typedef std::map<CURL*, LLCurl::Easy*> easy_active_map_t;
typedef std::map<CURL*, Easy*> easy_active_map_t;
easy_active_map_t mEasyActiveMap;
typedef std::set<LLCurl::Easy*> easy_free_list_t;
typedef std::set<Easy*> easy_free_list_t;
easy_free_list_t mEasyFreeList;
LLQueuedThread::handle_t mHandle ;
ePerformState mState;
BOOL mDead ;
BOOL mValid ;
LLMutex* mMutexp ;
LLMutex* mDeletionMutexp ;
LLMutex* mEasyMutexp ;
LLFrameTimer mIdleTimer ;
F32 mIdleTimeOut;
};
class LLCurlThread : public LLQueuedThread
namespace boost
{
public:
class CurlRequest : public LLQueuedThread::QueuedRequest
{
protected:
virtual ~CurlRequest(); // use deleteRequest()
public:
CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread);
/*virtual*/ bool processRequest();
/*virtual*/ void finishRequest(bool completed);
private:
// input
LLCurl::Multi* mMulti;
LLCurlThread* mCurlThread;
};
friend class CurlRequest;
public:
LLCurlThread(bool threaded = true) ;
virtual ~LLCurlThread() ;
S32 update(F32 max_time_ms);
void addMulti(LLCurl::Multi* multi) ;
void killMulti(LLCurl::Multi* multi) ;
private:
bool doMultiPerform(LLCurl::Multi* multi) ;
void deleteMulti(LLCurl::Multi* multi) ;
void cleanupMulti(LLCurl::Multi* multi) ;
} ;
void intrusive_ptr_add_ref(LLCurl::Responder* p);
void intrusive_ptr_release(LLCurl::Responder* p);
void intrusive_ptr_add_ref(LLCurl::Responder* p);
void intrusive_ptr_release(LLCurl::Responder* p);
};
class LLCurlRequest
@@ -410,6 +338,7 @@ private:
LLCurl::Multi* mActiveMulti;
S32 mActiveRequestCount;
BOOL mProcessing;
U32 mThreadID; // debug
};
class LLCurlEasyRequest
@@ -427,11 +356,9 @@ public:
void slist_append(const char* str);
void sendRequest(const std::string& url);
void requestComplete();
void perform();
bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL);
std::string getErrorString();
bool isCompleted() {return mMulti->isCompleted() ;}
bool wait() { return mMulti->waitToComplete(); }
bool isValid() {return mMulti && mMulti->isValid(); }
LLCurl::Easy* getEasy() const { return mEasy; }

52
indra/llmessage/llhttpclient.cpp
View File

@@ -25,7 +25,7 @@
*/
#include "linden_common.h"
#include <openssl/x509_vfy.h>
#include "llhttpclient.h"
#include "llassetstorage.h"
@@ -40,10 +40,8 @@
#include "message.h"
#include <curl/curl.h>
const F32 HTTP_REQUEST_EXPIRY_SECS = 60.0f;
LLURLRequest::SSLCertVerifyCallback LLHTTPClient::mCertVerifyCallback = NULL;
////////////////////////////////////////////////////////////////////////////
// Responder class moved to LLCurl
@@ -160,7 +158,7 @@ namespace
fstream.seekg(0, std::ios::end);
U32 fileSize = (U32)fstream.tellg();
fstream.seekg(0, std::ios::beg);
std::vector<char> fileBuffer(fileSize);
std::vector<char> fileBuffer(fileSize); //Mem leak fix'd
fstream.read(&fileBuffer[0], fileSize);
ostream.write(&fileBuffer[0], fileSize);
fstream.close();
@@ -189,11 +187,9 @@ namespace
LLVFile vfile(gVFS, mUUID, mAssetType, LLVFile::READ);
S32 fileSize = vfile.getSize();
U8* fileBuffer;
fileBuffer = new U8 [fileSize];
vfile.read(fileBuffer, fileSize);
ostream.write((char*)fileBuffer, fileSize);
delete [] fileBuffer;
std::vector<U8> fileBuffer(fileSize);
vfile.read(&fileBuffer[0], fileSize);
ostream.write((char*)&fileBuffer[0], fileSize);
eos = true;
return STATUS_DONE;
}
@@ -202,7 +198,6 @@ namespace
LLAssetType::EType mAssetType;
};
LLPumpIO* theClientPump = NULL;
}
@@ -217,8 +212,7 @@ static void request(
Injector* body_injector,
LLCurl::ResponderPtr responder,
const F32 timeout = HTTP_REQUEST_EXPIRY_SECS,
const LLSD& headers = LLSD()
)
const LLSD& headers = LLSD())
{
if (!LLHTTPClient::hasPump())
{
@@ -228,26 +222,12 @@ static void request(
LLPumpIO::chain_t chain;
LLURLRequest* req = new LLURLRequest(method, url);
if(!req->isValid())//failed
{
delete req ;
return ;
}
req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req);
lldebugs << LLURLRequest::actionAsVerb(method) << " " << url << " "
<< headers << llendl;
// Insert custom headers if the caller sent any
if (headers.isMap())
{
if (headers.has("Cookie"))
{
req->allowCookies();
}
// Insert custom headers is the caller sent any
if (headers.isMap())
{
LLSD::map_const_iterator iter = headers.beginMap();
LLSD::map_const_iterator end = headers.endMap();
@@ -429,16 +409,11 @@ static LLSD blocking_request(
{
lldebugs << "blockingRequest of " << url << llendl;
char curl_error_buffer[CURL_ERROR_SIZE] = "\0";
CURL* curlp = LLCurl::newEasyHandle();
llassert_always(curlp != NULL) ;
CURL* curlp = curl_easy_init();
LLHTTPBuffer http_buffer;
std::string body_str;
// other request method checks root cert first, we skip?
// Apply configured proxy settings
LLProxy::getInstance()->applyProxySettings(curlp);
// * Set curl handle options
curl_easy_setopt(curlp, CURLOPT_NOSIGNAL, 1); // don't use SIGALRM for timeouts
@@ -447,7 +422,7 @@ static LLSD blocking_request(
curl_easy_setopt(curlp, CURLOPT_WRITEDATA, &http_buffer);
curl_easy_setopt(curlp, CURLOPT_URL, url.c_str());
curl_easy_setopt(curlp, CURLOPT_ERRORBUFFER, curl_error_buffer);
// * Setup headers (don't forget to free them after the call!)
curl_slist* headers_list = NULL;
if (headers.isMap())
@@ -525,7 +500,7 @@ static LLSD blocking_request(
}
// * Cleanup
LLCurl::deleteEasyHandle(curlp);
curl_easy_cleanup(curlp);
return response;
}
@@ -625,8 +600,7 @@ bool LLHTTPClient::hasPump()
return theClientPump != NULL;
}
//static
LLPumpIO& LLHTTPClient::getPump()
LLPumpIO &LLHTTPClient::getPump()
{
return *theClientPump;
}

6
indra/llmessage/llhttpclient.h
View File

@@ -34,10 +34,11 @@
#include <string>
#include <boost/intrusive_ptr.hpp>
#include "llurlrequest.h"
#include "llassettype.h"
#include "llcurl.h"
#include "lliopipe.h"
#include "llurlrequest.h"
extern const F32 HTTP_REQUEST_EXPIRY_SECS;
@@ -55,7 +56,6 @@ public:
typedef LLCurl::Responder Responder;
typedef LLCurl::ResponderPtr ResponderPtr;
/** @name non-blocking API */
//@{
static void head(
@@ -155,7 +155,7 @@ public:
///< Hippo special
static void setCertVerifyCallback(LLURLRequest::SSLCertVerifyCallback callback);
static LLURLRequest::SSLCertVerifyCallback getCertVerifyCallback() { return mCertVerifyCallback; }
static LLURLRequest::SSLCertVerifyCallback getCertVerifyCallback() { return mCertVerifyCallback; }
protected:
static LLURLRequest::SSLCertVerifyCallback mCertVerifyCallback;

55
indra/llmessage/llurlrequest.cpp
View File

@@ -32,17 +32,17 @@
#include <algorithm>
#include <openssl/x509_vfy.h>
#include <openssl/ssl.h>
#include "llcurl.h"
#include "llfasttimer.h"
#include "llioutil.h"
#include "llmemtype.h"
#include "llproxy.h"
#include "llpumpio.h"
#include "llsd.h"
#include "llstring.h"
#include "apr_env.h"
#include "llapr.h"
#include "llscopedvolatileaprpool.h"
#include "llfasttimer.h"
static const U32 HTTP_STATUS_PIPE_ERROR = 499;
/**
@@ -66,7 +66,7 @@ public:
~LLURLRequestDetail();
std::string mURL;
LLCurlEasyRequest* mCurlRequest;
LLIOPipe::buffer_ptr_t mResponseBuffer;
LLBufferArray* mResponseBuffer;
LLChannelDescriptors mChannels;
U8* mLastRead;
U32 mBodyLimit;
@@ -77,26 +77,21 @@ public:
LLURLRequestDetail::LLURLRequestDetail() :
mCurlRequest(NULL),
mResponseBuffer(NULL),
mLastRead(NULL),
mBodyLimit(0),
mByteAccumulator(0),
mIsBodyLimitSet(false),
mSSLVerifyCallback(NULL)
mIsBodyLimitSet(false)
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
mCurlRequest = new LLCurlEasyRequest();
if(!mCurlRequest->isValid()) //failed.
{
delete mCurlRequest ;
mCurlRequest = NULL ;
}
}
LLURLRequestDetail::~LLURLRequestDetail()
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
delete mCurlRequest;
mResponseBuffer = NULL;
mLastRead = NULL;
}
@@ -105,7 +100,7 @@ void LLURLRequest::setSSLVerifyCallback(SSLCertVerifyCallback callback, void *pa
mDetail->mSSLVerifyCallback = callback;
mDetail->mCurlRequest->setSSLCtxCallback(LLURLRequest::_sslCtxCallback, (void *)this);
mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYPEER, true);
mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYHOST, 2);
mDetail->mCurlRequest->setopt(CURLOPT_SSL_VERIFYHOST, 2);
}
@@ -114,7 +109,7 @@ void LLURLRequest::setSSLVerifyCallback(SSLCertVerifyCallback callback, void *pa
// used to configure the context for custom cert validation
CURLcode LLURLRequest::_sslCtxCallback(CURL * curl, void *sslctx, void *param)
{
{
LLURLRequest *req = (LLURLRequest *)param;
if(req == NULL || req->mDetail->mSSLVerifyCallback == NULL)
{
@@ -128,7 +123,6 @@ CURLcode LLURLRequest::_sslCtxCallback(CURL * curl, void *sslctx, void *param)
SSL_CTX_set_cert_verify_callback(ctx, req->mDetail->mSSLVerifyCallback, (void *)req);
// the calls are void
return CURLE_OK;
}
/**
@@ -189,7 +183,6 @@ std::string LLURLRequest::getURL() const
{
return mDetail->mURL;
}
void LLURLRequest::addHeader(const char* header)
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
@@ -240,9 +233,9 @@ void LLURLRequest::useProxy(bool use_proxy)
}
}
lldebugs << "use_proxy = " << (use_proxy?'Y':'N') << ", env_proxy = \"" << env_proxy << "\"" << llendl;
LL_DEBUGS("Proxy") << "use_proxy = " << (use_proxy?'Y':'N') << ", env_proxy = " << (!env_proxy.empty() ? env_proxy : "(null)") << LL_ENDL;
if (use_proxy)
if (use_proxy && !env_proxy.empty())
{
mDetail->mCurlRequest->setoptString(CURLOPT_PROXY, env_proxy);
}
@@ -262,24 +255,12 @@ void LLURLRequest::allowCookies()
mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, "");
}
//virtual
bool LLURLRequest::isValid()
{
return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid();
}
// virtual
LLIOPipe::EStatus LLURLRequest::handleError(
LLIOPipe::EStatus status,
LLPumpIO* pump)
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
if(!isValid())
{
return STATUS_EXPIRED ;
}
if(mCompletionCallback && pump)
{
LLURLRequestComplete* complete = NULL;
@@ -309,7 +290,8 @@ LLIOPipe::EStatus LLURLRequest::process_impl(
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
//llinfos << "LLURLRequest::process_impl()" << llendl;
if (!buffer) return STATUS_ERROR;
if (!mDetail) return STATUS_ERROR; //Seems to happen on occasion. Need to hunt down why.
// we're still waiting or prcessing, check how many
// bytes we have accumulated.
const S32 MIN_ACCUMULATION = 100000;
@@ -348,7 +330,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl(
// *FIX: bit of a hack, but it should work. The configure and
// callback method expect this information to be ready.
mDetail->mResponseBuffer = buffer;
mDetail->mResponseBuffer = buffer.get();
mDetail->mChannels = channels;
if(!configure())
{
@@ -367,10 +349,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl(
static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform");
{
LLFastTimer t(FTM_URL_PERFORM);
if(!mDetail->mCurlRequest->wait())
{
return status ;
}
mDetail->mCurlRequest->perform();
}
while(1)
@@ -465,12 +444,6 @@ void LLURLRequest::initialize()
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
mState = STATE_INITIALIZED;
mDetail = new LLURLRequestDetail;
if(!isValid())
{
return ;
}
mDetail->mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1);
mDetail->mCurlRequest->setWriteCallback(&downCallback, (void*)this);
mDetail->mCurlRequest->setReadCallback(&upCallback, (void*)this);

11
indra/llmessage/llurlrequest.h
View File

@@ -40,7 +40,6 @@
#include "llerror.h"
#include "llcurl.h"
extern const std::string CONTEXT_REQUEST;
extern const std::string CONTEXT_DEST_URI_SD_LABEL;
extern const std::string CONTEXT_RESPONSE;
@@ -145,7 +144,6 @@ public:
*/
void setSSLVerifyCallback(SSLCertVerifyCallback callback, void * param);
/**
* @brief Return at most size bytes of body.
*
@@ -190,14 +188,11 @@ public:
*/
void allowCookies();
/*virtual*/ bool isValid() ;
public:
/**
* @brief Give this pipe a chance to handle a generated error
*/
virtual EStatus handleError(EStatus status, LLPumpIO* pump);
protected:
/**
@@ -223,11 +218,11 @@ protected:
ERequestAction mAction;
LLURLRequestDetail* mDetail;
LLIOPipe::ptr_t mCompletionCallback;
S32 mRequestTransferedBytes;
S32 mResponseTransferedBytes;
S32 mRequestTransferedBytes;
S32 mResponseTransferedBytes;
static CURLcode _sslCtxCallback(CURL * curl, void *sslctx, void *param);
private:
/**
* @brief Initialize the object. Called during construction.

5
indra/newview/llappviewer.cpp
View File

@@ -637,9 +637,8 @@ bool LLAppViewer::init()
mAlloc.setProfilingEnabled(gSavedSettings.getBOOL("MemProfiling"));
// *NOTE:Mani - LLCurl::initClass is not thread safe.
// Called before threads are created.
LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"),
gSavedSettings.getS32("CurlMaximumNumberOfHandles"),
gSavedSettings.getBOOL("CurlUseMultipleThreads"));
LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"));
LL_INFOS("InitInfo") << "LLCurl initialized." << LL_ENDL ;
initThreads();

14
indra/newview/llxmlrpctransaction.cpp
View File

@@ -227,16 +227,6 @@ void LLXMLRPCTransaction::Impl::init(XMLRPC_REQUEST request, bool useGzip)
mCurlRequest = new LLCurlEasyRequest();
}
if(!mCurlRequest->isValid())
{
llwarns << "mCurlRequest is invalid." << llendl ;
delete mCurlRequest ;
mCurlRequest = NULL ;
return ;
}
LLProxy::getInstance()->applyProxySettings(mCurlRequest);
// mCurlRequest->setopt(CURLOPT_VERBOSE, 1); // usefull for debugging
@@ -292,7 +282,7 @@ LLXMLRPCTransaction::Impl::~Impl()
bool LLXMLRPCTransaction::Impl::process()
{
if(!mCurlRequest || !mCurlRequest->isValid())
if(!mCurlRequest || !mCurlRequest->getEasy())
{
llwarns << "transaction failed." << llendl ;
@@ -326,7 +316,7 @@ bool LLXMLRPCTransaction::Impl::process()
//const F32 MAX_PROCESSING_TIME = 0.05f;
//LLTimer timer;
mCurlRequest->wait();
mCurlRequest->perform();
/*while (mCurlRequest->perform() > 0)
{