From 5fb4badb7ccd66b0d834f132c17e7531aec558a8 Mon Sep 17 00:00:00 2001 From: Aleric Inglewood Date: Thu, 2 Aug 2012 22:34:23 +0200 Subject: [PATCH] WIP --- indra/aistatemachine/aicurl.cpp | 94 ++++++++++++++++--- indra/aistatemachine/aicurl.h | 1 + .../aicurleasyrequeststatemachine.cpp | 52 ++++++---- .../aicurleasyrequeststatemachine.h | 23 +++-- indra/aistatemachine/aicurlprivate.h | 11 ++- indra/llcommon/llstring.h | 14 +++ indra/llmessage/llhttpclient.cpp | 70 +++++++------- indra/llmessage/llurlrequest.cpp | 4 +- 8 files changed, 192 insertions(+), 77 deletions(-) diff --git a/indra/aistatemachine/aicurl.cpp b/indra/aistatemachine/aicurl.cpp index 1cbf1afaa..054d78389 100644 --- a/indra/aistatemachine/aicurl.cpp +++ b/indra/aistatemachine/aicurl.cpp @@ -1091,6 +1091,12 @@ void CurlEasyRequest::added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_req mEventsTarget->added_to_multi_handle(curl_easy_request_w); } +void CurlEasyRequest::decoded_header(AICurlEasyRequest_wat& curl_easy_request_w, std::string const& key, std::string const& value) +{ + if (mEventsTarget) + mEventsTarget->decoded_header(curl_easy_request_w, key, value); +} + void CurlEasyRequest::finished(AICurlEasyRequest_wat& curl_easy_request_w) { if (mEventsTarget) @@ -1106,7 +1112,7 @@ void CurlEasyRequest::removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy //----------------------------------------------------------------------------- // CurlResponderBuffer -static unsigned int const MAX_REDIRECTS = 5; +static int const HTTP_REDIRECTS_DEFAULT = 10; static S32 const CURL_REQUEST_TIMEOUT = 30; // Seconds per operation. LLChannelDescriptors const CurlResponderBuffer::sChannels; @@ -1165,9 +1171,6 @@ void CurlResponderBuffer::resetState(AICurlEasyRequest_wat& curl_easy_request_w) mOutput.reset(); mInput.reset(); - - mHeaderOutput.str(""); - mHeaderOutput.clear(); } ThreadSafeBufferedCurlEasyRequest* CurlResponderBuffer::get_lockobj(void) @@ -1194,14 +1197,14 @@ void CurlResponderBuffer::prepRequest(AICurlEasyRequest_wat& curl_easy_request_w curl_easy_request_w->setReadCallback(&curlReadCallback, lockobj); curl_easy_request_w->setHeaderCallback(&curlHeaderCallback, lockobj); - // Allow up to five redirects. + // Allow up to ten redirects. if (responder && responder->followRedir()) { curl_easy_request_w->setopt(CURLOPT_FOLLOWLOCATION, 1); - curl_easy_request_w->setopt(CURLOPT_MAXREDIRS, MAX_REDIRECTS); + curl_easy_request_w->setopt(CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); } - curl_easy_request_w->setopt(CURLOPT_SSL_VERIFYPEER, true); + curl_easy_request_w->setopt(CURLOPT_SSL_VERIFYPEER, 1); // Don't verify host name so urls with scrubbed host names will work (improves DNS performance). curl_easy_request_w->setopt(CURLOPT_SSL_VERIFYHOST, 0); @@ -1261,14 +1264,74 @@ size_t CurlResponderBuffer::curlHeaderCallback(char* data, size_t size, size_t n { ThreadSafeBufferedCurlEasyRequest* lockobj = static_cast(user_data); - // We need to lock the curl easy request object too, because that lock is used + // We need to lock the curl easy request object, because that lock is used // to make sure that callbacks and destruction aren't done simultaneously. AICurlEasyRequest_wat buffered_easy_request_w(*lockobj); - AICurlResponderBuffer_wat buffer_w(*lockobj); - size_t n = size * nmemb; - buffer_w->getHeaderOutput().write(data, n); - return n; + // This used to be headerCallback() in llurlrequest.cpp. + + char const* const header_line = static_cast(data); + size_t const header_len = size * nmemb; + + if (!header_len) + { + return header_len; + } + std::string header(header_line, header_len); + if (!LLStringUtil::_isASCII(header)) + { + return header_len; + } + + // Per HTTP spec the first header line must be the status line. + if (header.substr(0, 5) == "HTTP/") + { + std::string::iterator const begin = header.begin(); + std::string::iterator const end = header.end(); + std::string::iterator pos1 = std::find(begin, end, ' '); + if (pos1 != end) ++pos1; + std::string::iterator pos2 = std::find(pos1, end, ' '); + if (pos2 != end) ++pos2; + std::string::iterator pos3 = std::find(pos2, end, '\r'); + U32 status; + std::string reason; + if (pos3 != end && std::isdigit(*pos1)) + { + status = atoi(&header_line[pos1 - begin]); + reason.assign(pos2, pos3); + } + else + { + status = HTTP_INTERNAL_ERROR; + reason = "Header parse error."; + llwarns << "Received broken header line from server: \"" << header << "\"" << llendl; + } + AICurlResponderBuffer_wat(*lockobj)->setStatusAndReason(status, reason); + return header_len; + } + + std::string::iterator sep = std::find(header.begin(), header.end(), ':'); + + if (sep != header.end()) + { + std::string key(header.begin(), sep); + std::string value(sep + 1, header.end()); + + key = utf8str_tolower(utf8str_trim(key)); + value = utf8str_trim(value); + + AICurlResponderBuffer_wat(*lockobj)->decoded_header(buffered_easy_request_w, key, value); + } + else + { + LLStringUtil::trim(header); + if (!header.empty()) + { + llwarns << "Unable to parse header: " << header << llendl; + } + } + + return header_len; } void CurlResponderBuffer::added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) @@ -1276,6 +1339,11 @@ void CurlResponderBuffer::added_to_multi_handle(AICurlEasyRequest_wat& curl_easy Dout(dc::curl, "Calling CurlResponderBuffer::added_to_multi_handle(@" << (void*)&*curl_easy_request_w << ") for this = " << (void*)this); } +void CurlResponderBuffer::decoded_header(AICurlEasyRequest_wat& curl_easy_request_w, std::string const& key, std::string const& value) +{ + Dout(dc::curl, "Calling CurlResponderBuffer::decoded_header(@" << (void*)&*curl_easy_request_w << ", \"" << key << "\", \"" << value << "\") for this = " << (void*)this); +} + void CurlResponderBuffer::finished(AICurlEasyRequest_wat& curl_easy_request_w) { Dout(dc::curl, "Calling CurlResponderBuffer::finished(@" << (void*)&*curl_easy_request_w << ") for this = " << (void*)this); @@ -1304,7 +1372,7 @@ void CurlResponderBuffer::processOutput(AICurlEasyRequest_wat& curl_easy_request if (code == CURLE_OK) { curl_easy_request_w->getinfo(CURLINFO_RESPONSE_CODE, &responseCode); - //*TODO: get reason from first line of mHeaderOutput + //AIFIXME: fill responseReason if (responseCode < 200 || responseCode >= 300). } else { diff --git a/indra/aistatemachine/aicurl.h b/indra/aistatemachine/aicurl.h index 90803e050..a8b4dda90 100644 --- a/indra/aistatemachine/aicurl.h +++ b/indra/aistatemachine/aicurl.h @@ -235,6 +235,7 @@ typedef AIAccess AICurlEasyRequest_wat; struct AICurlEasyHandleEvents { // Events. virtual void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0; + virtual void decoded_header(AICurlEasyRequest_wat& curl_easy_request_w, std::string const& key, std::string const& value) = 0; virtual void finished(AICurlEasyRequest_wat& curl_easy_request_w) = 0; virtual void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0; // Avoid compiler warning. diff --git a/indra/aistatemachine/aicurleasyrequeststatemachine.cpp b/indra/aistatemachine/aicurleasyrequeststatemachine.cpp index d6ab002db..b4e98a22b 100644 --- a/indra/aistatemachine/aicurleasyrequeststatemachine.cpp +++ b/indra/aistatemachine/aicurleasyrequeststatemachine.cpp @@ -77,6 +77,11 @@ void AICurlEasyRequestStateMachine::added_to_multi_handle(AICurlEasyRequest_wat& set_state(AICurlEasyRequestStateMachine_added); } +// CURL-THREAD +void AICurlEasyRequestStateMachine::decoded_header(AICurlEasyRequest_wat&, std::string const& key, std::string const& value) +{ +} + // CURL-THREAD void AICurlEasyRequestStateMachine::finished(AICurlEasyRequest_wat&) { @@ -116,12 +121,15 @@ void AICurlEasyRequestStateMachine::multiplex_impl(void) // 3) AICurlEasyRequestStateMachine_finished (running) // 4) AICurlEasyRequestStateMachine_removed_after_finished (running) - // Set an inactivity timer. - // This shouldn't really be necessary, except in the case of a bug - // in libcurl; but lets be sure and set a timer for inactivity. - mTimer = new AIPersistentTimer; // Do not delete timer upon expiration. - mTimer->setInterval(sCurlRequestTimeOut); - mTimer->run(this, AICurlEasyRequestStateMachine_timedOut, false, false); + if (mRequestTimeOut > 0.f) + { + // Set an inactivity timer. + // This shouldn't really be necessary, except in the case of a bug + // in libcurl; but lets be sure and set a timer for inactivity. + mTimer = new AIPersistentTimer; // Do not delete timer upon expiration. + mTimer->setInterval(mRequestTimeOut); + mTimer->run(this, AICurlEasyRequestStateMachine_timedOut, false, false); + } break; } case AICurlEasyRequestStateMachine_added: @@ -160,9 +168,12 @@ void AICurlEasyRequestStateMachine::multiplex_impl(void) // Only do this once. mHandled = true; - // Stop the timer. Note that it's the main thread that generates timer events, - // so we're certain that there will be no time out anymore if we reach this point. - mTimer->abort(); + if (mTimer) + { + // Stop the timer. Note that it's the main thread that generates timer events, + // so we're certain that there will be no time out anymore if we reach this point. + mTimer->abort(); + } // The request finished and either data or an error code is available. if (mBuffered) @@ -228,15 +239,19 @@ void AICurlEasyRequestStateMachine::finish_impl(void) curl_easy_request_w->send_events_to(NULL); curl_easy_request_w->revokeCallbacks(); } - // Note that even if the timer expired, it wasn't deleted because we used AIPersistentTimer; so mTimer is still valid. - // Stop the timer, if it's still running. - if (!mHandled) - mTimer->abort(); + if (mTimer) + { + // Note that even if the timer expired, it wasn't deleted because we used AIPersistentTimer; so mTimer is still valid. + // Stop the timer, if it's still running. + if (!mHandled) + mTimer->abort(); + } // Auto clean up ourselves. kill(); } -AICurlEasyRequestStateMachine::AICurlEasyRequestStateMachine(bool buffered) : mBuffered(buffered), mCurlEasyRequest(buffered) +AICurlEasyRequestStateMachine::AICurlEasyRequestStateMachine(bool buffered) : + mBuffered(buffered), mCurlEasyRequest(buffered), mTimer(NULL), mRequestTimeOut(sCurlRequestTimeOut) { Dout(dc::statemachine, "Calling AICurlEasyRequestStateMachine(" << (buffered ? "true" : "false") << ") [" << (void*)this << "] [" << (void*)mCurlEasyRequest.get() << "]"); if (!mBuffered) @@ -249,9 +264,14 @@ AICurlEasyRequestStateMachine::AICurlEasyRequestStateMachine(bool buffered) : mB F32 AICurlEasyRequestStateMachine::sCurlRequestTimeOut = 40.f; //static -void AICurlEasyRequestStateMachine::setCurlRequestTimeOut(F32 CurlRequestTimeOut) +void AICurlEasyRequestStateMachine::setDefaultRequestTimeOut(F32 defaultRequestTimeOut) { - sCurlRequestTimeOut = CurlRequestTimeOut; + sCurlRequestTimeOut = defaultRequestTimeOut; +} + +void AICurlEasyRequestStateMachine::setRequestTimeOut(F32 curlRequestTimeOut) +{ + mRequestTimeOut = curlRequestTimeOut; } AICurlEasyRequestStateMachine::~AICurlEasyRequestStateMachine() diff --git a/indra/aistatemachine/aicurleasyrequeststatemachine.h b/indra/aistatemachine/aicurleasyrequeststatemachine.h index 52134a02e..b3aae1c2e 100644 --- a/indra/aistatemachine/aicurleasyrequeststatemachine.h +++ b/indra/aistatemachine/aicurleasyrequeststatemachine.h @@ -58,18 +58,22 @@ class AICurlEasyRequestStateMachine : public AIStateMachine, public AICurlEasyHa AICurlEasyRequest mCurlEasyRequest; private: - bool mBuffered; // Argument used for construction of mCurlEasyRequest. - bool mAdded; // Set when the last command to the curl thread was to add the request. - bool mTimedOut; // Set if the expiration timer timed out. - bool mFinished; // Set by the curl thread to signal it finished. - bool mHandled; // Set when we processed the received data. - AITimer* mTimer; // Expiration timer. + bool mBuffered; // Argument used for construction of mCurlEasyRequest. + bool mAdded; // Set when the last command to the curl thread was to add the request. + bool mTimedOut; // Set if the expiration timer timed out. + bool mFinished; // Set by the curl thread to signal it finished. + bool mHandled; // Set when we processed the received data. + AITimer* mTimer; // Expiration timer. + F32 mRequestTimeOut; // The time out value for mTimer. - static F32 sCurlRequestTimeOut; // The time out value for mTimer. + static F32 sCurlRequestTimeOut; // The default time out value for mTimer (CurlRequestTimeOut debug setting). public: // Called once to set a different timeout then the default of 40 seconds. - static void setCurlRequestTimeOut(F32 CurlRequestTimeOut); + static void setDefaultRequestTimeOut(F32 defaultRequestTimeOut); + + // Called to set a specific time out, instead of the default one. + void setRequestTimeOut(F32 requestTimeOut); protected: // AICurlEasyRequest Events. @@ -77,6 +81,9 @@ class AICurlEasyRequestStateMachine : public AIStateMachine, public AICurlEasyHa // Called when this curl easy handle was added to a multi handle. /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat&); + // Called for each received HTTP header key/value pair. + /*virtual*/ void decoded_header(AICurlEasyRequest_wat&, std::string const& key, std::string const& value); + // Called when this curl easy handle finished processing (right before it is removed from the multi handle). /*virtual*/ void finished(AICurlEasyRequest_wat&); diff --git a/indra/aistatemachine/aicurlprivate.h b/indra/aistatemachine/aicurlprivate.h index a99224f96..d45b91fa1 100644 --- a/indra/aistatemachine/aicurlprivate.h +++ b/indra/aistatemachine/aicurlprivate.h @@ -267,6 +267,7 @@ class CurlEasyRequest : public CurlEasyHandle { protected: // Pass events to parent. /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void decoded_header(AICurlEasyRequest_wat& curl_easy_request_w, std::string const& key, std::string const& value); /*virtual*/ void finished(AICurlEasyRequest_wat& curl_easy_request_w); /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); }; @@ -292,10 +293,9 @@ class CurlResponderBuffer : protected AICurlEasyHandleEvents { void prepRequest(AICurlEasyRequest_wat& buffered_curl_easy_request_w, std::vector const& headers, AICurlInterface::ResponderPtr responder, S32 time_out = 0, bool post = false); LLIOPipe::buffer_ptr_t& getInput(void) { return mInput; } - std::stringstream& getHeaderOutput(void) { return mHeaderOutput; } LLIOPipe::buffer_ptr_t& getOutput(void) { return mOutput; } - // Called if libcurl doesn't deliver within CurlRequestTimeOut seconds. + // Called if libcurl doesn't deliver within mRequestTimeOut seconds. void timed_out(void); // Called after removed_from_multi_handle was called. @@ -306,16 +306,18 @@ class CurlResponderBuffer : protected AICurlEasyHandleEvents { protected: /*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); + /*virtual*/ void decoded_header(AICurlEasyRequest_wat& curl_easy_request_w, std::string const& key, std::string const& value); /*virtual*/ void finished(AICurlEasyRequest_wat& curl_easy_request_w); /*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w); private: LLIOPipe::buffer_ptr_t mInput; U8* mLastRead; // Pointer into mInput where we last stopped reading (or NULL to start at the beginning). - std::stringstream mHeaderOutput; LLIOPipe::buffer_ptr_t mOutput; AICurlInterface::ResponderPtr mResponder; //U32 mBodyLimit; // From the old LLURLRequestDetail::mBodyLimit, but never used. + U32 mStatus; // HTTP status, decoded from the first header line. + std::string mReason; // The "reason" from the same header line. S32 mRequestTransferedBytes; S32 mResponseTransferedBytes; @@ -334,6 +336,9 @@ class CurlResponderBuffer : protected AICurlEasyHandleEvents { static size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data); static size_t curlHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data); + // Called from curlHeaderCallback. + void setStatusAndReason(U32 status, std::string const& reason) { mStatus = status; mReason = reason; } + public: // Return pointer to the ThreadSafe (wrapped) version of this object. ThreadSafeBufferedCurlEasyRequest* get_lockobj(void); diff --git a/indra/llcommon/llstring.h b/indra/llcommon/llstring.h index bdbefe935..84f4d0f69 100644 --- a/indra/llcommon/llstring.h +++ b/indra/llcommon/llstring.h @@ -318,6 +318,7 @@ public: * should work. */ static void _makeASCII(std::basic_string& string); + static bool _isASCII(std::basic_string const& string); // Conversion to other data types static BOOL convertToBOOL(const std::basic_string& string, BOOL& value); @@ -1073,6 +1074,19 @@ void LLStringUtilBase::_makeASCII(std::basic_string& string) } } +template +bool LLStringUtilBase::_isASCII(std::basic_string const& string) +{ + size_type const len = string.length(); + T bit_collector = 0; + for (size_type i = 0; i < len; ++i) + { + bit_collector |= string[i]; + } + T const ascii_bits = 0x7f; + return !(bit_collector & ~ascii_bits); +} + // static template void LLStringUtilBase::copy( T* dst, const T* src, size_type dst_size ) diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index e954dcd1b..78d77db1d 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -251,7 +251,8 @@ static void request( return ; } - //AIFIXME: getCertVerifyCallback() always return NULL, so we might as well not do this call: req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req); + //AIFIXME: getCertVerifyCallback() always return NULL, so we might as well not do this call: + //req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req); lldebugs << LLURLRequest::actionAsVerb(method) << " " << url << " " << headers << llendl; @@ -283,50 +284,51 @@ static void request( lldebugs << "header = " << header.str() << llendl; req->addHeader(header.str().c_str()); } + + if (method == LLURLRequest::HTTP_PUT || method == LLURLRequest::HTTP_POST) + { + static std::string const CONTENT_TYPE("Content-Type"); + if(!headers.has(CONTENT_TYPE)) + { + // If the Content-Type header was passed in, it has + // already been added as a header through req->addHeader + // in the loop above. We defer to the caller's wisdom, but + // if they did not specify a Content-Type, then ask the + // injector. + req->addHeader(llformat("Content-Type: %s", body_injector->contentType()).c_str()); + } + } + else + { + // Check to see if we have already set Accept or not. If no one + // set it, set it to application/llsd+xml since that's what we + // almost always want. + static std::string const ACCEPT("Accept"); + if (!headers.has(ACCEPT)) + { + req->addHeader("Accept: application/llsd+xml"); + } + } } - // Check to see if we have already set Accept or not. If no one - // set it, set it to application/llsd+xml since that's what we - // almost always want. - if( method != LLURLRequest::HTTP_PUT && method != LLURLRequest::HTTP_POST ) + if (method == LLURLRequest::HTTP_POST && gMessageSystem) { - static const std::string ACCEPT("Accept"); - if(!headers.has(ACCEPT)) - { - req->addHeader("Accept: application/llsd+xml"); - } - } - - //AIFIXME: req->setCallback(new LLHTTPClientURLAdaptor(responder)); - llassert_always(false); - - if (method == LLURLRequest::HTTP_POST && gMessageSystem) - { - req->addHeader(llformat("X-SecondLife-UDP-Listen-Port: %d", - gMessageSystem->mPort).c_str()); + req->addHeader(llformat("X-SecondLife-UDP-Listen-Port: %d", gMessageSystem->mPort).c_str()); } if (method == LLURLRequest::HTTP_PUT || method == LLURLRequest::HTTP_POST) { - static const std::string CONTENT_TYPE("Content-Type"); - if(!headers.has(CONTENT_TYPE)) - { - // If the Content-Type header was passed in, it has - // already been added as a header through req->addHeader - // in the loop above. We defer to the caller's wisdom, but - // if they did not specify a Content-Type, then ask the - // injector. - req->addHeader( - llformat( - "Content-Type: %s", - body_injector->contentType()).c_str()); - } chain.push_back(LLIOPipe::ptr_t(body_injector)); } - //AIFIXEM: chain.push_back(LLIOPipe::ptr_t(req)); + //AIFIXME: chain.push_back(LLIOPipe::ptr_t(req)); - theClientPump->addChain(chain, timeout); + AICurlEasyRequest_wat buffered_easy_request_w(*req->mCurlEasyRequest); + AICurlResponderBuffer_wat buffer_w(*req->mCurlEasyRequest); + buffer_w->prepRequest(buffered_easy_request_w, headers, responder); + + req->setRequestTimeOut(timeout); + //AIFIXME: theClientPump->addChain(chain, timeout); } diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index e66dc3c1b..699214204 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -150,9 +150,7 @@ std::string LLURLRequest::actionAsVerb(LLURLRequest::ERequestAction action) LLURLRequest::LLURLRequest(LLURLRequest::ERequestAction action, std::string const& url) : AICurlEasyRequestStateMachine(true), mAction(action), mURL(url) { LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); - - //AIFIXME: start statemachine mState = STATE_INITIALIZED; - llassert_always(false); + run(); } #if 0