Be more verbose about internal (curl / timeout) errors.

Translates the CURLE_WRITE_ERROR into what it really means: that the low
speed check failed.

indra/llcommon/llhttpstatuscodes.h

@@ -90,6 +90,16 @@ const S32 HTTP_VERSION_NOT_SUPPORTED = 505;
 // These status codes should not be sent over the wire
 //   and indicate something went wrong internally.
 // If you get these they are not normal.
-const S32 HTTP_INTERNAL_ERROR = 499;
+// Note that these are only related to curl, not to webkit.
+const S32 HTTP_INTERNAL_ERROR_LOW_SPEED = 494;				// The transfer (receiving data) stalled or was too slow.
+const S32 HTTP_INTERNAL_ERROR_CURL_LOCKUP = 495;			// Curl never returned at all for 10 minutes!?!
+const S32 HTTP_INTERNAL_ERROR_CURL_BADSOCKET = 496;			// Curl was aborted because the socket went bad!?!
+const S32 HTTP_INTERNAL_ERROR_CURL_TIMEOUT = 497;			// Curl returned a timeout error.
+const S32 HTTP_INTERNAL_ERROR_CURL_OTHER = 498;				// Any other curl error.
+const S32 HTTP_INTERNAL_ERROR_OTHER = 499;					// Every other internal error.
+
+// Return true if status is an internal error (not received from a server but generated internally).
+bool inline is_internal_http_error(S32 status) { return status >= HTTP_INTERNAL_ERROR_LOW_SPEED && status <= HTTP_INTERNAL_ERROR_OTHER; }
+bool inline is_internal_http_error_that_warrants_a_retry(S32 status) { return status >= HTTP_INTERNAL_ERROR_LOW_SPEED && status <= HTTP_INTERNAL_ERROR_CURL_OTHER; }
 
 #endif

indra/llmessage/aicurl.cpp

@@ -873,13 +873,13 @@ void CurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* us
 static size_t noHeaderCallback(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
   llmaybewarns << "Calling noHeaderCallback(); curl session aborted." << llendl;
-  return 0;							// Cause a CURL_WRITE_ERROR
+  return 0;							// Cause a CURLE_WRITE_ERROR
 }
 
 static size_t noWriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata)
 {
   llmaybewarns << "Calling noWriteCallback(); curl session aborted." << llendl;
-  return 0;							// Cause a CURL_WRITE_ERROR
+  return 0;							// Cause a CURLE_WRITE_ERROR
 }
 
 static size_t noReadCallback(char* ptr, size_t size, size_t nmemb, void* userdata)
@@ -1277,7 +1277,7 @@ static int const HTTP_REDIRECTS_DEFAULT = 10;
 
 LLChannelDescriptors const BufferedCurlEasyRequest::sChannels;
 
-BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mResponseTransferedBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR)
+BufferedCurlEasyRequest::BufferedCurlEasyRequest() : mRequestTransferedBytes(0), mResponseTransferedBytes(0), mBufferEventsTarget(NULL), mStatus(HTTP_INTERNAL_ERROR_OTHER)
 {
   AICurlInterface::Stats::BufferedCurlEasyRequest_count++;
 }
@@ -1312,7 +1312,7 @@ BufferedCurlEasyRequest::~BufferedCurlEasyRequest()
 
 void BufferedCurlEasyRequest::timed_out(void)
 {
-  mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR, "Request timeout, aborted.", sChannels, mOutput);
+  mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR_CURL_LOCKUP, "Request timeout, aborted.", sChannels, mOutput);
   if (mResponder->needsHeaders())
   {
 	send_buffer_events_to(NULL);	// Revoke buffer events: we send them to the responder.
@@ -1322,7 +1322,7 @@ void BufferedCurlEasyRequest::timed_out(void)
 
 void BufferedCurlEasyRequest::bad_socket(void)
 {
-  mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR, "File descriptor went bad! Aborted.", sChannels, mOutput);
+  mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR_CURL_BADSOCKET, "File descriptor went bad! Aborted.", sChannels, mOutput);
   if (mResponder->needsHeaders())
   {
 	send_buffer_events_to(NULL);	// Revoke buffer events: we send them to the responder.
@@ -1343,7 +1343,7 @@ void BufferedCurlEasyRequest::resetState(void)
   mRequestTransferedBytes = 0;
   mResponseTransferedBytes = 0;
   mBufferEventsTarget = NULL;
-  mStatus = HTTP_INTERNAL_ERROR;
+  mStatus = HTTP_INTERNAL_ERROR_OTHER;
 }
 
 void BufferedCurlEasyRequest::print_diagnostics(CURLcode code)

indra/llmessage/aicurlprivate.h

@@ -435,7 +435,7 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
 	ThreadSafeBufferedCurlEasyRequest const* get_lockobj(void) const;
 	// Return true when an error code was received that can occur before the upload finished.
 	// So far the only such error I've seen is HTTP_BAD_REQUEST.
-	bool upload_error_status(void) const { return mStatus == HTTP_BAD_REQUEST /*&& mStatus != HTTP_INTERNAL_ERROR*/; }
+	bool upload_error_status(void) const { return mStatus == HTTP_BAD_REQUEST; }
 
 	// Return true when prepRequest was already called and the object has not been
 	// invalidated as a result of calling timed_out().

indra/llmessage/aicurlthread.cpp

@@ -1937,7 +1937,7 @@ void BufferedCurlEasyRequest::setStatusAndReason(U32 status, std::string const&
   // Sanity check. If the server replies with a redirect status then we better have that option turned on!
   if ((status >= 300 && status < 400) && mResponder && !mResponder->redirect_status_ok())
   {
-	llerrs << "Received " << status << " (" << reason << ") for responder \"" << mTimeoutPolicy->name() << "\" which has no followRedir()!" << llendl;
+	llerrs << "Received " << status << " (" << reason << ") for responder \"" << mResponder->getName() << "\" which has no followRedir()!" << llendl;
   }
 }
 
@@ -1949,7 +1949,7 @@ void BufferedCurlEasyRequest::processOutput(void)
   CURLcode code;
   AITransferInfo info;
   getResult(&code, &info);
-  if (code == CURLE_OK && mStatus != HTTP_INTERNAL_ERROR)
+  if (code == CURLE_OK && !is_internal_http_error(mStatus))
   {
 	getinfo(CURLINFO_RESPONSE_CODE, &responseCode);
 	// If getResult code is CURLE_OK then we should have decoded the first header line ourselves.
@@ -1961,8 +1961,30 @@ void BufferedCurlEasyRequest::processOutput(void)
   }
   else
   {
-	responseCode = HTTP_INTERNAL_ERROR;
 	responseReason = (code == CURLE_OK) ? mReason : std::string(curl_easy_strerror(code));
+	switch (code)
+	{
+	  case CURLE_FAILED_INIT:
+		responseCode = HTTP_INTERNAL_ERROR_OTHER;
+		break;
+	  case CURLE_OPERATION_TIMEDOUT:
+		responseCode = HTTP_INTERNAL_ERROR_CURL_TIMEOUT;
+		break;
+	  case CURLE_WRITE_ERROR:
+		responseCode = HTTP_INTERNAL_ERROR_LOW_SPEED;
+		break;
+	  default:
+		responseCode = HTTP_INTERNAL_ERROR_CURL_OTHER;
+		break;
+	}
+	if (responseCode == HTTP_INTERNAL_ERROR_LOW_SPEED)
+	{
+		// Rewrite error to something understandable.
+		responseReason = llformat("Download stalled: received less than %u bytes in %u seconds (in total received %u bytes, using responder %s). "
+			"To change these values, tune debug settings CurlTimeoutLowSpeedLimit and CurlTimeoutLowSpeedTime respectively.",
+			mResponder->getHTTPTimeoutPolicy().getLowSpeedLimit(), mResponder->getHTTPTimeoutPolicy().getLowSpeedTime(),
+			mResponseTransferedBytes, mResponder->getName());
+	}
 	setopt(CURLOPT_FRESH_CONNECT, TRUE);
   }
 
@@ -2099,7 +2121,7 @@ size_t BufferedCurlEasyRequest::curlHeaderCallback(char* data, size_t size, size
 	  }
 	  // Either way, this status value is not understood (or taken into account).
 	  // Set it to internal error so that the rest of code treats it as an error.
-	  status = HTTP_INTERNAL_ERROR;
+	  status = HTTP_INTERNAL_ERROR_OTHER;
 	}
 	self_w->received_HTTP_header();
 	self_w->setStatusAndReason(status, reason);

indra/llmessage/aihttptimeout.cpp

@@ -294,6 +294,7 @@ bool HTTPTimeout::lowspeed(size_t bytes)
 #endif
 		"aborting slow connection (average transfer rate below " << low_speed_limit <<
 		" for more than " << low_speed_time << " second" << ((low_speed_time == 1) ? "" : "s") << ")." << llendl;
+	  // This causes curl to exit with CURLE_WRITE_ERROR.
 	  return true;
 	}
   }

indra/llmessage/llhttpclient.cpp

@@ -304,7 +304,7 @@ AIHTTPTimeoutPolicy const& LLHTTPClient::ResponderBase::getHTTPTimeoutPolicy(voi
 void LLHTTPClient::ResponderBase::decode_llsd_body(U32 status, std::string const& reason, LLChannelDescriptors const& channels, buffer_ptr_t const& buffer, LLSD& content)
 {
 	AICurlInterface::Stats::llsd_body_count++;
-	if (status == HTTP_INTERNAL_ERROR)
+	if (is_internal_http_error(status))
 	{
 		// In case of an internal error (ie, a curl error), a description of the (curl) error is the best we can do.
 		// In any case, the body if anything was received at all, can not be relied upon.
@@ -355,7 +355,7 @@ void LLHTTPClient::ResponderBase::decode_llsd_body(U32 status, std::string const
 void LLHTTPClient::ResponderBase::decode_raw_body(U32 status, std::string const& reason, LLChannelDescriptors const& channels, buffer_ptr_t const& buffer, std::string& content)
 {
 	AICurlInterface::Stats::raw_body_count++;
-	if (status == HTTP_INTERNAL_ERROR)
+	if (is_internal_http_error(status))
 	{
 		// In case of an internal error (ie, a curl error), a description of the (curl) error is the best we can do.
 		// In any case, the body if anything was received at all, can not be relied upon.
@@ -612,11 +612,10 @@ static LLSD blocking_request(
 
 	responder->wait();
 
-	S32 http_status = HTTP_INTERNAL_ERROR;
 	LLSD response = LLSD::emptyMap();
 	CURLcode result = responder->result_code();
+	S32 http_status = responder->http_status();
 
-	http_status = responder->http_status();
 	bool http_success = http_status >= 200 && http_status < 300;
 	if (result == CURLE_OK && http_success)
 	{

indra/llmessage/llhttpclient.h

@@ -380,7 +380,7 @@ public:
 		}
 
 	public:
-		LegacyPolledResponder(void) : mStatus(HTTP_INTERNAL_ERROR) { }
+		LegacyPolledResponder(void) : mStatus(HTTP_INTERNAL_ERROR_OTHER) { }
 
 		// Accessors.
 		U32 http_status(void) const { return mStatus; }

indra/llmessage/llurlrequest.cpp

@@ -47,7 +47,6 @@
 #include "llscopedvolatileaprpool.h"
 #include "llfasttimer.h"
 #include "message.h"
-static const U32 HTTP_STATUS_PIPE_ERROR = 499;
 
 /**
  * String constants

indra/newview/llinventorymodelbackgroundfetch.cpp

@@ -547,7 +547,7 @@ void LLInventoryModelFetchDescendentsResponder::error(U32 status, const std::str
 						
 	fetcher->incrFetchCount(-1);
 
-	if (status==499) // timed out
+	if (is_internal_http_error_that_warrants_a_retry(status)) // timed out
 	{
 		for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray();
 			folder_it != mRequestSD["folders"].endArray();

indra/newview/llmarketplacefunctions.cpp

@@ -172,6 +172,7 @@ namespace LLMarketplaceImport
 
 			if ((status == MarketplaceErrorCodes::IMPORT_REDIRECT) ||
 				(status == MarketplaceErrorCodes::IMPORT_AUTHENTICATION_ERROR) ||
+				(status == MarketplaceErrorCodes::IMPORT_JOB_LOW_SPEED) ||
 				(status == MarketplaceErrorCodes::IMPORT_JOB_TIMEOUT))
 			{
 				if (gSavedSettings.getBOOL("InventoryOutboxLogging"))
@@ -228,6 +229,7 @@ namespace LLMarketplaceImport
 			}
 
 			if ((status == MarketplaceErrorCodes::IMPORT_AUTHENTICATION_ERROR) ||
+				(status == MarketplaceErrorCodes::IMPORT_JOB_LOW_SPEED) ||
 				(status == MarketplaceErrorCodes::IMPORT_JOB_TIMEOUT))
 			{
 				if (gSavedSettings.getBOOL("InventoryOutboxLogging"))

indra/newview/llmarketplacefunctions.h

@@ -35,6 +35,7 @@
 
 #include "llsingleton.h"
 #include "llstring.h"
+#include "llhttpstatuscodes.h"
 
 
 LLSD getMarketplaceStringSubstitutions();
@@ -44,13 +45,14 @@ namespace MarketplaceErrorCodes
 {
 	enum eCode
 	{
-		IMPORT_DONE = 200,
+		IMPORT_DONE = HTTP_OK,
-		IMPORT_PROCESSING = 202,
+		IMPORT_PROCESSING = HTTP_ACCEPTED,
-		IMPORT_REDIRECT = 302,
+		IMPORT_REDIRECT = HTTP_FOUND,
-		IMPORT_AUTHENTICATION_ERROR = 401,
+		IMPORT_AUTHENTICATION_ERROR = HTTP_UNAUTHORIZED,
-		IMPORT_DONE_WITH_ERRORS = 409,
+		IMPORT_DONE_WITH_ERRORS = HTTP_CONFLICT,
-		IMPORT_JOB_FAILED = 410,
+		IMPORT_JOB_FAILED = HTTP_GONE,
-		IMPORT_JOB_TIMEOUT = 499,
+		IMPORT_JOB_LOW_SPEED = HTTP_INTERNAL_ERROR_LOW_SPEED,
+		IMPORT_JOB_TIMEOUT = HTTP_INTERNAL_ERROR_CURL_TIMEOUT
 	};
 }
 

indra/newview/llmeshrepository.cpp

@@ -1780,7 +1780,7 @@ void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason,
 
 	if (data_size < (S32)mRequestedBytes)
 	{
-		if (status == 499 || status == 503)
+		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//timeout or service unavailable, try again
 			LLMeshRepository::sHTTPRetryCount++;
 			gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD);
@@ -1834,7 +1834,7 @@ void LLMeshSkinInfoResponder::completedRaw(U32 status, const std::string& reason
 
 	if (data_size < (S32)mRequestedBytes)
 	{
-		if (status == 499 || status == 503)
+		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//timeout or service unavailable, try again
 			LLMeshRepository::sHTTPRetryCount++;
 			gMeshRepo.mThread->loadMeshSkinInfo(mMeshID);
@@ -1888,7 +1888,7 @@ void LLMeshDecompositionResponder::completedRaw(U32 status, const std::string& r
 
 	if (data_size < (S32)mRequestedBytes)
 	{
-		if (status == 499 || status == 503)
+		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//timeout or service unavailable, try again
 			LLMeshRepository::sHTTPRetryCount++;
 			gMeshRepo.mThread->loadMeshDecomposition(mMeshID);
@@ -1942,7 +1942,7 @@ void LLMeshPhysicsShapeResponder::completedRaw(U32 status, const std::string& re
 
 	if (data_size < (S32)mRequestedBytes)
 	{
-		if (status == 499 || status == 503)
+		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//timeout or service unavailable, try again
 			LLMeshRepository::sHTTPRetryCount++;
 			gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID);
@@ -1993,13 +1993,13 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason,
 		//	<< "Header responder failed with status: "
 		//	<< status << ": " << reason << llendl;
 
-		// 503 (service unavailable) or 499 (timeout)
+		// HTTP_SERVICE_UNAVAILABLE (503) or HTTP_INTERNAL_ERROR_*'s.
 		// can be due to server load and can be retried
 
 		// TODO*: Add maximum retry logic, exponential backoff
 		// and (somewhat more optional than the others) retries
 		// again after some set period of time
-		if (status == 503 || status == 499)
+		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//retry
 			LLMeshRepository::sHTTPRetryCount++;
 			LLMeshRepoThread::HeaderRequest req(mMeshParams);

indra/newview/llpanelplace.cpp

@@ -203,16 +203,25 @@ void LLPanelPlace::setLandTypeString(const std::string& land_type)
 
 void LLPanelPlace::setErrorStatus(U32 status, const std::string& reason)
 {
-	// We only really handle 404 and 499 errors
+	// We only really handle 404 and timeout errors
 	std::string error_text;
-	if(status == 404)
+	if (status == HTTP_NOT_FOUND)
 	{	
 		error_text = getString("server_error_text");
 	}
-	else if(status == 499)
+	else if (status == HTTP_UNAUTHORIZED)		// AIFIXME: Is this indeed the error we get when we don't have access rights for this?
 	{
 		error_text = getString("server_forbidden_text");
 	}
+	else if (status == HTTP_INTERNAL_ERROR_LOW_SPEED || status == HTTP_INTERNAL_ERROR_CURL_TIMEOUT)
+	{
+		error_text = getString("internal_timeout_text");
+	}
+	else
+	{
+		llwarns << "Unexpected error (" << status << "): " << reason << llendl;
+		error_text = llformat("Unexpected Error (%u): %s", status, reason.c_str());
+	}
 	mDescEditor->setText(error_text);
 }
 

indra/newview/lltexturefetch.cpp

@@ -1369,14 +1369,21 @@ bool LLTextureFetchWorker::doWork(S32 param)
 			if (mRequestedSize < 0)
 			{
 				S32 max_attempts;
-				if (mGetStatus == HTTP_NOT_FOUND || mGetStatus == 499)
+				if (mGetStatus == HTTP_NOT_FOUND || mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT || mGetStatus == HTTP_INTERNAL_ERROR_LOW_SPEED)
 				{
 					mHTTPFailCount = max_attempts = 1; // Don't retry
 					if(mGetStatus == HTTP_NOT_FOUND)
 						llwarns << "Texture missing from server (404): " << mUrl << llendl;
-					else if (mGetStatus == 499) 
+					else if (mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT || mGetStatus == HTTP_INTERNAL_ERROR_LOW_SPEED)
 					{
-						llwarns << "No response from server (499): " << mUrl << llendl;
+						if (mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT)
+						{
+							llwarns << "No response from server (HTTP_INTERNAL_ERROR_CURL_TIMEOUT): " << mUrl << llendl;
+						}
+						else
+						{
+							llwarns << "Slow response from server (HTTP_INTERNAL_ERROR_LOW_SPEED): " << mUrl << llendl;
+						}
 						SGHostBlackList::add(mUrl, 60.0, mGetStatus);
 					}
 					//roll back to try UDP

indra/newview/llwebprofile.cpp

@@ -97,7 +97,7 @@ public:
 		}
 
 		// *TODO: 404 = not supported by the grid
-		// *TODO: increase timeout or handle 499 Expired
+		// *TODO: increase timeout or handle HTTP_INTERNAL_ERROR_* time errors.
 
 		// Convert config to LLSD.
 		const Json::Value data = root["data"];

indra/newview/skins/default/xui/en-us/panel_place.xml

@@ -52,4 +52,7 @@
 	<string name="server_forbidden_text">
 		Information about this location is unavailable due to access restrictions.  Please check your permissions with the parcel owner.
 	</string>
+	<string name="internal_timeout_text">
+		Information about this location is unavailable due to a network timeout, please try again later.
+	</string>
 </panel>