Add recovery for randomly closed socket desciptors.

Although it should never happen that a file descriptor is suddenly
closed, it appeared that this happens on linux 64bit when using
FMODex... Not really sure how useful this is, but at least now the
viewer just continues to work, as if -say- the socket was closed
remotely. Before the curl thread would go into a tight loop that it
wouldn't recover from until the watchdog thread terminated the viewer.

indra/llmessage/aicurl.cpp

@@ -1236,6 +1236,20 @@ void CurlEasyRequest::removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy
 	mHandleEventsTarget->removed_from_multi_handle(curl_easy_request_w);
 }
 
+void CurlEasyRequest::bad_file_descriptor(AICurlEasyRequest_wat& curl_easy_request_w)
+{
+  if (mHandleEventsTarget)
+	mHandleEventsTarget->bad_file_descriptor(curl_easy_request_w);
+}
+
+#ifdef SHOW_ASSERT
+void CurlEasyRequest::queued_for_removal(AICurlEasyRequest_wat& curl_easy_request_w)
+{
+  if (mHandleEventsTarget)
+	mHandleEventsTarget->queued_for_removal(curl_easy_request_w);
+}
+#endif
+
 PerHostRequestQueuePtr CurlEasyRequest::getPerHostPtr(void)
 {
   if (!mPerHostPtr)
@@ -1299,7 +1313,17 @@ void BufferedCurlEasyRequest::timed_out(void)
   mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR, "Request timeout, aborted.", sChannels, mOutput);
   if (mResponder->needsHeaders())
   {
-	send_buffer_events_to(NULL);	// Revoke buffer events: we sent them to the responder.
+	send_buffer_events_to(NULL);	// Revoke buffer events: we send them to the responder.
+  }
+  mResponder = NULL;
+}
+
+void BufferedCurlEasyRequest::bad_socket(void)
+{
+  mResponder->finished(CURLE_OK, HTTP_INTERNAL_ERROR, "File descriptor went bad! Aborted.", sChannels, mOutput);
+  if (mResponder->needsHeaders())
+  {
+	send_buffer_events_to(NULL);	// Revoke buffer events: we send them to the responder.
   }
   mResponder = NULL;
 }

indra/llmessage/aicurl.h

@@ -208,6 +208,10 @@ struct AICurlEasyHandleEvents {
 	virtual void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0;
 	virtual void finished(AICurlEasyRequest_wat& curl_easy_request_w) = 0;
 	virtual void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w) = 0;
+	virtual void bad_file_descriptor(AICurlEasyRequest_wat& curl_easy_request_w) = 0;
+#ifdef SHOW_ASSERT
+	virtual void queued_for_removal(AICurlEasyRequest_wat& curl_easy_request_w) = 0;
+#endif
 	// Avoid compiler warning.
 	virtual ~AICurlEasyHandleEvents() { }
 };

indra/llmessage/aicurleasyrequeststatemachine.cpp

@@ -40,7 +40,8 @@ enum curleasyrequeststatemachine_state_type {
   AICurlEasyRequestStateMachine_timedOut,	// This must be smaller than the rest, so they always overrule.
   AICurlEasyRequestStateMachine_finished,
   AICurlEasyRequestStateMachine_removed,	// The removed states must be largest two, so they are never ignored.
-  AICurlEasyRequestStateMachine_removed_after_finished
+  AICurlEasyRequestStateMachine_removed_after_finished,
+  AICurlEasyRequestStateMachine_bad_file_descriptor
 };
 
 char const* AICurlEasyRequestStateMachine::state_str_impl(state_type run_state) const
@@ -54,6 +55,7 @@ char const* AICurlEasyRequestStateMachine::state_str_impl(state_type run_state)
 	AI_CASE_RETURN(AICurlEasyRequestStateMachine_finished);
 	AI_CASE_RETURN(AICurlEasyRequestStateMachine_removed);
 	AI_CASE_RETURN(AICurlEasyRequestStateMachine_removed_after_finished);
+	AI_CASE_RETURN(AICurlEasyRequestStateMachine_bad_file_descriptor);
   }
   return "UNKNOWN STATE";
 }
@@ -94,6 +96,24 @@ void AICurlEasyRequestStateMachine::removed_from_multi_handle(AICurlEasyRequest_
   set_state(mFinished ? AICurlEasyRequestStateMachine_removed_after_finished : AICurlEasyRequestStateMachine_removed);
 }
 
+// CURL-THREAD
+void AICurlEasyRequestStateMachine::bad_file_descriptor(AICurlEasyRequest_wat&)
+{
+  if (!mFinished)
+  {
+	mFinished = true;
+	set_state(AICurlEasyRequestStateMachine_bad_file_descriptor);
+  }
+}
+
+#ifdef SHOW_ASSERT
+// CURL-THREAD
+void AICurlEasyRequestStateMachine::queued_for_removal(AICurlEasyRequest_wat&)
+{
+  llassert(mFinished || mTimedOut);		// See AICurlEasyRequestStateMachine::removed_from_multi_handle
+}
+#endif
+
 void AICurlEasyRequestStateMachine::multiplex_impl(void)
 {
   mSetStateLock.lock();
@@ -207,6 +227,11 @@ void AICurlEasyRequestStateMachine::multiplex_impl(void)
 
 	  break;
 	}
+	case AICurlEasyRequestStateMachine_bad_file_descriptor:
+	{
+	  AICurlEasyRequest_wat(*mCurlEasyRequest)->bad_socket();
+	  abort();
+	}
   }
 }
 

indra/llmessage/aicurleasyrequeststatemachine.h

@@ -81,6 +81,14 @@ class AICurlEasyRequestStateMachine : public AIStateMachine, public AICurlEasyHa
 	// Called after this curl easy handle was removed from a multi handle.
 	/*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat&);
 
+	// Called when the curl thread detected that the socket of this handle has become unusable.
+	/*virtual*/ void bad_file_descriptor(AICurlEasyRequest_wat&);
+
+#ifdef SHOW_ASSERT
+	// Called when a command was added to remove this easy handle.
+	/*virtual*/ void queued_for_removal(AICurlEasyRequest_wat&);
+#endif
+
   protected:
 	// AIStateMachine implementations.
 

indra/llmessage/aicurlprivate.h

@@ -420,6 +420,11 @@ class CurlEasyRequest : public CurlEasyHandle {
 	/*virtual*/ void added_to_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w);
 	/*virtual*/ void finished(AICurlEasyRequest_wat& curl_easy_request_w);
 	/*virtual*/ void removed_from_multi_handle(AICurlEasyRequest_wat& curl_easy_request_w);
+  public:
+	/*virtual*/ void bad_file_descriptor(AICurlEasyRequest_wat& curl_easy_request_w);
+#ifdef SHOW_ASSERT
+	/*virtual*/ void queued_for_removal(AICurlEasyRequest_wat& curl_easy_request_w);
+#endif
 };
 
 // This class adds input/output buffers to the request and hooks up the libcurl callbacks to use those buffers.
@@ -438,6 +443,9 @@ class BufferedCurlEasyRequest : public CurlEasyRequest {
 	// Called if libcurl doesn't deliver within AIHTTPTimeoutPolicy::mMaximumTotalDelay seconds.
 	void timed_out(void);
 
+	// Called if the underlaying socket went bad (ie, when accidently closed by a buggy library).
+	void bad_socket(void);
+
 	// Called after removed_from_multi_handle was called.
 	void processOutput(void);
 

indra/llmessage/aicurlthread.cpp

@@ -286,14 +286,36 @@ enum refresh_t {
   empty_and_complete = complete|empty
 };
 
+// A class with info for each socket that is in use by curl.
+class CurlSocketInfo
+{
+  public:
+	CurlSocketInfo(MultiHandle& multi_handle, CURL* easy, curl_socket_t s, int action, ThreadSafeBufferedCurlEasyRequest* lockobj);
+	~CurlSocketInfo();
+
+	void set_action(int action);
+	void mark_dead(void) { set_action(CURL_POLL_NONE); mDead = true; }
+	curl_socket_t getSocketFd(void) const { return mSocketFd; }
+	AICurlEasyRequest& getEasyRequest(void) { return mEasyRequest; }
+
+  private:
+	MultiHandle& mMultiHandle;
+	CURL const* mEasy;
+	curl_socket_t mSocketFd;
+	int mAction;
+	bool mDead;
+	AICurlEasyRequest mEasyRequest;
+	LLPointer<HTTPTimeout> mTimeout;
+};
+
 class PollSet
 {
   public:
 	PollSet(void);
 
 	// Add/remove a filedescriptor to/from mFileDescriptors.
-	void add(curl_socket_t s);
-	void remove(curl_socket_t s);
+	void add(CurlSocketInfo* sp);
+	void remove(CurlSocketInfo* sp);
 
 	// Copy mFileDescriptors to an internal fd_set that is returned by access().
 	// Returns if all fds could be copied (complete) and/or if the resulting fd_set is empty.
@@ -307,8 +329,8 @@ class PollSet
 	curl_socket_t get_max_fd(void) const { return mMaxFdSet; }
 #endif
 
-	// Return true if a filedescriptor is set in mFileDescriptors (used for debugging).
-	bool contains(curl_socket_t s) const;
+	// Return a pointer to the corresponding CurlSocketInfo if a filedescriptor is set in mFileDescriptors, or NULL if s is not set.
+	CurlSocketInfo* contains(curl_socket_t s) const;
 
 	// Return true if a filedescriptor is set in mFdSet.
 	bool is_set(curl_socket_t s) const;
@@ -323,7 +345,7 @@ class PollSet
 	void next(void);				// Advance to next filedescriptor.
 
   private:
-	curl_socket_t* mFileDescriptors;
+	CurlSocketInfo** mFileDescriptors;
 	int mNrFds;						// The number of filedescriptors in the array.
 	int mNext;						// The index of the first file descriptor to start copying, the next call to refresh().
 
@@ -332,8 +354,8 @@ class PollSet
 #if !WINDOWS_CODE
 	curl_socket_t mMaxFd;			// The largest filedescriptor in the array, or CURL_SOCKET_BAD when it is empty.
 	curl_socket_t mMaxFdSet;		// The largest filedescriptor set in mFdSet by refresh(), or CURL_SOCKET_BAD when it was empty.
-	std::vector<curl_socket_t> mCopiedFileDescriptors;	// Filedescriptors copied by refresh to mFdSet.
-	std::vector<curl_socket_t>::iterator mIter;			// Index into mCopiedFileDescriptors for next(); loop variable.
+	std::vector<CurlSocketInfo*> mCopiedFileDescriptors;	// Filedescriptors copied by refresh to mFdSet.
+	std::vector<CurlSocketInfo*>::iterator mIter;			// Index into mCopiedFileDescriptors for next(); loop variable.
 #else
 	unsigned int mIter;				// Index into fd_set::fd_array.
 #endif
@@ -359,7 +381,7 @@ class PollSet
 static size_t const MAXSIZE = llmax(1024, FD_SETSIZE);
 
 // Create an empty PollSet.
-PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [MAXSIZE]),
+PollSet::PollSet(void) : mFileDescriptors(new CurlSocketInfo* [MAXSIZE]),
                          mNrFds(0), mNext(0)
 #if !WINDOWS_CODE
 						 , mMaxFd(-1), mMaxFdSet(-1)
@@ -369,17 +391,17 @@ PollSet::PollSet(void) : mFileDescriptors(new curl_socket_t [MAXSIZE]),
 }
 
 // Add filedescriptor s to the PollSet.
-void PollSet::add(curl_socket_t s)
+void PollSet::add(CurlSocketInfo* sp)
 {
   llassert_always(mNrFds < (int)MAXSIZE);
-  mFileDescriptors[mNrFds++] = s;
+  mFileDescriptors[mNrFds++] = sp;
 #if !WINDOWS_CODE
-  mMaxFd = llmax(mMaxFd, s);
+  mMaxFd = llmax(mMaxFd, sp->getSocketFd());
 #endif
 }
 
 // Remove filedescriptor s from the PollSet.
-void PollSet::remove(curl_socket_t s)
+void PollSet::remove(CurlSocketInfo* sp)
 {
   // The number of open filedescriptors is relatively small,
   // and on top of that we rather do something CPU intensive
@@ -391,15 +413,15 @@ void PollSet::remove(curl_socket_t s)
   // back, keeping it compact and keeping the filedescriptors
   // in the same order (which is supposedly their priority).
   //
-  // The general case is where mFileDescriptors contains s at an index
+  // The general case is where mFileDescriptors contains sp at an index
   // between 0 and mNrFds:
   //                              mNrFds = 6
   //                                v
   // index: 0   1   2   3   4   5
   //        a   b   c   s   d   e
 
-  // This function should never be called unless s is actually in mFileDescriptors,
-  // as a result of a previous call to PollSet::add().
+  // This function should never be called unless sp is actually in mFileDescriptors,
+  // as a result of a previous call to PollSet::add(sp).
   llassert(mNrFds > 0);
 
   // Correct mNrFds for when the descriptor is removed.
@@ -409,17 +431,18 @@ void PollSet::remove(curl_socket_t s)
   //                            v
   // index: 0   1   2   3   4   5
   //        a   b   c   s   d   e
-  curl_socket_t cur = mFileDescriptors[i];		// cur = 'e'
+  curl_socket_t const s = sp->getSocketFd();
+  CurlSocketInfo* cur = mFileDescriptors[i];		// cur = 'e'
 #if !WINDOWS_CODE
   curl_socket_t max = -1;
 #endif
-  while (cur != s)
+  while (cur != sp)
   {
 	llassert(i > 0);
-	curl_socket_t next = mFileDescriptors[--i];	// next = 'd'
+	CurlSocketInfo* next = mFileDescriptors[--i];	// next = 'd'
 	mFileDescriptors[i] = cur;					// Overwrite 'd' with 'e'.
 #if !WINDOWS_CODE
-	max = llmax(max, cur);					// max is the maximum value in 'i' or higher.
+	max = llmax(max, cur->getSocketFd());		// max is the maximum value in 'i' or higher.
 #endif
 	cur = next;									// cur = 'd'
 	//                        i  NrFds = 5
@@ -427,21 +450,21 @@ void PollSet::remove(curl_socket_t s)
 	// index: 0   1   2   3   4
 	//        a   b   c   s   e					// cur = 'd'
 	//
-	// Next loop iteration: next = 's', overwrite 's' with 'd', cur = 's'; loop terminates.
+	// Next loop iteration: next = 'sp', overwrite 'sp' with 'd', cur = 'sp'; loop terminates.
 	//                    i      NrFds = 5
 	//                    v       v
 	// index: 0   1   2   3   4
-	//        a   b   c   d   e					// cur = 's'
+	//        a   b   c   d   e					// cur = 'sp'
   }
-  llassert(cur == s);
+  llassert(cur == sp);
   // At this point i was decremented once more and points to the element before the old s.
   //                i          NrFds = 5
   //                v           v
   // index: 0   1   2   3   4
   //        a   b   c   d   e					// max = llmax('d', 'e')
 
-  // If mNext pointed to an element before s, it should be left alone. Otherwise, if mNext pointed
-  // to s it must now point to 'd', or if it pointed beyond 's' it must be decremented by 1.
+  // If mNext pointed to an element before sp, it should be left alone. Otherwise, if mNext pointed
+  // to sp it must now point to 'd', or if it pointed beyond 'sp' it must be decremented by 1.
   if (mNext > i)								// i is where s was.
 	--mNext;
 
@@ -451,8 +474,8 @@ void PollSet::remove(curl_socket_t s)
   {
 	while (i > 0)
 	{
-	  curl_socket_t next = mFileDescriptors[--i];
-	  max = llmax(max, next);
+	  CurlSocketInfo* next = mFileDescriptors[--i];
+	  max = llmax(max, next->getSocketFd());
 	}
 	mMaxFd = max;
 	llassert(mMaxFd < s);
@@ -487,12 +510,12 @@ void PollSet::remove(curl_socket_t s)
 #endif
 }
 
-bool PollSet::contains(curl_socket_t fd) const
+CurlSocketInfo* PollSet::contains(curl_socket_t fd) const
 {
   for (int i = 0; i < mNrFds; ++i)
-	if (mFileDescriptors[i] == fd)
-	  return true;
-  return false;
+	if (mFileDescriptors[i]->getSocketFd() == fd)
+	  return mFileDescriptors[i];
+  return NULL;
 }
 
 inline bool PollSet::is_set(curl_socket_t fd) const
@@ -536,7 +559,7 @@ refresh_t PollSet::refresh(void)
 	// Calculate mMaxFdSet.
 	// Run over FD_SETSIZE - 1 elements, starting at mNext, wrapping to 0 when we reach the end.
 	int max = -1, i = mNext, count = 0;
-	while (++count < FD_SETSIZE) { max = llmax(max, mFileDescriptors[i]); if (++i == mNrFds) i = 0; }
+	while (++count < FD_SETSIZE) { max = llmax(max, mFileDescriptors[i]->getSocketFd()); if (++i == mNrFds) i = 0; }
 	mMaxFdSet = max;
 #endif
   }
@@ -556,7 +579,7 @@ refresh_t PollSet::refresh(void)
 	  mNext = i;
 	  return not_complete_not_empty;
 	}
-	FD_SET(mFileDescriptors[i], &mFdSet);
+	FD_SET(mFileDescriptors[i]->getSocketFd(), &mFdSet);
 #if !WINDOWS_CODE
 	mCopiedFileDescriptors.push_back(mFileDescriptors[i]);
 #endif
@@ -603,7 +626,7 @@ void PollSet::reset(void)
   else
   {
 	mIter = mCopiedFileDescriptors.begin();
-	if (!FD_ISSET(*mIter, &mFdSet))
+	if (!FD_ISSET((*mIter)->getSocketFd(), &mFdSet))
 	  next();
   }
 #endif
@@ -614,7 +637,7 @@ inline curl_socket_t PollSet::get(void) const
 #if WINDOWS_CODE
   return (mIter >= mFdSet.fd_count) ? CURL_SOCKET_BAD : mFdSet.fd_array[mIter];
 #else
-  return (mIter == mCopiedFileDescriptors.end()) ? CURL_SOCKET_BAD : *mIter;
+  return (mIter == mCopiedFileDescriptors.end()) ? CURL_SOCKET_BAD : (*mIter)->getSocketFd();
 #endif
 }
 
@@ -625,7 +648,7 @@ void PollSet::next(void)
   ++mIter;
 #else
   llassert(mIter != mCopiedFileDescriptors.end());	// Only call next() if the last call to get() didn't return -1.
-  while (++mIter != mCopiedFileDescriptors.end() && !FD_ISSET(*mIter, &mFdSet));
+  while (++mIter != mCopiedFileDescriptors.end() && !FD_ISSET((*mIter)->getSocketFd(), &mFdSet));
 #endif
 }
 
@@ -743,26 +766,8 @@ std::ostream& operator<<(std::ostream& os, DebugFdSet const& s)
 }
 #endif
 
-// A class with info for each socket that is in use by curl.
-class CurlSocketInfo
-{
-  public:
-	CurlSocketInfo(MultiHandle& multi_handle, CURL* easy, curl_socket_t s, int action, ThreadSafeBufferedCurlEasyRequest* lockobj);
-	~CurlSocketInfo();
-
-	void set_action(int action);
-
-  private:
-	MultiHandle& mMultiHandle;
-	CURL const* mEasy;
-	curl_socket_t mSocketFd;
-	int mAction;
-	AICurlEasyRequest mEasyRequest;
-	LLPointer<HTTPTimeout> mTimeout;
-};
-
 CurlSocketInfo::CurlSocketInfo(MultiHandle& multi_handle, CURL* easy, curl_socket_t s, int action, ThreadSafeBufferedCurlEasyRequest* lockobj) :
-    mMultiHandle(multi_handle), mEasy(easy), mSocketFd(s), mAction(CURL_POLL_NONE), mEasyRequest(lockobj)
+    mMultiHandle(multi_handle), mEasy(easy), mSocketFd(s), mAction(CURL_POLL_NONE), mDead(false), mEasyRequest(lockobj)
 {
   llassert(*AICurlEasyRequest_wat(*mEasyRequest) == easy);
   mMultiHandle.assign(s, this);
@@ -785,23 +790,28 @@ CurlSocketInfo::~CurlSocketInfo()
 
 void CurlSocketInfo::set_action(int action)
 {
+  if (mDead)
+  {
+	return;
+  }
+
   Dout(dc::curl, "CurlSocketInfo::set_action(" << action_str(mAction) << " --> " << action_str(action) << ") [" << (void*)mEasyRequest.get_ptr().get() << "]");
   int toggle_action = mAction ^ action; 
   mAction = action;
   if ((toggle_action & CURL_POLL_IN))
   {
 	if ((action & CURL_POLL_IN))
-	  mMultiHandle.mReadPollSet->add(mSocketFd);
+	  mMultiHandle.mReadPollSet->add(this);
 	else
-	  mMultiHandle.mReadPollSet->remove(mSocketFd);
+	  mMultiHandle.mReadPollSet->remove(this);
   }
   if ((toggle_action & CURL_POLL_OUT))
   {
 	if ((action & CURL_POLL_OUT))
-	  mMultiHandle.mWritePollSet->add(mSocketFd);
+	  mMultiHandle.mWritePollSet->add(this);
 	else
 	{
-	  mMultiHandle.mWritePollSet->remove(mSocketFd);
+	  mMultiHandle.mWritePollSet->remove(this);
 
 	  // The following is a bit of a hack, needed because of the lack of proper timeout callbacks in libcurl.
 	  // The removal of CURL_POLL_OUT could be part of the SSL handshake, therefore check if we're already connected:
@@ -1269,6 +1279,26 @@ void AICurlThread::process_commands(AICurlMultiHandle_wat const& multi_handle_w)
   }
 }
 
+// Return true if fd is a 'bad' socket.
+static bool is_bad(curl_socket_t fd, bool for_writing)
+{
+  fd_set tmp;
+  FD_ZERO(&tmp);
+  FD_SET(fd, &tmp);
+  fd_set* readfds = for_writing ? NULL : &tmp;
+  fd_set* writefds = for_writing ? &tmp : NULL;
+#if !WINDOWS_CODE
+  int nfds = fd + 1;
+#else
+  int nfds = 64;
+#endif
+  struct timeval timeout;
+  timeout.tv_sec = 0;
+  timeout.tv_usec = 10;
+  int ret = select(nfds, readfds, writefds, NULL, &timeout);
+  return ret == -1;
+}
+
 // The main loop of the curl thread.
 void AICurlThread::run(void)
 {
@@ -1280,7 +1310,7 @@ void AICurlThread::run(void)
 	{
 	  // If mRunning is true then we can only get here if mWakeUpFd != CURL_SOCKET_BAD.
 	  llassert(mWakeUpFd != CURL_SOCKET_BAD);
-	  // Copy the next batch of file descriptors from the PollSets mFiledescriptors into their mFdSet.
+	  // Copy the next batch of file descriptors from the PollSets mFileDescriptors into their mFdSet.
 	  multi_handle_w->mReadPollSet->refresh();
 	  refresh_t wres = multi_handle_w->mWritePollSet->refresh();
 	  // Add wake up fd if any, and pass NULL to select() if a set is empty.
@@ -1397,6 +1427,51 @@ void AICurlThread::run(void)
 	  if (ready == -1)
 	  {
 		llwarns << "select() failed: " << errno << ", " << strerror(errno) << llendl;
+		if (errno == EBADF)
+		{
+		  // Somewhere (fmodex?) one of our file descriptors was closed. Try to recover by finding out which.
+		  llassert_always(!is_bad(mWakeUpFd, false));		// We can't recover from this.
+		  PollSet* found = NULL;
+		  // Run over all read file descriptors.
+		  multi_handle_w->mReadPollSet->refresh();
+		  multi_handle_w->mReadPollSet->reset();
+		  curl_socket_t fd;
+		  while ((fd = multi_handle_w->mReadPollSet->get()) != CURL_SOCKET_BAD)
+		  {
+			if (is_bad(fd, false))
+			{
+			  found = multi_handle_w->mReadPollSet;
+			  break;
+			}
+			multi_handle_w->mReadPollSet->next();
+		  }
+		  if (!found)
+		  {
+			// Try all write file descriptors.
+			refresh_t wres = multi_handle_w->mWritePollSet->refresh();
+			if (!(wres & empty))
+			{
+			  multi_handle_w->mWritePollSet->reset();
+			  while ((fd = multi_handle_w->mWritePollSet->get()) != CURL_SOCKET_BAD)
+			  {
+				if (is_bad(fd, true))
+				{
+				  found = multi_handle_w->mWritePollSet;
+				  break;
+				}
+				multi_handle_w->mWritePollSet->next();
+			  }
+			}
+		  }
+		  llassert_always(found);	// It makes no sense to continue if we can't recover.
+		  // Find the corresponding CurlSocketInfo
+		  CurlSocketInfo* sp = found->contains(fd);
+		  llassert_always(sp);		// fd was just *read* from this sp.
+		  sp->mark_dead();													// Make sure it's never used again.
+		  AICurlEasyRequest_wat curl_easy_request_w(*sp->getEasyRequest());
+		  curl_easy_request_w->pause(CURLPAUSE_ALL);						// Keep libcurl at bay.
+		  curl_easy_request_w->bad_file_descriptor(curl_easy_request_w);	// Make the main thread cleanly terminate this transaction.
+		}
 		continue;
 	  }
 	  // Clock count used for timeouts.
@@ -2210,7 +2285,7 @@ void BufferedCurlEasyRequest::processOutput(void)
   CURLcode code;
   AITransferInfo info;
   getResult(&code, &info);
-  if (code == CURLE_OK)
+  if (code == CURLE_OK && mStatus != HTTP_INTERNAL_ERROR)
   {
 	getinfo(CURLINFO_RESPONSE_CODE, &responseCode);
 	// If getResult code is CURLE_OK then we should have decoded the first header line ourselves.
@@ -2223,7 +2298,7 @@ void BufferedCurlEasyRequest::processOutput(void)
   else
   {
 	responseCode = HTTP_INTERNAL_ERROR;
-	responseReason = curl_easy_strerror(code);
+	responseReason = (code == CURLE_OK) ? mReason : std::string(curl_easy_strerror(code));
 	setopt(CURLOPT_FRESH_CONNECT, TRUE);
   }
 
@@ -2604,6 +2679,14 @@ void AICurlEasyRequest::removeRequest(void)
 		}
 	  }
 	}
+	{
+	  AICurlEasyRequest_wat curl_easy_request_w(*get());
+	  // As soon as the lock on the command queue is released, it could be picked up by
+	  // the curl thread and executed. At that point it (already) demands that the easy
+	  // request either timed out or is finished. So, to avoid race conditions that already
+	  // has to be true right now. The call to queued_for_removal() checks this.
+	  curl_easy_request_w->queued_for_removal(curl_easy_request_w);
+	}
 #endif
 	// Add a command to remove this request from the multi session to the command queue.
 	command_queue_w->push_back(Command(*this, cmd_remove));