Added more verbose statemachine debug output.

indra/aistatemachine/aistatemachine.cpp

@@ -293,7 +293,36 @@ void AIEngine::add(AIStateMachine* state_machine)
   }
 }
 
-extern void print_statemachine_diagnostics(U64 total_clocks, U64 max_delta, AIEngine::queued_type::const_reference slowest_state_machine);
+#if STATE_MACHINE_PROFILING
+// Called from AIStateMachine::mainloop
+void print_statemachine_diagnostics(U64 total_clocks, AIStateMachine::StateTimerBase::TimeData& slowest_timer, AIEngine::queued_type::const_reference slowest_element)
+{
+	AIStateMachine const& slowest_state_machine = slowest_element.statemachine();
+	F64 const tfactor = 1000 / calc_clock_frequency();
+	std::ostringstream msg;
+
+	U64 max_delta = slowest_timer.GetDuration();
+
+	if (total_clocks > max_delta)
+	{
+		msg << "AIStateMachine::mainloop did run for " << (total_clocks * tfactor) << " ms. The slowest ";
+	}
+	else
+	{
+		msg << "AIStateMachine::mainloop: A ";
+	}
+	msg << "state machine " << "(" << slowest_state_machine.getName() << ") " << "ran for " << (max_delta * tfactor) << " ms";
+	if (slowest_state_machine.getRuntime() > max_delta)
+	{
+		msg << " (" << (slowest_state_machine.getRuntime() * tfactor) << " ms in total now)";
+	}
+	msg << ".\n";
+
+	AIStateMachine::StateTimerBase::DumpTimers(msg);
+
+	llwarns << msg.str() << llendl;
+}
+#endif
 
 // MAIN-THREAD
 void AIEngine::mainloop(void)
@@ -305,28 +334,33 @@ void AIEngine::mainloop(void)
 	queued_element = engine_state_w->list.begin();
   }
   U64 total_clocks = 0;
-#ifndef LL_RELEASE_FOR_DOWNLOAD
-  U64 max_delta = 0;
+#if STATE_MACHINE_PROFILING
   queued_type::value_type slowest_element(NULL);
+  AIStateMachine::StateTimerRoot::TimeData slowest_timer;
 #endif
   while (queued_element != end)
   {
 	AIStateMachine& state_machine(queued_element->statemachine());
-	U64 start = get_clock_count();
-	if (!state_machine.sleep(start))
+	AIStateMachine::StateTimerBase::TimeData time_data;
+	if (!state_machine.sleep(get_clock_count()))
 	{
-	  state_machine.multiplex(AIStateMachine::normal_run);
+		AIStateMachine::StateTimerRoot timer(state_machine.getName());
+		state_machine.multiplex(AIStateMachine::normal_run);
+		time_data = timer.GetTimerData();
 	}
-	U64 delta = get_clock_count() - start;
-	state_machine.add(delta);
-	total_clocks += delta;
-#ifndef LL_RELEASE_FOR_DOWNLOAD
-	if (delta > max_delta)
+	if (U64 delta = time_data.GetDuration())
 	{
-	  max_delta = delta;
-	  slowest_element = *queued_element;
-	}
+		state_machine.add(delta);
+		total_clocks += delta;
+#if STATE_MACHINE_PROFILING
+		if (delta > slowest_timer.GetDuration())
+		{
+			slowest_element = *queued_element;
+			slowest_timer = time_data;
+		}
 #endif
+	}
+
 	bool active = state_machine.active(this);		// This locks mState shortly, so it must be called before locking mEngineState because add() locks mEngineState while holding mState.
 	engine_state_type_wat engine_state_w(mEngineState);
 	if (!active)
@@ -340,8 +374,8 @@ void AIEngine::mainloop(void)
 	}
 	if (total_clocks >= sMaxCount)
 	{
-#ifndef LL_RELEASE_FOR_DOWNLOAD
-	  print_statemachine_diagnostics(total_clocks, max_delta, slowest_element);
+#if STATE_MACHINE_PROFILING
+		print_statemachine_diagnostics(total_clocks, slowest_timer, slowest_element);
 #endif
 	  Dout(dc::statemachine, "Sorting " << engine_state_w->list.size() << " state machines.");
 	  engine_state_w->list.sort(QueueElementComp());
@@ -752,6 +786,22 @@ void AIStateMachine::multiplex(event_type event)
   }
 }
 
+#if STATE_MACHINE_PROFILING
+std::vector<AIStateMachine::StateTimerBase*> AIStateMachine::StateTimerBase::mTimerStack;
+AIStateMachine::StateTimerBase::TimeData AIStateMachine::StateTimerBase::TimeData::sRoot("");
+void AIStateMachine::StateTimer::TimeData::DumpTimer(std::ostringstream& msg, std::string prefix)
+{
+	F64 const tfactor = 1000 / calc_clock_frequency();
+	msg << prefix << mName << " " << (mEnd - mStart)*tfactor << "ms" << std::endl;
+	prefix.push_back(' ');
+	std::vector<TimeData>::iterator it;
+	for (it = mChildren.begin(); it != mChildren.end(); ++it)
+	{
+		it->DumpTimer(msg, prefix);
+	}
+}
+#endif
+
 AIStateMachine::state_type AIStateMachine::begin_loop(base_state_type base_state)
 {
   DoutEntering(dc::statemachine(mSMDebug), "AIStateMachine::begin_loop(" << state_str(base_state) << ") [" << (void*)this << "]");

indra/aistatemachine/aistatemachine.h

@@ -36,6 +36,7 @@
 
 #include "aithreadsafe.h"
 #include <llpointer.h>
+#include "lltimer.h"
 #include <list>
 #include <boost/signals2.hpp>
 
@@ -98,11 +99,140 @@ class AIEngine
 extern AIEngine gMainThreadEngine;
 extern AIEngine gStateMachineThreadEngine;
 
+#ifndef STATE_MACHINE_PROFILING
+#define STATE_MACHINE_PROFILING (LL_RELEASE_FOR_DOWNLOAD)
+#endif
+
 class AIStateMachine : public LLThreadSafeRefCount
 {
   public:
 	typedef U32 state_type;		//!< The type of run_state
 
+	// A simple timer class that will calculate time delta between ctor and GetTimerData call.
+	// Time data is stored as a nested TimeData object.
+	// If STATE_MACHINE_PROFILING is defined then a stack of all StateTimers from root is maintained for debug output.
+	class StateTimerBase
+	{
+	public:
+		class TimeData
+		{
+			friend class StateTimerBase;
+		public:
+			TimeData() : mStart(-1), mEnd(-1) {}
+			U64 GetDuration() {	return mEnd - mStart; }
+		private:
+			U64 mStart, mEnd;
+
+#if !STATE_MACHINE_PROFILING
+			TimeData(const std::string& name) : mStart(get_clock_count()), mEnd(get_clock_count()) {}
+#else
+			TimeData(const std::string& name) : mName(name), mStart(get_clock_count()), mEnd(get_clock_count()) {}
+			void DumpTimer(std::ostringstream& msg, std::string prefix);
+			std::vector<TimeData> mChildren;
+			std::string mName;	
+			static TimeData sRoot;
+#endif
+		};
+	protected:
+#if !STATE_MACHINE_PROFILING
+		StateTimerBase(const std::string& name) : mData(name) {}
+		~StateTimerBase() {}
+		TimeData mData;
+		// Return a copy of the underlying timer data.
+		// This allows the data live beyond the scope of the state timer.
+	public:
+		const TimeData GetTimerData()
+		{
+			mData.mEnd = get_clock_count();	//set mEnd to current time, since GetTimerData() will always be called before the dtor, obv.
+			return mData;
+		}
+#else
+		// Ctors/dtors are hidden. Only StateTimerRoot and StateTimer are permitted to access them.
+		StateTimerBase() : mData(NULL) {}
+		~StateTimerBase()
+		{
+			// If mData is null then the timer was not registered due to being in the wrong thread or the root timer wasn't in the expected state.
+			if (!mData)
+				return;
+			mData->mEnd = get_clock_count();
+			mTimerStack.pop_back();
+		}
+
+		// Also hide internals from everything except StateTimerRoot and StateTimer
+		bool AddAsRoot(const std::string& name)
+		{
+			
+			if (!is_main_thread())
+				return true;	//Ignoring this timer, but pretending it was added.
+			if (!mTimerStack.empty())
+				return false;
+			TimeData::sRoot = TimeData(name);
+			mData = &TimeData::sRoot;
+			mData->mChildren.clear();	
+			mTimerStack.push_back(this);
+			return true;
+		}
+		bool AddAsChild(const std::string& name)
+		{
+			if (!is_main_thread())
+				return true;	//Ignoring this timer, but pretending it was added.
+			if (mTimerStack.empty())
+				return false;
+			mTimerStack.back()->mData->mChildren.push_back(TimeData(name));
+			mData = &mTimerStack.back()->mData->mChildren.back();
+			mTimerStack.push_back(this);
+			return true;
+		}
+
+		TimeData* mData;
+		static std::vector<StateTimerBase*> mTimerStack;
+
+	public:
+		// Debug spew
+		static void DumpTimers(std::ostringstream& msg)
+		{
+			TimeData::sRoot.DumpTimer(msg, "");
+		}
+
+		// Return a copy of the underlying timer data.
+		// This allows the data live beyond the scope of the state timer.
+		const TimeData GetTimerData() const
+		{
+			if (mData)
+			{
+				TimeData ret = *mData;
+				ret.mEnd = get_clock_count();	//set mEnd to current time, since GetTimerData() will always be called before the dtor, obv.
+				return ret;
+			}
+			return TimeData();
+		}
+#endif
+	};
+public:
+#if !STATE_MACHINE_PROFILING
+	typedef StateTimerBase StateTimerRoot;
+	typedef StateTimerBase StateTimer;
+#else
+	class StateTimerRoot : public StateTimerBase
+	{ //A StateTimerRoot can become a child if a root already exists.
+	public:
+		StateTimerRoot(const std::string& name)
+		{
+			if(!AddAsRoot(name))
+				AddAsChild(name);	
+		}
+	};
+	class StateTimer : public StateTimerBase
+	{ //A StateTimer can never become a root
+	public:
+		StateTimer(const std::string& name)
+		{
+			AddAsChild(name);
+		}
+	};
+#endif
+
+
   protected:
 	// The type of event that causes multiplex() to be called.
 	enum event_type {
@@ -302,6 +432,9 @@ class AIStateMachine : public LLThreadSafeRefCount
 	void add(U64 count) { mRuntime += count; }
 	U64 getRuntime(void) const { return mRuntime; }
 
+	// For diagnostics. Every derived class must override this.
+	virtual const char* getName() const = 0;
+
   protected:
 	virtual void initialize_impl(void) = 0;
 	virtual void multiplex_impl(state_type run_state) = 0;

indra/aistatemachine/aistatemachinethread.h

@@ -232,6 +232,13 @@ class AIStateMachineThread : public AIStateMachineThreadBase {
 	// Accessor.
 	THREAD_IMPL& thread_impl(void) { return mThreadImpl; }
 
+	/*virtual*/ const char* getName() const
+	{
+#define STRIZE(arg) #arg
+		return "AIStateMachineThread<"STRIZE(THREAD_IMPL)">";
+#undef STRIZE
+	}
+
   protected:
 	/*virtual*/ AIThreadImpl& impl(void) { return mThreadImpl; }
 };

indra/aistatemachine/aitimer.h

@@ -98,6 +98,8 @@ class AITimer : public AIStateMachine {
 	 */
 	F64 getInterval(void) const { return mInterval; }
 
+	/*virtual*/ const char* getName() const { return "AITimer"; }
+
   protected:
 	// Call finish() (or abort()), not delete.
 	/*virtual*/ ~AITimer() { DoutEntering(dc::statemachine(mSMDebug), "~AITimer() [" << (void*)this << "]"); mFrameTimer.cancel(); }

indra/llmessage/llurlrequest.cpp

@@ -78,7 +78,7 @@ LLURLRequest::LLURLRequest(LLURLRequest::ERequestAction action, std::string cons
 	LLHTTPClient::ResponderPtr responder, AIHTTPHeaders& headers, AIPerService::Approvement* approved,
 	bool keepalive, bool is_auth, bool compression) :
     mAction(action), mURL(url), mKeepAlive(keepalive), mIsAuth(is_auth), mNoCompression(!compression),
-	mBody(body), mResponder(responder), mHeaders(headers), mResponderNameCache(responder ? responder->getName() : "<uninitialized>")
+	mBody(body), mResponder(responder), mHeaders(headers), mResponderNameCache(std::string("LLURLRequest:") + std::string(responder ? responder->getName() : "<uninitialized>"))
 {
 	if (approved)
 	{
@@ -276,32 +276,4 @@ bool LLURLRequest::configure(AICurlEasyRequest_wat const& curlEasyRequest_w)
 	return rv;
 }
 
-// Called from AIStateMachine::mainloop, but put here because we don't want to include llurlrequest.h there of course.
-void print_statemachine_diagnostics(U64 total_clocks, U64 max_delta, AIEngine::queued_type::const_reference slowest_element)
-{
-  AIStateMachine const& slowest_state_machine = slowest_element.statemachine();
-  LLURLRequest const* request = dynamic_cast<LLURLRequest const*>(&slowest_state_machine);
-  F64 const tfactor = 1000 / calc_clock_frequency();
-  std::ostringstream msg;
-  if (total_clocks > max_delta)
-  {
-	  msg << "AIStateMachine::mainloop did run for " << (total_clocks * tfactor) << " ms. The slowest ";
-  }
-  else
-  {
-	  msg << "AIStateMachine::mainloop: A ";
-  }
-  msg << "state machine ";
-  if (request)
-  {
-	  msg << "(" << request->getResponderName() << ") ";
-  }
-  msg << "ran for " << (max_delta * tfactor) << " ms";
-  if (slowest_state_machine.getRuntime() > max_delta)
-  {
-	  msg << " (" << (slowest_state_machine.getRuntime() * tfactor) << " ms in total now)";
-  }
-  msg << ".";
-  llwarns << msg.str() << llendl;
-}
 

indra/llmessage/llurlrequest.h

@@ -72,7 +72,7 @@ class LLURLRequest : public AICurlEasyRequestStateMachine {
 	/**
 	 * @brief Cached value of responder->getName() as passed to the constructor.
 	 */
-	char const* getResponderName(void) const { return mResponderNameCache; }
+	/*virtual*/ const char* getName() const { return mResponderNameCache.c_str(); }
 
   protected:
 	// Call abort(), not delete.
@@ -113,7 +113,7 @@ class LLURLRequest : public AICurlEasyRequestStateMachine {
 	U32 mBodySize;
 	LLHTTPClient::ResponderPtr mResponder;
 	AIHTTPHeaders mHeaders;
-	char const* mResponderNameCache;
+	std::string mResponderNameCache;
 
   protected:
 	// Handle initializing the object.

indra/llui/llnotifications.cpp

@@ -1543,6 +1543,8 @@ class UpdateItemSM : public AIStateMachine
 
 	static void add(UpdateItem const& ui);
 
+	/*virtual*/ const char* getName() const { return "UpdateItemSM"; }
+
   private:
 	static UpdateItemSM* sSelf;
 	typedef std::deque<UpdateItem> updateQueue_type;

indra/newview/llmeshrepository.cpp

@@ -1245,16 +1245,21 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat
 
 bool LLMeshRepoThread::lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size)
 {
+	AIStateMachine::StateTimer timer("lodReceived");
 	LLPointer<LLVolume> volume = new LLVolume(mesh_params, LLVolumeLODGroup::getVolumeScaleFromDetail(lod));
 	std::string mesh_string((char*) data, data_size);
 	std::istringstream stream(mesh_string);
 
+	AIStateMachine::StateTimer timer2("unpackVolumeFaces");
 	if (volume->unpackVolumeFaces(stream, data_size))
 	{
+		AIStateMachine::StateTimer timer("getNumFaces");
 		if (volume->getNumFaces() > 0)
 		{
+			AIStateMachine::StateTimer timer("LoadedMesh");
 			LoadedMesh mesh(volume, mesh_params, lod);
 			{
+				AIStateMachine::StateTimer timer("LLMutexLock");
 				LLMutexLock lock(mMutex);
 				mLoadedQ.push(mesh);
 			}
@@ -1910,6 +1915,7 @@ void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason,
 	{
 		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//timeout or service unavailable, try again
+			AIStateMachine::StateTimer timer("loadMeshLOD");
 			llwarns << "Timeout or service unavailable, retrying." << llendl;
 			LLMeshRepository::sHTTPRetryCount++;
 			gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD);
@@ -1928,12 +1934,14 @@ void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason,
 
 	if (data_size > 0)
 	{
+		AIStateMachine::StateTimer timer("readAfter");
 		data = new U8[data_size];
 		buffer->readAfter(channels.in(), NULL, data, data_size);
 	}
 
 	if (gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size))
 	{
+		AIStateMachine::StateTimer timer("FileOpen");
 		//good fetch from sim, write to VFS for caching
 		LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE);
 
@@ -1942,6 +1950,7 @@ void LLMeshLODResponder::completedRaw(U32 status, const std::string& reason,
 
 		if (file.getSize() >= offset+size)
 		{
+			AIStateMachine::StateTimer timer("WriteData");
 			file.seek(offset);
 			file.write(data, size);
 			LLMeshRepository::sCacheBytesWritten += size;
@@ -2171,6 +2180,7 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason,
 
 		if (is_internal_http_error_that_warrants_a_retry(status) || status == HTTP_SERVICE_UNAVAILABLE)
 		{	//retry
+			AIStateMachine::StateTimer timer("Retry");
 			llwarns << "Timeout or service unavailable, retrying." << llendl;
 			LLMeshRepository::sHTTPRetryCount++;
 			LLMeshRepoThread::HeaderRequest req(mMeshParams);
@@ -2191,12 +2201,14 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason,
 
 	if (data_size > 0)
 	{
+		AIStateMachine::StateTimer timer("readAfter");
 		data = new U8[data_size];
 		buffer->readAfter(channels.in(), NULL, data, data_size);
 	}
 
 	LLMeshRepository::sBytesReceived += llmin(data_size, 4096);
 
+	AIStateMachine::StateTimer timer("headerReceived");
 	bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size);
 	
 	llassert(success);
@@ -2239,11 +2251,13 @@ void LLMeshHeaderResponder::completedRaw(U32 status, const std::string& reason,
 			//only allocate as much space in the VFS as is needed for the local cache
 			data_size = llmin(data_size, bytes);
 
+			AIStateMachine::StateTimer timer("FileOpen");
 			LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE);
 			if (file.getMaxSize() >= bytes || file.setMaxSize(bytes))
 			{
 				LLMeshRepository::sCacheBytesWritten += data_size;
 
+				AIStateMachine::StateTimer timer("WriteData");
 				file.write((const U8*) data, data_size);
 			
 				//zero out the rest of the file 

indra/newview/llmeshrepository.h

@@ -450,6 +450,8 @@ public:
 
 	void setWholeModelUploadURL(std::string const& whole_model_upload_url) { mWholeModelUploadURL = whole_model_upload_url; }
 
+	/*virtual*/ const char* getName() const { return "AIMeshUpload"; }
+
 protected:
 	// Implement AIStateMachine.
 	/*virtual*/ const char* state_str_impl(state_type run_state) const;

indra/newview/statemachine/aidirpicker.h

@@ -63,6 +63,8 @@ public:
 	bool hasDirname(void) const { return hasFilename(); }
 	std::string const& getDirname(void) const { return getFilename(); }
 
+	/*virtual*/ const char* getName() const { return "AIDirPicker"; }
+
 public:
 	// Basically all public members of AIStateMachine could made accessible here,
 	// but I don't think others will ever be needed (not even these, actually).

indra/newview/statemachine/aifilepicker.h

@@ -188,6 +188,8 @@ public:
 	std::string getFolder(void) const;
 	std::vector<std::string> const& getFilenames(void) const { return mFilenames; }
 
+	/*virtual*/ const char* getName() const { return "AIFilePicker"; }
+
 	// Load the sContextMap from disk.
 	static bool loadFile(std::string const& filename);
 	// Save the sContextMap to disk.