From c47f6405d2405f3ea3ad33c105db34a06ce2dde4 Mon Sep 17 00:00:00 2001 From: Shyotl Date: Mon, 23 Sep 2013 17:19:44 -0500 Subject: [PATCH] Texfetch/cache changes: Added SEND_UDP_REQ and WAIT_UDP_REQ fetch states. Parse 'content-range' from headers. Purge 'complete' textures from the cache if they lack 'end of codestream' marker. Added boostlevel/category to textureview display. More diagnostic output. Discard handling tweaks/bugfixes from v3. --- indra/llimagej2coj/llimagej2coj.cpp | 22 +- indra/newview/lldebugview.cpp | 2 +- indra/newview/lltexturecache.cpp | 4 +- indra/newview/lltexturefetch.cpp | 720 ++++++++++++++++++-------- indra/newview/lltexturefetch.h | 10 +- indra/newview/lltextureview.cpp | 52 +- indra/newview/llviewertexture.cpp | 36 +- indra/newview/llviewertexturelist.cpp | 1 + 8 files changed, 612 insertions(+), 235 deletions(-) diff --git a/indra/llimagej2coj/llimagej2coj.cpp b/indra/llimagej2coj/llimagej2coj.cpp index 7de5ba69d..3db9c077d 100644 --- a/indra/llimagej2coj/llimagej2coj.cpp +++ b/indra/llimagej2coj/llimagej2coj.cpp @@ -133,6 +133,26 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod parameters.cp_reduce = base.getRawDiscardLevel(); + if(parameters.cp_reduce == 0 && *(U16*)(base.getData() + base.getDataSize() - 2) != 0xD9FF) + { + bool failed = true; + for(S32 i = base.getDataSize()-1; i > 42; --i) + { + if(base.getData()[i] != 0x00) + { + failed = *(U16*)(base.getData()+i-1) != 0xD9FF; + break; + } + } + if(failed) + { + opj_image_destroy(image); + base.decodeFailed(); + return TRUE; + } + } + + /* decode the code-stream */ /* ---------------------- */ @@ -212,7 +232,7 @@ BOOL LLImageJ2COJ::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decod if(image->numcomps <= first_channel) { - llwarns << "trying to decode more channels than are present in image: numcomps: " << image->numcomps << " first_channel: " << first_channel << llendl; + LL_WARNS("Texture") << "trying to decode more channels than are present in image: numcomps: " << image->numcomps << " first_channel: " << first_channel << llendl; if (image) { opj_image_destroy(image); diff --git a/indra/newview/lldebugview.cpp b/indra/newview/lldebugview.cpp index e1451a411..cf991376b 100644 --- a/indra/newview/lldebugview.cpp +++ b/indra/newview/lldebugview.cpp @@ -85,7 +85,7 @@ LLDebugView::LLDebugView(const std::string& name, const LLRect &rect) mFastTimerView->setVisible(FALSE); // start invisible addChild(mFastTimerView); - r.set(150, rect.getHeight() - 50, 870, 100); + r.set(150, rect.getHeight() - 50, 970, 100); LLTextureView::Params tvp; tvp.name("gTextureView"); tvp.rect(r); diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 051c3e4c8..bb2b9c539 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -373,9 +373,9 @@ bool LLTextureCacheRemoteWorker::doRead() if (!done && (mState == LOCAL)) { llassert(local_size != 0); // we're assuming there is a non empty local file here... - if (!mDataSize || mDataSize > local_size - mOffset) + if (!mDataSize || mDataSize > local_size/* - mOffset*/) { - mDataSize = local_size - mOffset; + mDataSize = local_size/* - mOffset*/; } // Allocate read buffer mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index cdc31ae43..b4eb65594 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -62,6 +62,9 @@ #include "llbuffer.h" #include "hippogridmanager.h" +#include +#include + class AIHTTPTimeoutPolicy; extern AIHTTPTimeoutPolicy HTTPGetResponder_timeout; extern AIHTTPTimeoutPolicy lcl_responder_timeout; @@ -164,7 +167,8 @@ public: ~LLTextureFetchWorker(); // void relese() { --mActiveCount; } - S32 callbackHttpGet(const LLChannelDescriptors& channels, + S32 callbackHttpGet(U32 offset, U32 length, + const LLChannelDescriptors& channels, const LLHTTPClient::ResponderBase::buffer_ptr_t& buffer, bool partial, bool success); void callbackCacheRead(bool success, LLImageFormatted* image, @@ -224,6 +228,8 @@ private: CACHE_POST, LOAD_FROM_NETWORK, LOAD_FROM_SIMULATOR, + SEND_UDP_REQ, + WAIT_UDP_REQ, SEND_HTTP_REQ, WAIT_HTTP_REQ, DECODE_IMAGE, @@ -246,6 +252,7 @@ private: }; static const char* sStateDescs[]; e_state mState; + void setState(e_state new_state); e_write_to_cache_state mWriteToCacheState; LLTextureFetch* mFetcher; LLPointer mFormattedImage; @@ -309,6 +316,8 @@ private: U8 mImageCodec; LLViewerAssetStats::duration_t mMetricsStartTime; + unsigned int mHttpReplySize; // Actual received data size + unsigned int mHttpReplyOffset; // Actual received data offset // State history U32 mCacheReadCount; U32 mCacheWriteCount; @@ -322,38 +331,62 @@ public: HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) : mFetcher(fetcher) , mID(id) - , mStartTime(startTime) + , mMetricsStartTime(startTime) , mRequestedSize(requestedSize) , mRequestedOffset(offset) , mFollowRedir(redir) + , mReplyOffset(0) + , mReplyLength(0) + , mReplyFullLength(0) { } ~HTTPGetResponder() { } -#if 0 //Apparently, SL never sends content-range and instead sends transfer-encoding: chunked, so disabling for now /*virtual*/ bool needsHeaders(void) const { return true; } /*virtual*/ void completedHeaders(U32 status, std::string const& reason, AIHTTPReceivedHeaders const& headers) { - llinfos << "Texture fetch HTTP status: " << status << llendl; - llinfos << "Texture fetch headers: " << headers << llendl; - //example: Content-Range: 1000-3979/3980 Content-Length: 2980 - static const boost::regex pattern("\\w*bytes\\w+(\\d+)-(\\d+)/(\\d+)"); - std::string rangehdr; - if (headers.getFirstValue("content-range", rangehdr)){ - llinfos << "Have content-range header" < tokens; + boost::split(tokens,rangehdr,boost::is_any_of(" -/")); + if(tokens.size() == 4 && !stricmp(tokens[0].c_str(),"bytes")) + { + U32 first(0), last(0), len(0); + try + { + first = boost::lexical_cast(tokens[1].c_str()); + last = boost::lexical_cast(tokens[2].c_str()); + } + catch( boost::bad_lexical_cast& ) + { + return; + } + if(tokens[3] != "*") + { + try + { + len = boost::lexical_cast(tokens[3].c_str()); + } + catch( boost::bad_lexical_cast& ) + { + len = 0; + } + } + if(first <= last && (!len || last < len)) + { + mReplyOffset = first; + mReplyLength = last - first + 1; + mReplyFullLength = len; + LL_DEBUGS("Texture") << " mReplyOffset=" << mReplyOffset << " mReplyLength=" << mReplyLength << " mReplyFullLength=" << mReplyFullLength << llendl; + } } } } -#endif /*virtual*/ void completedRaw(U32 status, const std::string& reason, const LLChannelDescriptors& channels, @@ -362,11 +395,12 @@ public: static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); static LLCachedControl log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; - + if (log_to_viewer_log || log_to_sim) { - mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime); + U64 timeNow = LLTimer::getTotalTime(); + mFetcher->mTextureInfo.setRequestStartTime(mID, mMetricsStartTime); mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); mFetcher->mTextureInfo.setRequestOffset(mID, mRequestedOffset); @@ -377,6 +411,7 @@ public: LLTextureFetchWorker* worker = mFetcher->getWorker(mID); if (worker) { + worker->lockWorkMutex(); bool success = false; bool partial = false; if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) @@ -390,9 +425,9 @@ public: if (!success) { worker->setGetStatus(status, reason); -// llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; + llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; } - S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); + S32 data_size = worker->callbackHttpGet(mReplyOffset, mReplyLength, channels, buffer, partial, success); if(log_texture_traffic && data_size > 0) { @@ -405,6 +440,7 @@ public: mFetcher->removeFromHTTPQueue(mID, data_size); worker->recordTextureDone(true); + worker->unlockWorkMutex(); } else { @@ -419,11 +455,16 @@ public: /*virtual*/ char const* getName(void) const { return "HTTPGetResponder"; } private: + LLTextureFetch* mFetcher; LLUUID mID; - U64 mStartTime; + U64 mMetricsStartTime; S32 mRequestedSize; U32 mRequestedOffset; + U32 mReplyOffset; + U32 mReplyLength; + U32 mReplyFullLength; + bool mFollowRedir; }; @@ -727,13 +768,15 @@ const char* LLTextureFetchWorker::sStateDescs[] = { "CACHE_POST", "LOAD_FROM_NETWORK", "LOAD_FROM_SIMULATOR", + "SEND_UDP_REQ", + "WAIT_UDP_REQ", "SEND_HTTP_REQ", "WAIT_HTTP_REQ", "DECODE_IMAGE", "DECODE_IMAGE_UPDATE", "WRITE_TO_CACHE", "WAIT_ON_WRITE", - "DONE", + "DONE" }; // static @@ -789,6 +832,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mTotalPackets(0), mImageCodec(IMG_CODEC_INVALID), mMetricsStartTime(0), + mHttpReplySize(0U), + mHttpReplyOffset(0U), mCacheReadCount(0U), mCacheWriteCount(0U) { @@ -918,7 +963,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); if ((prioritize && mState == INIT) || mState == DONE) { - mState = INIT; + setState(INIT); U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; setPriority(work_priority); } @@ -944,6 +989,8 @@ void LLTextureFetchWorker::resetFormattedData() { mFormattedImage->deleteData(); } + mHttpReplySize = 0; + mHttpReplyOffset = 0; mHaveAllData = FALSE; } @@ -969,14 +1016,16 @@ bool LLTextureFetchWorker::doWork(S32 param) } if(mImagePriority < F_ALMOST_ZERO) { - if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR) + if (mState == INIT || mState == LOAD_FROM_NETWORK/* || mState == LOAD_FROM_SIMULATOR*/) //If we've already sent out requests.. might as well continue. { + LL_WARNS("Texture") << mID << " abort: mImagePriority < F_ALMOST_ZERO" << llendl; return true; // abort } } if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) { //nowhere to get data, abort. + LL_WARNS("Texture") << mID << " abort, nowhere to get data" << llendl; return true ; } @@ -1000,7 +1049,7 @@ bool LLTextureFetchWorker::doWork(S32 param) gAssetStorage->mBlackListedAsset.end(),mID) != gAssetStorage->mBlackListedAsset.end()) { llinfos << "Blacklisted asset " << mID.asString() << " was trying to be accessed!!!!!!" << llendl; - mState = DONE; + setState(DONE); return true; } @@ -1017,11 +1066,13 @@ bool LLTextureFetchWorker::doWork(S32 param) mDecoded = FALSE; mWritten = FALSE; std::vector().swap(mHttpBuffer); + mHttpReplySize = 0; + mHttpReplyOffset = 0; mHaveAllData = FALSE; clearPackets(); // TODO: Shouldn't be necessary mCacheReadHandle = LLTextureCache::nullHandle(); mCacheWriteHandle = LLTextureCache::nullHandle(); - mState = LOAD_FROM_TEXTURE_CACHE; + setState(LOAD_FROM_TEXTURE_CACHE); mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); // min desired size is TEXTURE_CACHE_ENTRY_SIZE LL_DEBUGS("Texture") << mID << ": Priority: " << llformat("%8.0f",mImagePriority) << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; @@ -1037,7 +1088,7 @@ bool LLTextureFetchWorker::doWork(S32 param) S32 size = mDesiredSize - offset; if (size <= 0) { - mState = CACHE_POST; + setState(CACHE_POST); return false; } mFileSize = 0; @@ -1073,12 +1124,12 @@ bool LLTextureFetchWorker::doWork(S32 param) llwarns << "Unknown URL Type: " << mUrl << llendl; } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = SEND_HTTP_REQ; + setState(SEND_HTTP_REQ); } else { setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = LOAD_FROM_NETWORK; + setState(LOAD_FROM_NETWORK); } } @@ -1088,7 +1139,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mFetcher->mTextureCache->readComplete(mCacheReadHandle, false)) { mCacheReadHandle = LLTextureCache::nullHandle(); - mState = CACHE_POST; + setState(CACHE_POST); // fall through } else @@ -1096,6 +1147,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // //This should never happen // + LL_WARNS("Texture") << mID << " this should never happen" << llendl; return false; } } @@ -1114,7 +1166,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // we have enough data, decode it llassert_always(mFormattedImage->getDataSize() > 0); mLoadedDiscard = mDesiredDiscard; - mState = DECODE_IMAGE; + if (mLoadedDiscard < 0) + { + LL_WARNS("Texture") << mID << " mLoadedDiscard is " << mLoadedDiscard + << ", should be >=0" << llendl; + } + setState(DECODE_IMAGE); mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) @@ -1126,13 +1183,14 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { // failed to load local file, we're done. + LL_WARNS("Texture") << mID << ": abort, failed to load local file " << mUrl << LL_ENDL; return true; } // need more data else { LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; - mState = LOAD_FROM_NETWORK; + setState(LOAD_FROM_NETWORK); } // fall through @@ -1179,7 +1237,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (mCanUseHTTP && !mUrl.empty()) { - mState = LLTextureFetchWorker::SEND_HTTP_REQ; + setState(SEND_HTTP_REQ); setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); if(mWriteToCacheState != NOT_WRITE) { @@ -1189,17 +1247,13 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mSentRequest == UNSENT && mCanUseNET) { - // Add this to the network queue and sit here. - // LLTextureFetch::update() will send off a request which will change our state - mWriteToCacheState = CAN_WRITE ; - mRequestedSize = mDesiredSize; - mRequestedDiscard = mDesiredDiscard; - mSentRequest = QUEUED; - mFetcher->addToNetworkQueue(this); - recordTextureStart(false); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - - return false; + LL_WARNS("Texture") << mID << " moving to UDP fetch. mSentRequest=" << mSentRequest << " mCanUseNET = " << mCanUseNET << llendl; + setState(SEND_UDP_REQ); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + if(mWriteToCacheState != NOT_WRITE) + { + mWriteToCacheState = CAN_WRITE ; + } } else { @@ -1207,12 +1261,15 @@ bool LLTextureFetchWorker::doWork(S32 param) //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); + //recordTextureStart(false); //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + + LL_WARNS("Texture") << mID << " does this happen? mSentRequest=" << mSentRequest << " mCanUseNET = " << mCanUseNET << llendl; return false; } } - if (mState == LOAD_FROM_SIMULATOR) + if (mState == LOAD_FROM_SIMULATOR) //UDP. From LLTextureFetch::receiveImageHeader or LLTextureFetch::receiveImagePacket { if (mFormattedImage.isNull()) { @@ -1226,118 +1283,163 @@ bool LLTextureFetchWorker::doWork(S32 param) { // processSimulatorPackets() failed // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; + LL_WARNS("Texture") << mID << " processSimulatorPackets() failed to load buffer" << llendl; return true; // failed } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = DECODE_IMAGE; + if (mLoadedDiscard < 0) + { + LL_WARNS("Texture") << mID << " mLoadedDiscard is " << mLoadedDiscard + << ", should be >=0" << llendl; + } + setState(DECODE_IMAGE); + llassert_always(strstr(mUrl.c_str(), "map.secondlife") == NULL); mWriteToCacheState = SHOULD_WRITE; recordTextureDone(false); } else { - mFetcher->addToNetworkQueue(this); // failsafe - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - recordTextureStart(false); + llassert(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); + //mFetcher->addToNetworkQueue(this); // failsafe + //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + //recordTextureStart(false); } return false; } + + if (mState == SEND_UDP_REQ) + { + if (! mCanUseNET) + { + LL_WARNS("Texture") << mID << " abort: SEND_UDP_REQ but !mCanUseNet" << llendl; + return true ; //abort + } + + LL_WARNS("Texture") << mID << " sendind to UDP fetch. mSentRequest=" << mSentRequest << " mCanUseNET = " << mCanUseNET << llendl; + + mRequestedSize = mDesiredSize; + mRequestedDiscard = mDesiredDiscard; + mSentRequest = QUEUED; + mFetcher->addToNetworkQueue(this); + recordTextureStart(false); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setState(WAIT_UDP_REQ); + } + + if (mState == WAIT_UDP_REQ) + { + //do nothing. + if (! mCanUseNET) + { + LL_WARNS("Texture") << mID << " abort: SEND_UDP_REQ but !mCanUseNet" << llendl; + return true ; //abort + } + llassert(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); + //Should migrate to LOAD_FROM_SIMULATOR upon receipt of data. + } if (mState == SEND_HTTP_REQ) { - if(mCanUseHTTP) - { - S32 cur_size = 0; - if (mFormattedImage.notNull()) - { - cur_size = mFormattedImage->getDataSize(); // amount of data we already have - if (mFormattedImage->getDiscardLevel() == 0) - { - // Already have all data. - mFetcher->removeFromNetworkQueue(this, false); // Note sure this is necessary, but it's what the old did --Aleric - if(cur_size > 0) - { - // We already have all the data, just decode it - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - return true ; //abort. - } - } - } - - // Let AICurl decide if we can process more HTTP requests at the moment or not. - - // AIPerService::approveHTTPRequestFor returns approvement for ONE request. - // This object keeps track of whether or not that is honored. - LLPointer approved = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_texture); - if (!approved) - { - return false ; //wait. - } - - mFetcher->removeFromNetworkQueue(this, false); - - mRequestedSize = mDesiredSize - cur_size; - mRequestedDiscard = mDesiredDiscard; - mRequestedOffset = cur_size; - - bool res = false; - if (!mUrl.empty()) - { - mLoaded = FALSE; - mGetStatus = 0; - mGetReason.clear(); - LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset - << " Bytes: " << mRequestedSize - << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mState = WAIT_HTTP_REQ; - - mFetcher->addToHTTPQueue(mID); - - if(mRequestedOffset>0) - { - // Texture fetching often issues 'speculative' loads that - // start beyond the end of the actual asset. Some cache/web - // systems, e.g. Varnish, will respond to this not with a - // 416 but with a 200 and the entire asset in the response - // body. By ensuring that we always have a partially - // satisfiable Range request, we avoid that hit to the network. - // We just have to deal with the overlapping data which is made - // somewhat harder by the fact that grid services don't necessarily - // return the Content-Range header on 206 responses. *Sigh* - mRequestedSize++; - mRequestedOffset--; - } - - // Will call callbackHttpGet when curl request completes - AIHTTPHeaders headers("Accept", "image/x-j2c"); - // Call LLHTTPClient::request directly instead of LLHTTPClient::getByteRange, because we want to pass a NULL AIEngine. - if (mRequestedOffset > 0 || mRequestedSize > 0) - { - headers.addHeader("Range", llformat("bytes=%d-%d", mRequestedOffset, mRequestedOffset + mRequestedSize - 1)); - } - LLHTTPClient::request(mUrl, LLHTTPClient::HTTP_GET, NULL, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, mRequestedOffset, true), - headers, approved/*,*/ DEBUG_CURLIO_PARAM(debug_off), keep_alive, no_does_authentication, allow_compressed_reply, NULL, 0, NULL); - res = true; - } - if (!res) - { - llwarns << "HTTP GET request failed for " << mID << llendl; - resetFormattedData(); - ++mHTTPFailCount; - return true; // failed - } - // fall through - } - else //can not use http fetch. + if (! mCanUseHTTP) { + LL_WARNS("Texture") << mID << " abort: SEND_HTTP_REQ but !mCanUseHTTP" << llendl; return true ; //abort } + S32 cur_size = 0; + if (mFormattedImage.notNull()) + { + cur_size = mFormattedImage->getDataSize(); // amount of data we already have + if (mFormattedImage->getDiscardLevel() == 0) + { + // Already have all data. + mFetcher->removeFromNetworkQueue(this, false); // Note sure this is necessary, but it's what the old did --Aleric + if (cur_size > 0) + { + // We already have all the data, just decode it + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + if (mLoadedDiscard < 0) + { + LL_WARNS("Texture") << mID << " mLoadedDiscard is " << mLoadedDiscard + << ", should be >=0" << llendl; + } + setState(DECODE_IMAGE); + return false; + } + else + { + LL_WARNS("Texture") << mID << " SEND_HTTP_REQ abort: cur_size " << cur_size << " <=0" << llendl; + return true ; //abort. + } + } + } + + // Let AICurl decide if we can process more HTTP requests at the moment or not. + + // AIPerService::approveHTTPRequestFor returns approvement for ONE request. + // This object keeps track of whether or not that is honored. + LLPointer approved = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_texture); + if (!approved) + { + return false ; //wait. + } + + mFetcher->removeFromNetworkQueue(this, false); + + mRequestedSize = mDesiredSize; + mRequestedDiscard = mDesiredDiscard; + mRequestedSize -= cur_size; + mRequestedOffset = cur_size; + + if (mRequestedOffset) + { + // Texture fetching often issues 'speculative' loads that + // start beyond the end of the actual asset. Some cache/web + // systems, e.g. Varnish, will respond to this not with a + // 416 but with a 200 and the entire asset in the response + // body. By ensuring that we always have a partially + // satisfiable Range request, we avoid that hit to the network. + // We just have to deal with the overlapping data which is made + // somewhat harder by the fact that grid services don't necessarily + // return the Content-Range header on 206 responses. *Sigh* + mRequestedOffset -= 1; + mRequestedSize += 1; + } + + if (!mUrl.empty()) + { + mRequestedTimer.reset(); + mLoaded = FALSE; + mGetStatus = 0; + mGetReason.clear(); + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset + << " Bytes: " << mRequestedSize + << LL_ENDL; + // Will call callbackHttpGet when curl request completes + AIHTTPHeaders headers("Accept", "image/x-j2c"); + // Call LLHTTPClient::request directly instead of LLHTTPClient::getByteRange, because we want to pass a NULL AIEngine. + if (mRequestedOffset > 0 || mRequestedSize > 0) + { + headers.addHeader("Range", llformat("bytes=%d-%d", mRequestedOffset, mRequestedOffset + mRequestedSize - 1)); + } + LLHTTPClient::request(mUrl, LLHTTPClient::HTTP_GET, NULL, + new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, mRequestedOffset, true), + headers, approved/*,*/ DEBUG_CURLIO_PARAM(debug_off), keep_alive, no_does_authentication, allow_compressed_reply, NULL, 0, NULL); + } + else + { + llwarns << "HTTP GET request failed for " << mID << llendl; + resetFormattedData(); + ++mHTTPFailCount; + return true; // failed + } + + mFetcher->addToHTTPQueue(mID); + recordTextureStart(true); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setState(WAIT_HTTP_REQ); + + // fall through } if (mState == WAIT_HTTP_REQ) @@ -1348,11 +1450,73 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mRequestedSize < 0) { S32 max_attempts; + switch(mGetStatus) + { +#define HTTP_CASE(name) case name: LL_DEBUGS("TexDebug") << mID << " status = " << mGetStatus << " (" << ##name << ")" << " Failcount = " << mHTTPFailCount << llendl; break; + HTTP_CASE(HTTP_CONTINUE) + HTTP_CASE(HTTP_SWITCHING_PROTOCOLS) + HTTP_CASE(HTTP_OK) + HTTP_CASE(HTTP_CREATED) + HTTP_CASE(HTTP_ACCEPTED) + HTTP_CASE(HTTP_NON_AUTHORITATIVE_INFORMATION) + HTTP_CASE(HTTP_NO_CONTENT) + HTTP_CASE(HTTP_RESET_CONTENT) + HTTP_CASE(HTTP_PARTIAL_CONTENT) + HTTP_CASE(HTTP_MULTIPLE_CHOICES) + HTTP_CASE(HTTP_MOVED_PERMANENTLY) + HTTP_CASE(HTTP_FOUND) + HTTP_CASE(HTTP_SEE_OTHER) + HTTP_CASE(HTTP_NOT_MODIFIED) + HTTP_CASE(HTTP_USE_PROXY) + HTTP_CASE(HTTP_TEMPORARY_REDIRECT) + HTTP_CASE(HTTP_BAD_REQUEST) + HTTP_CASE(HTTP_UNAUTHORIZED) + HTTP_CASE(HTTP_PAYMENT_REQUIRED) + HTTP_CASE(HTTP_FORBIDDEN) + HTTP_CASE(HTTP_NOT_FOUND) + HTTP_CASE(HTTP_METHOD_NOT_ALLOWED) + HTTP_CASE(HTTP_NOT_ACCEPTABLE) + HTTP_CASE(HTTP_PROXY_AUTHENTICATION_REQUIRED) + HTTP_CASE(HTTP_REQUEST_TIME_OUT) + HTTP_CASE(HTTP_CONFLICT) + HTTP_CASE(HTTP_GONE) + HTTP_CASE(HTTP_LENGTH_REQUIRED) + HTTP_CASE(HTTP_PRECONDITION_FAILED) + HTTP_CASE(HTTP_REQUEST_ENTITY_TOO_LARGE) + HTTP_CASE(HTTP_REQUEST_URI_TOO_LARGE) + HTTP_CASE(HTTP_UNSUPPORTED_MEDIA_TYPE) + HTTP_CASE(HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) + HTTP_CASE(HTTP_EXPECTATION_FAILED) + HTTP_CASE(HTTP_INTERNAL_SERVER_ERROR) + HTTP_CASE(HTTP_NOT_IMPLEMENTED) + HTTP_CASE(HTTP_BAD_GATEWAY) + HTTP_CASE(HTTP_SERVICE_UNAVAILABLE) + HTTP_CASE(HTTP_GATEWAY_TIME_OUT) + HTTP_CASE(HTTP_VERSION_NOT_SUPPORTED) + HTTP_CASE(HTTP_INTERNAL_ERROR_LOW_SPEED) + HTTP_CASE(HTTP_INTERNAL_ERROR_CURL_LOCKUP) + HTTP_CASE(HTTP_INTERNAL_ERROR_CURL_BADSOCKET) + HTTP_CASE(HTTP_INTERNAL_ERROR_CURL_TIMEOUT) + HTTP_CASE(HTTP_INTERNAL_ERROR_CURL_OTHER) + HTTP_CASE(HTTP_INTERNAL_ERROR_OTHER) + default: + LL_DEBUGS("TexDebug") << mID << " status = " << mGetStatus << " (??)" << " Failcount = " << mHTTPFailCount << llendl; break; + } + if (mGetStatus == HTTP_NOT_FOUND || mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT || mGetStatus == HTTP_INTERNAL_ERROR_LOW_SPEED) { mHTTPFailCount = max_attempts = 1; // Don't retry if(mGetStatus == HTTP_NOT_FOUND) + { + if(mWriteToCacheState == NOT_WRITE) //map tiles + { + resetFormattedData(); + setState(DONE); + //LL_INFOS("Texture") << mID << " abort: WAIT_HTTP_REQ not found" << llendl; + return true; // failed, means no map tile on the empty region. + } llwarns << "Texture missing from server (404): " << mUrl << llendl; + } else if (mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT || mGetStatus == HTTP_INTERNAL_ERROR_LOW_SPEED) { if (mGetStatus == HTTP_INTERNAL_ERROR_CURL_TIMEOUT) @@ -1368,14 +1532,18 @@ bool LLTextureFetchWorker::doWork(S32 param) //roll back to try UDP if(mCanUseNET) { + LL_DEBUGS("TexDebug") << mID << " falling back to udp mSentRequest=" << mSentRequest << " mCanUseNET = " << mCanUseNET << llendl; resetFormattedData(); - mState = INIT ; + setState(INIT); mCanUseHTTP = false ; + mUrl.clear(); setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + LL_DEBUGS("TexDebug") << mID << " .. mSentRequest=" << mSentRequest << " mCanUseNET = " << mCanUseNET << llendl; return false ; } else { + LL_INFOS("Texture") << mID << " aborted. no udp fallback" << llendl; // UDP is not an option, we are dead resetFormattedData(); return true; // failed @@ -1390,6 +1558,12 @@ bool LLTextureFetchWorker::doWork(S32 param) max_attempts = mHTTPFailCount+1; // Keep retrying LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; } + else if (mGetStatus == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) + { + // Allowed, we'll accept whatever data we have as complete. + mHaveAllData = TRUE; + max_attempts = mHTTPFailCount+1; + } else { const S32 HTTP_MAX_RETRY_COUNT = 3; @@ -1408,65 +1582,84 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + if (mLoadedDiscard < 0) + { + LL_WARNS("Texture") << mID << " mLoadedDiscard is " << mLoadedDiscard + << ", should be >=0" << llendl; + } + setState(DECODE_IMAGE); return false; } - else - { - //roll back to try UDP - if(mCanUseNET) - { - resetFormattedData(); - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; - } - else - { - // UDP is not an option, we are dead - resetFormattedData(); - mState = DONE; - return true; // failed - } - } + else + { + //roll back to try UDP + if(mCanUseNET) + { + LL_DEBUGS("TexDebug") << mID << " falling back to udp (2)" << LL_ENDL; + resetFormattedData(); + setState(INIT); + mCanUseHTTP = false ; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + return false ; + } + else + { + // UDP is not an option, we are dead + resetFormattedData(); + setState(DONE); + LL_INFOS("Texture") << mID << " abort: fail harder" << llendl; + return true; // failed + } } + } else { - mState = SEND_HTTP_REQ; + setState(SEND_HTTP_REQ); return false; // retry } } + // Clear the url since we're done with the fetch + // Note: mUrl is used to check is fetching is required so failure to clear it will force an http fetch + // next time the texture is requested, even if the data have already been fetched. + if(mWriteToCacheState != NOT_WRITE) + { + // Why do we want to keep url if NOT_WRITE - is this a proxy for map tiles? + mUrl.clear(); + } + if(mHttpBuffer.empty())//no data received. { //abort. - mState = DONE; + setState(DONE); + LL_WARNS("Texture") << mID << " abort: no data received" << llendl; return true; } - S32 total_size(cur_size + mRequestedSize); + S32 append_size(mHttpBuffer.size()); + S32 total_size(cur_size + append_size); S32 src_offset(0); - - if(mRequestedOffset && mRequestedOffset != cur_size) + llassert_always(append_size == mRequestedSize); + if (mHttpReplyOffset && mHttpReplyOffset != cur_size) { // In case of a partial response, our offset may // not be trivially contiguous with the data we have. // Get back into alignment. - if (mRequestedOffset > cur_size) + if ((S32)mHttpReplyOffset > cur_size) { LL_WARNS("Texture") << "Partial HTTP response produces break in image data for texture " << mID << ". Aborting load." << LL_ENDL; - mState = DONE; + setState(DONE); return true; } - src_offset = cur_size - mRequestedOffset; + src_offset = cur_size - mHttpReplyOffset; + append_size -= src_offset; total_size -= src_offset; - mRequestedSize -= src_offset; // Make requested values reflect useful part + mRequestedSize -= src_offset; // Make requested values reflect useful part mRequestedOffset += src_offset; } - llassert(total_size == cur_size + mRequestedSize); if (mFormattedImage.isNull()) { @@ -1478,8 +1671,8 @@ bool LLTextureFetchWorker::doWork(S32 param) mFormattedImage = new LLImageJ2C; // default } } - - if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. + + if (mHaveAllData) //the image file is fully loaded. { mFileSize = total_size; } @@ -1493,16 +1686,24 @@ bool LLTextureFetchWorker::doWork(S32 param) { memcpy(buffer, mFormattedImage->getData(), cur_size); } - if (mRequestedSize > 0) + if (append_size > 0) { - memcpy(buffer + mRequestedOffset, &mHttpBuffer[src_offset], mRequestedSize); // append + memcpy(buffer + cur_size, &mHttpBuffer[src_offset], append_size); } // NOTE: setData releases current data and owns new data (buffer) mFormattedImage->setData(buffer, total_size); // delete temp data std::vector().swap(mHttpBuffer); + mHttpReplySize = 0; + mHttpReplyOffset = 0; + mLoadedDiscard = mRequestedDiscard; - mState = DECODE_IMAGE; + if (mLoadedDiscard < 0) + { + LL_WARNS("Texture") << mID << " mLoadedDiscard is " << mLoadedDiscard + << ", should be >=0" << llendl; + } + setState(DECODE_IMAGE); if(mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = SHOULD_WRITE ; @@ -1525,31 +1726,34 @@ bool LLTextureFetchWorker::doWork(S32 param) if (textures_decode_disabled) { // for debug use, don't decode - mState = DONE; + setState(DONE); return true; } if (mDesiredDiscard < 0) { // We aborted, don't decode - mState = DONE; + setState(DONE); + LL_WARNS("Texture") << mID << " DECODE_IMAGE abort: desired discard " << mDesiredDiscard << "<0" << llendl; return true; } if (mFormattedImage->getDataSize() <= 0) { - //llerrs << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; + llwarns << "Decode entered with invalid mFormattedImage. ID = " << mID << llendl; //abort, don't decode - mState = DONE; + setState(DONE); + LL_WARNS("Texture") << mID << " DECODE_IMAGE abort: (mFormattedImage->getDataSize() <= 0)" << llendl; return true; } if (mLoadedDiscard < 0) { - //llerrs << "Decode entered with invalid mLoadedDiscard. ID = " << mID << llendl; + llwarns << "Decode entered with invalid mLoadedDiscard. ID = " << mID << llendl; //abort, don't decode - mState = DONE; + setState(DONE); + LL_WARNS("Texture") << mID << " DECODE_IMAGE abort: mLoadedDiscard < 0" << llendl; return true; } @@ -1559,7 +1763,7 @@ bool LLTextureFetchWorker::doWork(S32 param) S32 discard = mHaveAllData ? 0 : mLoadedDiscard; U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; mDecoded = FALSE; - mState = DECODE_IMAGE_UPDATE; + setState(DECODE_IMAGE_UPDATE); LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard << " All Data: " << mHaveAllData << LL_ENDL; mDecodeHandle = mFetcher->mImageDecodeThread->decodeImage(mFormattedImage, image_priority, discard, mNeedsAux, @@ -1573,7 +1777,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { if (mDecodedDiscard < 0) { - LL_DEBUGS("Texture") << mID << ": Failed to Decode." << LL_ENDL; + LL_WARNS("Texture") << mID << ": Failed to Decode." << LL_ENDL; if (mCachedSize > 0 && !mInLocalCache && mRetryAttempt == 0) { // Cache file should be deleted, try again @@ -1582,13 +1786,13 @@ bool LLTextureFetchWorker::doWork(S32 param) mFormattedImage = NULL; ++mRetryAttempt; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = INIT; + setState(INIT); return false; } else { // llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; - mState = DONE; // failed + setState(DONE); // failed } } else @@ -1597,7 +1801,7 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - mState = WRITE_TO_CACHE; + setState(WRITE_TO_CACHE); } // fall through } @@ -1613,7 +1817,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // If we're in a local cache or we didn't actually receive any new data, // or we failed to load anything, skip - mState = DONE; + setState(DONE); return false; } S32 datasize = mFormattedImage->getDataSize(); @@ -1632,7 +1836,7 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; - mState = WAIT_ON_WRITE; + setState(WAIT_ON_WRITE); ++mCacheWriteCount; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, @@ -1645,7 +1849,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { if (writeToCacheComplete()) { - mState = DONE; + setState(DONE); // fall through } else @@ -1666,7 +1870,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT - mState = INIT; + setState(INIT); + LL_WARNS("Texture") << mID << " more data requested, returning to INIT: " + << " mDecodedDiscard " << mDecodedDiscard << ">= 0 && mDesiredDiscard " << mDesiredDiscard + << "<" << " mDecodedDiscard " << mDecodedDiscard << llendl; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } @@ -1818,14 +2025,13 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// -S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, +S32 LLTextureFetchWorker::callbackHttpGet(U32 offset, U32 length, + const LLChannelDescriptors& channels, const LLHTTPClient::ResponderBase::buffer_ptr_t& buffer, bool partial, bool success) { S32 data_size = 0 ; - LLMutexLock lock(&mWorkMutex); - if (mState != WAIT_HTTP_REQ) { llwarns << "callbackHttpGet for unrequested fetch worker: " << mID @@ -1850,7 +2056,44 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, llassert(mHttpBuffer.empty()); mHttpBuffer.resize(data_size); buffer->readAfter(channels.in(), NULL, &mHttpBuffer[0], data_size); - if (data_size < mRequestedSize && mRequestedDiscard == 0) + + if (partial) + { + if (! offset && ! length) + { + // This is the case where we receive a 206 status but + // there wasn't a useful Content-Range header in the response. + // This could be because it was badly formatted but is more + // likely due to capabilities services which scrub headers + // from responses. Assume we got what we asked for... + mHttpReplySize = data_size; + mHttpReplyOffset = mRequestedOffset; + } + else + { + mHttpReplySize = length; + mHttpReplyOffset = offset; + } + } + + if (! partial) + { + // Response indicates this is the entire asset regardless + // of our asking for a byte range. Mark it so and drop + // any partial data we might have so that the current + // response body becomes the entire dataset. + if (data_size <= mRequestedOffset) + { + LL_WARNS("Texture") << "Fetched entire texture " << mID + << " when it was expected to be marked complete. mImageSize: " + << mFileSize << " datasize: " << mFormattedImage->getDataSize() + << LL_ENDL; + } + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + } + else if (data_size < mRequestedSize/* && mRequestedDiscard == 0*/) { mHaveAllData = TRUE; } @@ -1859,7 +2102,6 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, // *TODO: This shouldn't be happening any more llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; mHaveAllData = TRUE; - mRequestedOffset = 0; llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; // discard any previous data we had } @@ -2045,6 +2287,12 @@ LLTextureFetch::~LLTextureFetch() { clearDeleteList() ; + while (! mCommands.empty()) + { + TFRequest * req(mCommands.front()); + mCommands.erase(mCommands.begin()); + delete req; + } // ~LLQueuedThread() called here } @@ -2074,6 +2322,7 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con std::string exten = gDirUtilp->getExtension(url); if (!url.empty() && (!exten.empty() && LLImageBase::getCodecFromExtension(exten) != IMG_CODEC_J2C)) { + LL_DEBUGS("Texture") << "full request for " << id << " exten is not J2C: " << exten << llendl; // Only do partial requests for J2C at the moment //llinfos << "Merov : LLTextureFetch::createRequest(), blocking fetch on " << url << llendl; desired_size = MAX_IMAGE_DATA_SIZE; @@ -2115,8 +2364,8 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con worker->setCanUseHTTP(can_use_http) ; if (!worker->haveWork()) { - worker->mState = LLTextureFetchWorker::INIT; - worker->unlockWorkMutex(); + worker->setState(LLTextureFetchWorker::INIT); + worker->unlockWorkMutex(); // -Mw worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); } else @@ -2137,8 +2386,8 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con worker->setCanUseHTTP(can_use_http) ; worker->unlockWorkMutex(); } - - //llinfos << "REQUESTED: " << id << " Discard: " << desired_discard << llendl; + + LL_DEBUGS("Texture") << "REQUESTED: " << id << " Discard: " << desired_discard << " size " << desired_size << llendl; return true; } @@ -2241,6 +2490,17 @@ S32 LLTextureFetch::getNumRequests() return size ; } +// Threads: T* +S32 LLTextureFetch::getNumHTTPRequests() +{ + mNetworkQueueMutex.lock(); // +Mfq + S32 size = (S32)mHTTPTextureQueue.size(); + mNetworkQueueMutex.unlock(); // -Mfq + + return size; +} + +// Threads: T* U32 LLTextureFetch::getTotalNumHTTPRequests() { mNetworkQueueMutex.lock() ; @@ -2353,8 +2613,9 @@ S32 LLTextureFetch::getPending() LLMutexLock lock(&mQueueMutex); res = mRequestQueue.size(); - } - unlockData(); + res += mCommands.size(); + } // -Mfq + unlockData(); // -Ct return res; } @@ -2520,7 +2781,8 @@ void LLTextureFetch::sendRequestListToSimulators() mNetworkQueue.erase(curiter); continue; } - if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && + if ((req->mState != LLTextureFetchWorker::SEND_UDP_REQ) && + (req->mState != LLTextureFetchWorker::WAIT_UDP_REQ) && //Workers remain in the queue. May be re-requested upon timeout. (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) { // We already received our URL, remove from the queue @@ -2712,6 +2974,32 @@ bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) return true; } +void LLTextureFetchWorker::setState(e_state new_state) +{ + static const char* e_state_name[] = + { + "INVALID", + "INIT", + "LOAD_FROM_TEXTURE_CACHE", + "CACHE_POST", + "LOAD_FROM_NETWORK", + "LOAD_FROM_SIMULATOR", + "SEND_UDP_REQ", + "WAIT_UDP_REQ", + "SEND_HTTP_REQ", + "WAIT_HTTP_REQ", + "DECODE_IMAGE", + "DECODE_IMAGE_UPDATE", + "WRITE_TO_CACHE", + "WAIT_ON_WRITE", + "DONE" + }; + //if(mState != new_state) + // LL_INFOS("Texture") << "id: " << mID << " disc: " << mDesiredDiscard << " sz: " << mDesiredSize << " state: " << e_state_name[mState] << " => " << e_state_name[new_state] << llendl; + mState = new_state; +} + +// Threads: T* bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data) { @@ -2725,7 +3013,7 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 // llwarns << "Received header for non active worker: " << id << llendl; res = false; } - else if (worker->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK || + else if (worker->mState != LLTextureFetchWorker::WAIT_UDP_REQ || worker->mSentRequest != LLTextureFetchWorker::SENT_SIM) { // llwarns << "receiveImageHeader for worker: " << id @@ -2762,14 +3050,14 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 // Copy header data into image object worker->mImageCodec = codec; worker->mTotalPackets = packets; - worker->mFileSize = (S32)totalbytes; - llassert_always(totalbytes > 0); - llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); - res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; - worker->unlockWorkMutex(); - return res; + worker->mFileSize = (S32)totalbytes; + llassert_always(totalbytes > 0); + llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); + res = worker->insertPacket(0, data, data_size); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->setState(LLTextureFetchWorker::LOAD_FROM_SIMULATOR); + worker->unlockWorkMutex(); // -Mw + return res; } bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data) @@ -2812,10 +3100,10 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 res = worker->insertPacket(packet_num, data, data_size); if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || - (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) + (worker->mState == LLTextureFetchWorker::WAIT_UDP_REQ)) { worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); - worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; + worker->setState(LLTextureFetchWorker::LOAD_FROM_SIMULATOR); } else { @@ -3049,7 +3337,7 @@ public: { if (status) { - LL_WARNS("Texture") << "Successfully delivered asset metrics to grid." + LL_DEBUGS("Texture") << "Successfully delivered asset metrics to grid." << LL_ENDL; } else diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 4062c310a..adabfcf91 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -84,8 +84,14 @@ public: S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p, U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http); void dump(); - S32 getNumRequests() ; - U32 getTotalNumHTTPRequests() ; + // Threads: T* + S32 getNumRequests(); + + // Threads: T* + S32 getNumHTTPRequests(); + + // Threads: T* + U32 getTotalNumHTTPRequests(); // Public for access by callbacks S32 getPending(); diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index 70c6d0b5e..ec771cce6 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -81,7 +81,7 @@ static std::string title_string1a("Tex UUID Area DDis(Req) DecodePri(Fetch) static std::string title_string1b("Tex UUID Area DDis(Req) Fetch(DecodePri) [download] pk/max"); static std::string title_string2("State"); static std::string title_string3("Pkt Bnd"); -static std::string title_string4(" W x H (Dis) Mem"); +static std::string title_string4(" W x H (Dis) Mem Type"); static S32 title_x1 = 0; static S32 title_x2 = 460; @@ -211,6 +211,9 @@ void LLTextureBar::draw() std::string uuid_str; mImagep->mID.toString(uuid_str); uuid_str = uuid_str.substr(0,7); + + std::string vsstr = llformat("%f",mImagep->mMaxVirtualSize); + std::string dpstr = llformat("%f",mImagep->getDecodePriority()); if (mTextureView->mOrderFetch) { tex_str = llformat("%s %7.0f %d(%d) 0x%08x(%8.0f)", @@ -223,14 +226,14 @@ void LLTextureBar::draw() } else { - tex_str = llformat("%s %7.0f %d(%d) %8.0f(0x%08x) %1.2f", + tex_str = llformat("%s %7.0f %d(%d) %8.0f(0x%08x) %3d%%", uuid_str.c_str(), mImagep->mMaxVirtualSize, mImagep->mDesiredDiscardLevel, mImagep->mRequestedDiscardLevel, mImagep->getDecodePriority(), mImagep->mFetchPriority, - mImagep->mDownloadProgress); + llfloor(mImagep->mDownloadProgress*100.f)); } LLFontGL::getFontMonospace()->renderUTF8(tex_str, 0, title_x1, getRect().getHeight(), @@ -239,12 +242,14 @@ void LLTextureBar::draw() // State // Hack: mirrored from lltexturefetch.cpp struct { const std::string desc; LLColor4 color; } fetch_state_desc[] = { - { "---", LLColor4::red }, // INVALID + { "-?-", LLColor4::red }, // INVALID { "INI", LLColor4::white }, // INIT { "DSK", LLColor4::cyan }, // LOAD_FROM_TEXTURE_CACHE { "DSK", LLColor4::blue }, // CACHE_POST { "NET", LLColor4::green }, // LOAD_FROM_NETWORK { "SIM", LLColor4::green }, // LOAD_FROM_SIMULATOR + { "REQ", LLColor4::magenta },// SEND_UDP_REQ + { "UDP", LLColor4::cyan }, // WAIT_UDP_REQ { "REQ", LLColor4::yellow },// SEND_HTTP_REQ { "HTP", LLColor4::green }, // WAIT_HTTP_REQ { "DEC", LLColor4::yellow },// DECODE_IMAGE @@ -252,12 +257,12 @@ void LLTextureBar::draw() { "WRT", LLColor4::purple },// WRITE_TO_CACHE { "WRT", LLColor4::orange },// WAIT_ON_WRITE { "END", LLColor4::red }, // DONE -#define LAST_STATE 12 +#define LAST_STATE 14 { "CRE", LLColor4::magenta }, // LAST_STATE+1 { "FUL", LLColor4::green }, // LAST_STATE+2 { "BAD", LLColor4::red }, // LAST_STATE+3 { "MIS", LLColor4::red }, // LAST_STATE+4 - { "---", LLColor4::white }, // LAST_STATE+5 + { "-!-", LLColor4::white }, // LAST_STATE+5 }; const S32 fetch_state_desc_size = (S32)LL_ARRAY_SIZE(fetch_state_desc); S32 state = @@ -370,8 +375,39 @@ void LLTextureBar::draw() // draw the image size at the end { - std::string num_str = llformat("%3dx%3d (%d) %7d", mImagep->getWidth(), mImagep->getHeight(), - mImagep->getDiscardLevel(), mImagep->hasGLTexture() ? mImagep->getTextureMemory() : 0); + std::string boost_lvl("UNKNOWN"); + switch(mImagep->getBoostLevel()) + { +#define BOOST_LVL(type) case LLGLTexture::BOOST_##type: boost_lvl="B_"#type; break; +#define CAT_LVL(type) case LLGLTexture::type: boost_lvl=#type; break; + BOOST_LVL(NONE) + BOOST_LVL(AVATAR_BAKED) + BOOST_LVL(AVATAR) + BOOST_LVL(CLOUDS) + BOOST_LVL(SCULPTED) + BOOST_LVL(HIGH) + BOOST_LVL(BUMP) + BOOST_LVL(TERRAIN) + BOOST_LVL(SELECTED) + BOOST_LVL(AVATAR_BAKED_SELF) + BOOST_LVL(AVATAR_SELF) + BOOST_LVL(SUPER_HIGH) + BOOST_LVL(HUD) + BOOST_LVL(ICON) + BOOST_LVL(UI) + BOOST_LVL(PREVIEW) + BOOST_LVL(MAP) + BOOST_LVL(MAP_VISIBLE) + CAT_LVL(LOCAL) + CAT_LVL(AVATAR_SCRATCH_TEX) + CAT_LVL(DYNAMIC_TEX) + CAT_LVL(MEDIA) + CAT_LVL(OTHER) + CAT_LVL(MAX_GL_IMAGE_CATEGORY) + default:; + }; + std::string num_str = llformat("%4dx%4d (%+d) %7d %s", mImagep->getWidth(), mImagep->getHeight(), + mImagep->getDiscardLevel(), mImagep->hasGLTexture() ? mImagep->getTextureMemory() : 0, boost_lvl.c_str()); LLFontGL::getFontMonospace()->renderUTF8(num_str, 0, title_x4, getRect().getHeight(), color, LLFontGL::LEFT, LLFontGL::TOP); } diff --git a/indra/newview/llviewertexture.cpp b/indra/newview/llviewertexture.cpp index 19681d38e..f8e0f43ee 100644 --- a/indra/newview/llviewertexture.cpp +++ b/indra/newview/llviewertexture.cpp @@ -926,7 +926,7 @@ void LLViewerFetchedTexture::init(bool firstinit) mIsMissingAsset = FALSE; mLoadedCallbackDesiredDiscardLevel = S8_MAX; - mPauseLoadedCallBacks = TRUE ; + mPauseLoadedCallBacks = FALSE ; mNeedsCreateTexture = FALSE; @@ -1288,6 +1288,7 @@ BOOL LLViewerFetchedTexture::createTexture(S32 usename/*= 0*/) // An inappropriately-sized image was uploaded (through a non standard client) // We treat these images as missing assets which causes them to // be renderd as 'missing image' and to stop requesting data + llwarns << "!size_ok, setting as missing" << llendl; setIsMissingAsset(); destroyRawImage(); return FALSE; @@ -1712,6 +1713,7 @@ bool LLViewerFetchedTexture::updateFetch() { //discard all oversized textures. destroyRawImage(); + llwarns << "oversize, setting as missing" << llendl; setIsMissingAsset(); mRawDiscardLevel = INVALID_DISCARD_LEVEL ; mIsFetching = FALSE ; @@ -1741,6 +1743,10 @@ bool LLViewerFetchedTexture::updateFetch() // We finished but received no data if (current_discard < 0) { + llwarns << "!mIsFetching, setting as missing, decode_priority " << decode_priority + << " mRawDiscardLevel " << mRawDiscardLevel + << " current_discard " << current_discard + << llendl; setIsMissingAsset(); desired_discard = -1; } @@ -1781,6 +1787,10 @@ bool LLViewerFetchedTexture::updateFetch() { make_request = false; } + else if(mDesiredDiscardLevel > getMaxDiscardLevel()) + { + make_request = false; + } else if (mNeedsCreateTexture || mIsMissingAsset) { make_request = false; @@ -1789,6 +1799,11 @@ bool LLViewerFetchedTexture::updateFetch() { make_request = false; } + else if(mCachedRawImage.notNull() && (current_discard < 0 || current_discard > mCachedRawDiscardLevel)) + { + make_request = false; + switchToCachedImage() ; //use the cached raw data first + } //else if (!isJustBound() && mCachedRawImageReady) //{ // make_request = false; @@ -1881,11 +1896,14 @@ void LLViewerFetchedTexture::forceToDeleteRequest() { if (mHasFetcher) { - LLAppViewer::getTextureFetch()->deleteRequest(getID(), true); + //LLAppViewer::getTextureFetch()->deleteRequest(getID(), true); mHasFetcher = FALSE; mIsFetching = FALSE ; - resetTextureStats(); } + + resetTextureStats(); + + mDesiredDiscardLevel = getMaxDiscardLevel() + 1; //defer LLAppViewer::getTextureFetch()->deleteRequest to updateFetch? } void LLViewerFetchedTexture::setIsMissingAsset() @@ -1928,10 +1946,18 @@ void LLViewerFetchedTexture::setLoadedCallback( loaded_callback_func loaded_call mLoadedCallbackDesiredDiscardLevel = llmin(mLoadedCallbackDesiredDiscardLevel, (S8)discard_level) ; } - if(mPauseLoadedCallBacks && !pause) + if(mPauseLoadedCallBacks) { - unpauseLoadedCallbacks(src_callback_list) ; + if(!pause) + { + unpauseLoadedCallbacks(src_callback_list) ; + } } + else if(pause) + { + pauseLoadedCallbacks(src_callback_list) ; + } + LLLoadedCallbackEntry* entryp = new LLLoadedCallbackEntry(loaded_callback, discard_level, keep_imageraw, userdata, src_callback_list, this, pause); mLoadedCallbackList.push_back(entryp); diff --git a/indra/newview/llviewertexturelist.cpp b/indra/newview/llviewertexturelist.cpp index 298640e74..17f4343f4 100644 --- a/indra/newview/llviewertexturelist.cpp +++ b/indra/newview/llviewertexturelist.cpp @@ -1466,6 +1466,7 @@ void LLViewerTextureList::processImageNotInDatabase(LLMessageSystem *msg,void ** LLViewerFetchedTexture* image = gTextureList.findImage( image_id ); if( image ) { + llwarns << "not in db" << llendl; image->setIsMissingAsset(); } }