diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 8195e1cc8..c4be02818 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -60,6 +60,27 @@ LLWorkerThread::~LLWorkerThread() // ~LLQueuedThread() will be called here } +//called only in destructor. +void LLWorkerThread::clearDeleteList() +{ + // Delete any workers in the delete queue (should be safe - had better be!) + if (!mDeleteList.empty()) + { + llwarns << "Worker Thread: " << mName << " destroyed with " << mDeleteList.size() + << " entries in delete list." << llendl; + + mDeleteMutex->lock(); + for (delete_list_t::iterator iter = mDeleteList.begin(); iter != mDeleteList.end(); ++iter) + { + (*iter)->mRequestHandle = LLWorkerThread::nullHandle(); + (*iter)->clearFlags(LLWorkerClass::WCF_HAVE_WORK); + delete *iter ; + } + mDeleteList.clear() ; + mDeleteMutex->unlock() ; + } +} + // virtual S32 LLWorkerThread::update(U32 max_time_ms) { @@ -181,6 +202,7 @@ void LLWorkerThread::WorkRequest::finishRequest(bool completed) LLWorkerClass::LLWorkerClass(LLWorkerThread* workerthread, const std::string& name) : mWorkerThread(workerthread), + mRequestPriority(LLWorkerThread::PRIORITY_NORMAL), mWorkerClassName(name), mRequestHandle(LLWorkerThread::nullHandle()), mMutex(NULL), @@ -314,7 +336,20 @@ bool LLWorkerClass::checkWork(bool aborting) if (mRequestHandle != LLWorkerThread::nullHandle()) { LLWorkerThread::WorkRequest* workreq = (LLWorkerThread::WorkRequest*)mWorkerThread->getRequest(mRequestHandle); + if(!workreq) + { + if(mWorkerThread->isQuitting() || mWorkerThread->isStopped()) //the mWorkerThread is not running + { + mRequestHandle = LLWorkerThread::nullHandle(); + clearFlags(WCF_HAVE_WORK); + } + else + { llassert_always(workreq); + } + return true ; + } + LLQueuedThread::status_t status = workreq->getStatus(); if (status == LLWorkerThread::STATUS_ABORTED) { @@ -364,7 +399,7 @@ void LLWorkerClass::scheduleDelete() void LLWorkerClass::setPriority(U32 priority) { mMutex.lock(); - if (mRequestHandle != LLWorkerThread::nullHandle()) + if (mRequestHandle != LLWorkerThread::nullHandle() && mRequestPriority != priority) { mRequestPriority = priority; mWorkerThread->setPriority(mRequestHandle, priority); diff --git a/indra/llcommon/llworkerthread.h b/indra/llcommon/llworkerthread.h index a1e85d2ec..3d761dc87 100644 --- a/indra/llcommon/llworkerthread.h +++ b/indra/llcommon/llworkerthread.h @@ -80,6 +80,9 @@ public: S32 mParam; }; +protected: + void clearDeleteList(); + private: typedef std::list delete_list_t; delete_list_t mDeleteList; diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 006f82c1a..1ddb86ec2 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -11236,7 +11236,7 @@ Type F32 Value - 500.0 + 2000.0 ToolHelpRect diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index f8bbccf47..2455b7d34 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -1428,6 +1428,8 @@ bool LLAppViewer::cleanup() sTextureCache->shutdown(); sTextureFetch->shutdown(); sImageDecodeThread->shutdown(); + sTextureFetch->shutDownTextureCacheThread(); + sTextureFetch->shutDownImageDecodeThread(); delete sTextureCache; sTextureCache = NULL; delete sTextureFetch; @@ -2853,6 +2855,22 @@ void LLAppViewer::migrateCacheDirectory() bool LLAppViewer::initCache() { mPurgeCache = false; + BOOL read_only = mSecondInstance ? TRUE : FALSE; + LLAppViewer::getTextureCache()->setReadOnly(read_only); + + BOOL texture_cache_mismatch = FALSE ; + static const S32 cache_version = 7; + if (gSavedSettings.getS32("LocalCacheVersion") != cache_version) + { + texture_cache_mismatch = TRUE ; + if (!read_only) + { + gSavedSettings.setS32("LocalCacheVersion", cache_version); + } + } + + if (!read_only) + { // Purge cache if user requested it if (gSavedSettings.getBOOL("PurgeCacheOnStartup") || gSavedSettings.getBOOL("PurgeCacheOnNextStartup")) @@ -2860,16 +2878,6 @@ bool LLAppViewer::initCache() gSavedSettings.setBOOL("PurgeCacheOnNextStartup", false); mPurgeCache = true; } - // Purge cache if it belongs to an old version - else - { - static const S32 cache_version = 5; - if (gSavedSettings.getS32("LocalCacheVersion") != cache_version) - { - mPurgeCache = true; - gSavedSettings.setS32("LocalCacheVersion", cache_version); - } - } // We have moved the location of the cache directory over time. migrateCacheDirectory(); @@ -2883,6 +2891,7 @@ bool LLAppViewer::initCache() purgeCache(); // purge old cache gSavedSettings.setString("CacheLocation", new_cache_location); } + } if (!gDirUtilp->setCacheDir(gSavedSettings.getString("CacheLocation"))) { @@ -2890,7 +2899,7 @@ bool LLAppViewer::initCache() gSavedSettings.setString("CacheLocation", ""); } - if (mPurgeCache) + if (mPurgeCache && !read_only) { LLSplashScreen::update("Clearing cache..."); purgeCache(); @@ -2900,13 +2909,12 @@ bool LLAppViewer::initCache() // Init the texture cache // Allocate 80% of the cache size for textures - BOOL read_only = mSecondInstance ? TRUE : FALSE; const S32 MB = 1024*1024; S64 cache_size = (S64)(gSavedSettings.getU32("CacheSize")) * MB; const S64 MAX_CACHE_SIZE = 1024*MB; cache_size = llmin(cache_size, MAX_CACHE_SIZE); S64 texture_cache_size = ((cache_size * 8)/10); - S64 extra = LLAppViewer::getTextureCache()->initCache(LL_PATH_CACHE, texture_cache_size, read_only); + S64 extra = LLAppViewer::getTextureCache()->initCache(LL_PATH_CACHE, texture_cache_size, texture_cache_mismatch); texture_cache_size -= extra; LLSplashScreen::update("Initializing VFS..."); diff --git a/indra/newview/llface.cpp b/indra/newview/llface.cpp index 3d93dd4d2..f8e79e735 100644 --- a/indra/newview/llface.cpp +++ b/indra/newview/llface.cpp @@ -1252,16 +1252,17 @@ F32 LLFace::getTextureVirtualSize() { F32 radius; F32 cos_angle_to_view_dir; - mPixelArea = calcPixelArea(cos_angle_to_view_dir, radius); + BOOL in_frustum = calcPixelArea(cos_angle_to_view_dir, radius); - if (mPixelArea <= 0) + if (mPixelArea < F_ALMOST_ZERO || !in_frustum) { + setVirtualSize(0.f) ; return 0.f; } //get area of circle in texture space LLVector2 tdim = mTexExtents[1] - mTexExtents[0]; - F32 texel_area = (tdim * 0.5f).lengthSquared()*3.14159f; + F32 texel_area = (tdim * 0.5f).lengthSquared() * 3.14159f; if (texel_area <= 0) { // Probably animated, use default @@ -1278,59 +1279,73 @@ F32 LLFace::getTextureVirtualSize() { //apply texel area to face area to get accurate ratio //face_area /= llclamp(texel_area, 1.f/64.f, 16.f); - - //face_area = mPixelArea / llclamp(texel_area, 0.015625f, 1024.f); - face_area = mPixelArea / llclamp(texel_area, 0.015625f, 128.f); // see SNOW-207 + face_area = mPixelArea / llclamp(texel_area, 0.015625f, 128.f); } - if(face_area > LLViewerImage::sMaxSmallImageSize) + face_area = LLFace::adjustPixelArea(mImportanceToCamera, face_area); + if (mImportanceToCamera < 1.0f && face_area > LLViewerImage::sMinLargeImageSize) //if is large image, shrink face_area by considering the partial overlapping. { - if(mImportanceToCamera < LEAST_IMPORTANCE) //if the face is not important, do not load hi-res. + if (mImportanceToCamera > LEAST_IMPORTANCE_FOR_LARGE_IMAGE && mTexture.notNull() && mTexture->isLargeImage()) { - face_area = LLViewerImage::sMaxSmallImageSize ; + face_area *= adjustPartialOverlapPixelArea(cos_angle_to_view_dir, radius); + } } - else if(face_area > LLViewerImage::sMinLargeImageSize) //if is large image, shrink face_area by considering the partial overlapping. - { - if(mImportanceToCamera < LEAST_IMPORTANCE_FOR_LARGE_IMAGE)//if the face is not important, do not load hi-res. - { - face_area = LLViewerImage::sMinLargeImageSize ; - } - else if(mTexture.notNull() && mTexture->isLargeImage()) - { - face_area *= adjustPartialOverlapPixelArea(cos_angle_to_view_dir, radius ); - } - } - } + + setVirtualSize(face_area); return face_area; } -F32 LLFace::calcPixelArea(F32& cos_angle_to_view_dir, F32& radius) +//static +F32 LLFace::adjustPixelArea(F32 importance, F32 pixel_area) +{ + if (pixel_area > LLViewerImage::sMaxSmallImageSize) + { + if (importance < LEAST_IMPORTANCE) //if the face is not important, do not load hi-res. + { + static const F32 MAX_LEAST_IMPORTANCE_IMAGE_SIZE = 128.0f * 128.0f; + pixel_area = llmin(pixel_area * 0.5f, MAX_LEAST_IMPORTANCE_IMAGE_SIZE); + } + else if (pixel_area > LLViewerImage::sMinLargeImageSize) //if is large image, shrink face_area by considering the partial overlapping. + { + if (importance < LEAST_IMPORTANCE_FOR_LARGE_IMAGE) //if the face is not important, do not load hi-res. + { + pixel_area = LLViewerImage::sMinLargeImageSize; + } + } + } + + return pixel_area ; +} + +BOOL LLFace::calcPixelArea(F32& cos_angle_to_view_dir, F32& radius) { //get area of circle around face LLVector3 center = getPositionAgent(); LLVector3 size = (mExtents[1] - mExtents[0]) * 0.5f; + LLViewerCamera* camera = LLViewerCamera::getInstance(); - LLVector3 lookAt = center - LLViewerCamera::getInstance()->getOrigin(); + F32 size_squared = size.lengthSquared() ; + LLVector3 lookAt = center - camera->getOrigin(); F32 dist = lookAt.normVec() ; //get area of circle around node - F32 app_angle = atanf(size.length()/dist); + F32 app_angle = atanf(fsqrtf(size_squared) / dist); radius = app_angle*LLDrawable::sCurPixelAngle; - F32 face_area = radius*radius * 3.14159f; + mPixelArea = radius*radius * 3.14159f; + cos_angle_to_view_dir = lookAt * camera->getXAxis(); - if(dist < mBoundingSphereRadius) //camera is very close + if (dist < mBoundingSphereRadius || dist < 10.0f) //camera is very close { - cos_angle_to_view_dir = 1.0f ; - mImportanceToCamera = 1.0f ; + cos_angle_to_view_dir = 1.0f; + mImportanceToCamera = 1.0f; } else { - cos_angle_to_view_dir = lookAt * LLViewerCamera::getInstance()->getXAxis() ; - mImportanceToCamera = LLFace::calcImportanceToCamera(cos_angle_to_view_dir, dist) ; + mImportanceToCamera = LLFace::calcImportanceToCamera(cos_angle_to_view_dir, dist); } - return face_area ; + return true; } //the projection of the face partially overlaps with the screen diff --git a/indra/newview/llface.h b/indra/newview/llface.h index 4893e825d..f8a3fa50d 100644 --- a/indra/newview/llface.h +++ b/indra/newview/llface.h @@ -194,9 +194,10 @@ public: private: F32 adjustPartialOverlapPixelArea(F32 cos_angle_to_view_dir, F32 radius ); - F32 calcPixelArea(F32& cos_angle_to_view_dir, F32& radius) ; + BOOL calcPixelArea(F32& cos_angle_to_view_dir, F32& radius); public: static F32 calcImportanceToCamera(F32 to_view_dir, F32 dist); + static F32 adjustPixelArea(F32 importance, F32 pixel_area); public: diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index a1a9a3985..567c02500 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -48,11 +48,11 @@ // Unordered array of Entry structs // cache/texture.cache // First TEXTURE_CACHE_ENTRY_SIZE bytes of each texture in texture.entries in same order -// Entry size same as header packet, so we're not 0-padding unless whole image is contained in header. // cache/textures/[0-F]/UUID.texture // Actual texture body files -const S32 TEXTURE_CACHE_ENTRY_SIZE = FIRST_PACKET_SIZE; +//note: there is no good to define 1024 for TEXTURE_CACHE_ENTRY_SIZE while FIRST_PACKET_SIZE is 600 on sim side. +const S32 TEXTURE_CACHE_ENTRY_SIZE = FIRST_PACKET_SIZE;//1024; const F32 TEXTURE_CACHE_PURGE_AMOUNT = .20f; // % amount to reduce the cache by when it exceeds its limit const F32 TEXTURE_CACHE_LRU_SIZE = .10f; // % amount for LRU list (low overhead to regenerate) @@ -250,9 +250,9 @@ bool LLTextureCacheLocalFileWorker::doRead() } } #else - if (!mDataSize || mDataSize > local_size) + if (!mDataSize || mDataSize > local_size - mOffset) { - mDataSize = local_size; + mDataSize = local_size - mOffset; } mReadData = new U8[mDataSize]; @@ -372,8 +372,7 @@ bool LLTextureCacheRemoteWorker::doRead() if (!done && (mState == LOCAL)) { llassert(local_size != 0); // we're assuming there is a non empty local file here... - llassert(mReadData == NULL); - if (!mDataSize || mDataSize > (local_size - mOffset)) + if (!mDataSize || mDataSize > local_size - mOffset) { mDataSize = local_size - mOffset; } @@ -391,6 +390,7 @@ bool LLTextureCacheRemoteWorker::doRead() } else { + //llinfos << "texture " << mID.asString() << " found in local_assets" << llendl; mImageSize = local_size; mImageLocal = TRUE; } @@ -401,7 +401,8 @@ bool LLTextureCacheRemoteWorker::doRead() // Second state / stage : identify the cache or not... if (!done && (mState == CACHE)) { - idx = mCache->getHeaderCacheEntry(mID, mImageSize); + LLTextureCache::Entry entry; + idx = mCache->getHeaderCacheEntry(mID, entry); if (idx < 0) { // The texture is *not* cached. We're done here... @@ -410,6 +411,7 @@ bool LLTextureCacheRemoteWorker::doRead() } else { + mImageSize = entry.mImageSize; // If the read offset is bigger than the header cache, we read directly from the body // Note that currently, we *never* read with offset from the cache, so the result is *always* HEADER mState = mOffset < TEXTURE_CACHE_ENTRY_SIZE ? HEADER : BODY; @@ -427,8 +429,7 @@ bool LLTextureCacheRemoteWorker::doRead() size = llmin(size, mDataSize); // Allocate the read buffer mReadData = new U8[size]; - S32 bytes_read = LLAPRFile::readEx(mCache->mHeaderDataFileName, - mReadData, offset, size); + S32 bytes_read = LLAPRFile::readEx(mCache->mHeaderDataFileName, mReadData, offset, size); if (bytes_read != size) { llwarns << "LLTextureCacheWorker: " << mID @@ -494,14 +495,13 @@ bool LLTextureCacheRemoteWorker::doRead() mReadData = data; // Read the data at last - S32 bytes_read = LLAPRFile::readEx(filename, - mReadData + data_offset, + S32 bytes_read = LLAPRFile::readEx(filename, mReadData + data_offset, file_offset, file_size); if (bytes_read != file_size) { - llwarns << "LLTextureCacheWorker: " << mID + LL_DEBUGS("TextureCache") << "LLTextureCacheWorker: " << mID << " incorrect number of bytes read from body: " << bytes_read - << " / " << file_size << llendl; + << " / " << file_size << LL_ENDL; delete[] mReadData; mReadData = NULL; mDataSize = -1; // failed @@ -510,9 +510,19 @@ bool LLTextureCacheRemoteWorker::doRead() } else { + if (mImageSize > TEXTURE_CACHE_ENTRY_SIZE) + { + LL_DEBUGS("TextureCache") << "LLTextureCacheWorker: no body for texture: " << mID << LL_ENDL; + delete[] mReadData; + mReadData = NULL; + mDataSize = -1; // failed + done = true; + } + else + { // No body, we're done. mDataSize = llmax(TEXTURE_CACHE_ENTRY_SIZE - mOffset, 0); - lldebugs << "No body file for: " << filename << llendl; + } } // Nothing else to do at that point... done = true; @@ -537,6 +547,7 @@ bool LLTextureCacheRemoteWorker::doWrite() { llassert_always(mOffset == 0); // We currently do not support write offsets llassert_always(mDataSize > 0); // Things will go badly wrong if mDataSize is nul or negative... + llassert_always(mImageSize >= mDataSize); mState = CACHE; } @@ -546,14 +557,19 @@ bool LLTextureCacheRemoteWorker::doWrite() if (!done && (mState == CACHE)) { bool alreadyCached = false; - S32 cur_imagesize = 0; + LLTextureCache::Entry entry; + // Checks if this image is already in the entry list - idx = mCache->getHeaderCacheEntry(mID, cur_imagesize); - if (idx >= 0 && (cur_imagesize >= 0)) + idx = mCache->getHeaderCacheEntry(mID, entry); + if(idx < 0) { - alreadyCached = true; // already there and non empty + idx = mCache->setHeaderCacheEntry(mID, entry, mImageSize, mDataSize); // create the new entry. } - idx = mCache->setHeaderCacheEntry(mID, mImageSize); // create or touch the entry + else + { + alreadyCached = mCache->updateEntry(idx, entry, mImageSize, mDataSize); // update the existing entry. + } + if (idx < 0) { llwarns << "LLTextureCacheWorker: " << mID @@ -563,10 +579,6 @@ bool LLTextureCacheRemoteWorker::doWrite() } else { - if (cur_imagesize > 0 && (mImageSize != cur_imagesize)) - { - alreadyCached = false; // re-write the header if the size changed in all cases - } if (alreadyCached && (mDataSize <= TEXTURE_CACHE_ENTRY_SIZE)) { // Small texture already cached case: we're done with writing @@ -629,14 +641,12 @@ bool LLTextureCacheRemoteWorker::doWrite() { llassert(mDataSize > TEXTURE_CACHE_ENTRY_SIZE); // wouldn't make sense to be here otherwise... S32 file_size = mDataSize - TEXTURE_CACHE_ENTRY_SIZE; - if ((file_size > 0) && mCache->updateTextureEntryList(mID, file_size)) + { // build the cache file name from the UUID std::string filename = mCache->getTextureFileName(mID); // llinfos << "Writing Body: " << filename << " Bytes: " << file_offset+file_size << llendl; - S32 bytes_written = LLAPRFile::writeEx( filename, - mWriteData + TEXTURE_CACHE_ENTRY_SIZE, - 0, file_size); + S32 bytes_written = LLAPRFile::writeEx(filename, mWriteData + TEXTURE_CACHE_ENTRY_SIZE, 0, file_size); if (bytes_written <= 0) { llwarns << "LLTextureCacheWorker: " << mID @@ -646,10 +656,7 @@ bool LLTextureCacheRemoteWorker::doWrite() done = true; } } - else - { - mDataSize = 0; // no data written - } + // Nothing else to do at that point... done = true; } @@ -740,14 +747,18 @@ LLTextureCache::LLTextureCache(bool threaded) mHeaderMutex(NULL), mListMutex(NULL), mHeaderAPRFile(NULL), - mReadOnly(FALSE), + mReadOnly(TRUE), //do not allow to change the texture cache until setReadOnly() is called. mTexturesSizeTotal(0), mDoPurge(FALSE) { + purgeTextures(false, true); + clearDeleteList(); + writeUpdatedEntries(); } LLTextureCache::~LLTextureCache() { + purgeTextureFilesTimeSliced(TRUE); // VWR-3878 - NB - force-flush all pending file deletes } ////////////////////////////////////////////////////////////////////////////// @@ -755,6 +766,9 @@ LLTextureCache::~LLTextureCache() //virtual S32 LLTextureCache::update(U32 max_time_ms) { + static LLFrameTimer timer; + static const F32 MAX_TIME_INTERVAL = 300.f; //seconds. + S32 res; res = LLWorkerThread::update(max_time_ms); @@ -790,6 +804,12 @@ S32 LLTextureCache::update(U32 max_time_ms) responder->completed(success); } + if(!res && timer.getElapsedTimeF32() > MAX_TIME_INTERVAL) + { + timer.reset(); + writeUpdatedEntries(); + } + return res; } @@ -812,74 +832,73 @@ std::string LLTextureCache::getTextureFileName(const LLUUID& id) return filename; } -bool LLTextureCache::updateTextureEntryList(const LLUUID& id, S32 bodysize) +//debug +BOOL LLTextureCache::isInCache(const LLUUID& id) { - bool res = false; - bool purge = false; - { - mHeaderMutex.lock(); - size_map_t::iterator iter1 = mTexturesSizeMap.find(id); - if (iter1 == mTexturesSizeMap.end() || iter1->second < bodysize) - { - llassert_always(bodysize > 0); + LLMutexLock lock(&mHeaderMutex); + id_map_t::const_iterator iter = mHeaderIDMap.find(id); - S32 oldbodysize = 0; - if (iter1 != mTexturesSizeMap.end()) - { - oldbodysize = iter1->second; - } + return (iter != mHeaderIDMap.end()); +} - Entry entry; - S32 idx = openAndReadEntry(id, entry, false); - if (idx < 0) +//debug +BOOL LLTextureCache::isInLocal(const LLUUID& id) +{ + S32 local_size = 0; + std::string local_filename; + + std::string filename = getLocalFileName(id); + // Is it a JPEG2000 file? { - llwarns << "Failed to open entry: " << id << llendl; - mHeaderMutex.unlock(); - removeFromCache(id); - return false; - } - else if (oldbodysize != entry.mBodySize) + local_filename = filename + ".j2c"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) { - llwarns << "Entry mismatch in mTextureSizeMap / mHeaderIDMap" - << " idx=" << idx << " oldsize=" << oldbodysize << " entrysize=" << entry.mBodySize << llendl; + return TRUE; + } } - entry.mBodySize = bodysize; - writeEntryAndClose(idx, entry); - mTexturesSizeTotal -= oldbodysize; - mTexturesSizeTotal += bodysize; - - if (mTexturesSizeTotal > sCacheMaxTexturesSize) + // If not, is it a jpeg file? { - purge = true; - } - res = true; + local_filename = filename + ".jpg"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) + { + return TRUE; } } - if (purge) + + // Hmm... What about a targa file? (used for UI texture mostly) { - mDoPurge = TRUE; + local_filename = filename + ".tga"; + local_size = LLAPRFile::size(local_filename); + if (local_size > 0) + { + return TRUE; + } } - mHeaderMutex.unlock(); - return res; -} + return FALSE; +} ////////////////////////////////////////////////////////////////////////////// //static const S32 MAX_REASONABLE_FILE_SIZE = 512*1024*1024; // 512 MB -F32 LLTextureCache::sHeaderCacheVersion = 1.3f; +F32 LLTextureCache::sHeaderCacheVersion = 1.4f; U32 LLTextureCache::sCacheMaxEntries = MAX_REASONABLE_FILE_SIZE / TEXTURE_CACHE_ENTRY_SIZE; S64 LLTextureCache::sCacheMaxTexturesSize = 0; // no limit const char* entries_filename = "texture.entries"; const char* cache_filename = "texture.cache"; -const char* textures_dirname = "textures"; +const char* old_textures_dirname = "textures"; +//change the location of the texture cache to prevent from being deleted by old version viewers. +const char* textures_dirname = "texturecache"; void LLTextureCache::setDirNames(ELLPath location) { std::string delem = gDirUtilp->getDirDelimiter(); - mHeaderEntriesFileName = gDirUtilp->getExpandedFilename(location, entries_filename); - mHeaderDataFileName = gDirUtilp->getExpandedFilename(location, cache_filename); + + mHeaderEntriesFileName = gDirUtilp->getExpandedFilename(location, textures_dirname, entries_filename); + mHeaderDataFileName = gDirUtilp->getExpandedFilename(location, textures_dirname, cache_filename); mTexturesDirName = gDirUtilp->getExpandedFilename(location, textures_dirname); } @@ -891,15 +910,43 @@ void LLTextureCache::purgeCache(ELLPath location) { setDirNames(location); llassert_always(mHeaderAPRFile == NULL); - LLAPRFile::remove(mHeaderEntriesFileName); - LLAPRFile::remove(mHeaderDataFileName); + + //remove the legacy cache if exists + std::string texture_dir = mTexturesDirName; + mTexturesDirName = gDirUtilp->getExpandedFilename(location, old_textures_dirname); + if(LLFile::isdir(mTexturesDirName)) + { + std::string file_name = gDirUtilp->getExpandedFilename(location, entries_filename); + if (LLAPRFile::isExist(file_name)) + { + LLAPRFile::remove(file_name); + } + + file_name = gDirUtilp->getExpandedFilename(location, cache_filename); + if (LLAPRFile::isExist(file_name)) + { + LLAPRFile::remove(file_name); + } + + purgeAllTextures(true); + } + mTexturesDirName = texture_dir; } + + //remove the current texture cache. purgeAllTextures(true); } -S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) +//is called in the main thread before initCache(...) is called. +void LLTextureCache::setReadOnly(BOOL read_only) { mReadOnly = read_only; +} + +//called in the main thread. +S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL texture_cache_mismatch) +{ + llassert_always(getPending() == 0); //should not start accessing the texture cache before initialized. S64 header_size = (max_size * 2) / 10; S64 max_entries = header_size / TEXTURE_CACHE_ENTRY_SIZE; @@ -917,9 +964,22 @@ S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) setDirNames(location); + if(texture_cache_mismatch) + { + //if readonly, disable the texture cache, + //otherwise wipe out the texture cache. + purgeAllTextures(true); + + if(mReadOnly) + { + return max_size; + } + } + if (!mReadOnly) { LLFile::mkdir(mTexturesDirName); + const char* subdirs = "0123456789abcdef"; for (S32 i=0; i<16; i++) { @@ -930,6 +990,8 @@ S64 LLTextureCache::initCache(ELLPath location, S64 max_size, BOOL read_only) readHeaderCache(); purgeTextures(true); // calc mTexturesSize and make some room in the texture cache if we need it + llassert_always(getPending() == 0); //should not start accessing the texture cache before initialized. + return max_size; // unused cache space } @@ -941,13 +1003,20 @@ LLAPRFile* LLTextureCache::openHeaderEntriesFile(bool readonly, S32 offset) llassert_always(mHeaderAPRFile == NULL); apr_int32_t flags = readonly ? APR_READ|APR_BINARY : APR_READ|APR_WRITE|APR_BINARY; mHeaderAPRFile = new LLAPRFile(mHeaderEntriesFileName, flags, LLAPRFile::local); + if(offset > 0) + { mHeaderAPRFile->seek(APR_SET, offset); + } return mHeaderAPRFile; } void LLTextureCache::closeHeaderEntriesFile() { - llassert_always(mHeaderAPRFile != NULL); + if(!mHeaderAPRFile) + { + return; + } + delete mHeaderAPRFile; mHeaderAPRFile = NULL; } @@ -960,6 +1029,12 @@ void LLTextureCache::readEntriesHeader() { LLAPRFile::readEx(mHeaderEntriesFileName, (U8*)&mHeaderEntriesInfo, 0, sizeof(EntriesInfo)); } + else //create an empty entries header. + { + mHeaderEntriesInfo.mVersion = sHeaderCacheVersion; + mHeaderEntriesInfo.mEntries = 0; + writeEntriesHeader(); + } } void LLTextureCache::writeEntriesHeader() @@ -971,8 +1046,7 @@ void LLTextureCache::writeEntriesHeader() } } -static S32 mHeaderEntriesMaxWriteIdx = 0; - +//mHeaderMutex is locked before calling this. S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create) { S32 idx = -1; @@ -1012,8 +1086,7 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create if (iter3 != mHeaderIDMap.end() && iter3->second >= 0) { idx = iter3->second; - mHeaderIDMap.erase(oldid); - mTexturesSizeMap.erase(oldid); + removeCachedTexture(oldid);//remove the existing cached texture to release the entry index. break; } } @@ -1023,20 +1096,9 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create } if (idx >= 0) { - // Set the header index - mHeaderIDMap[id] = idx; - llassert_always(mTexturesSizeMap.erase(id) == 0); - // Initialize the entry (will get written later) - entry.init(id, time(NULL)); - // Update Header - writeEntriesHeader(); - // Write Entry - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(false, offset); - S32 bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, idx); - closeHeaderEntriesFile(); + entry.mID = id; + entry.mImageSize = -1; //mark it is a brand-new entry. + entry.mBodySize = 0; } } } @@ -1045,46 +1107,155 @@ S32 LLTextureCache::openAndReadEntry(const LLUUID& id, Entry& entry, bool create // Remove this entry from the LRU if it exists mLRU.erase(id); // Read the entry - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(true, offset); - S32 bytes_read = aprfile->read((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_read == sizeof(Entry)); - llassert_always(entry.mImageSize == 0 || entry.mImageSize == -1 || entry.mImageSize > entry.mBodySize); - closeHeaderEntriesFile(); + idx_entry_map_t::iterator iter = mUpdatedEntryMap.find(idx); + if(iter != mUpdatedEntryMap.end()) + { + entry = iter->second; + } + else + { + readEntryFromHeaderImmediately(idx, entry); + } + if(entry.mImageSize <= entry.mBodySize)//it happens on 64-bit systems, do not know why + { + llwarns << "corrupted entry: " << id << " entry image size: " << entry.mImageSize << " entry body size: " << entry.mBodySize << llendl; + + //erase this entry and the cached texture from the cache. + std::string tex_filename = getTextureFileName(id); + removeEntry(idx, entry, tex_filename); + mUpdatedEntryMap.erase(idx); + idx = -1; + } } return idx; } -void LLTextureCache::writeEntryAndClose(S32 idx, Entry& entry) +//mHeaderMutex is locked before calling this. +void LLTextureCache::writeEntryToHeaderImmediately(S32& idx, Entry& entry, bool write_header) +{ + LLAPRFile* aprfile; + S32 bytes_written; + S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); + if(write_header) + { + aprfile = openHeaderEntriesFile(false, 0); + bytes_written = aprfile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)); + if(bytes_written != sizeof(EntriesInfo)) + { + clearCorruptedCache(); //clear the cache. + idx = -1; //mark the idx invalid. + return; + } + + mHeaderAPRFile->seek(APR_SET, offset); + } + else + { + aprfile = openHeaderEntriesFile(false, offset); + } + bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); + if(bytes_written != sizeof(Entry)) + { + clearCorruptedCache(); //clear the cache. + idx = -1; //mark the idx invalid. + + return; + } + + closeHeaderEntriesFile(); + mUpdatedEntryMap.erase(idx); +} + +//mHeaderMutex is locked before calling this. +void LLTextureCache::readEntryFromHeaderImmediately(S32& idx, Entry& entry) { + S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); + LLAPRFile* aprfile = openHeaderEntriesFile(true, offset); + S32 bytes_read = aprfile->read((void*)&entry, (S32)sizeof(Entry)); + closeHeaderEntriesFile(); + + if(bytes_read != sizeof(Entry)) + { + clearCorruptedCache(); //clear the cache. + idx = -1;//mark the idx invalid. + } +} + +//mHeaderMutex is locked before calling this. +//update an existing entry time stamp, delay writing. +void LLTextureCache::updateEntryTimeStamp(S32 idx, Entry& entry) +{ + static const U32 MAX_ENTRIES_WITHOUT_TIME_STAMP = (U32)(LLTextureCache::sCacheMaxEntries * 0.75f); + + if(mHeaderEntriesInfo.mEntries < MAX_ENTRIES_WITHOUT_TIME_STAMP) + { + return; //there are enough empty entry index space, no need to stamp time. + } + if (idx >= 0) { if (!mReadOnly) { entry.mTime = time(NULL); - if(entry.mImageSize < entry.mBodySize) - { - // Just say no, due to my messing around to cache discards other than 0 we can end up here - // after recalling an image from cache at a lower discard than cached. RC - return; - } - - llassert_always(entry.mImageSize == 0 || entry.mImageSize == -1 || entry.mImageSize > entry.mBodySize); - if (entry.mBodySize > 0) - { - mTexturesSizeMap[entry.mID] = entry.mBodySize; - } -// llinfos << "Updating TE: " << idx << ": " << id << " Size: " << entry.mBodySize << " Time: " << entry.mTime << llendl; - S32 offset = sizeof(EntriesInfo) + idx * sizeof(Entry); - LLAPRFile* aprfile = openHeaderEntriesFile(false, offset); - S32 bytes_written = aprfile->write((void*)&entry, (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, idx); - closeHeaderEntriesFile(); + mUpdatedEntryMap[idx] = entry; } } } +//update an existing entry, write to header file immediately. +bool LLTextureCache::updateEntry(S32& idx, Entry& entry, S32 new_image_size, S32 new_data_size) +{ + S32 new_body_size = llmax(0, new_data_size - TEXTURE_CACHE_ENTRY_SIZE); + + if(new_image_size == entry.mImageSize && new_body_size == entry.mBodySize) + { + return true; //nothing changed. + } + else + { + bool purge = false; + + lockHeaders(); + + bool update_header = false; + if(entry.mImageSize < 0) //is a brand-new entry + { + mHeaderIDMap[entry.mID] = idx; + mTexturesSizeMap[entry.mID] = new_body_size; + mTexturesSizeTotal += new_body_size; + + // Update Header + update_header = true; + } + else if (entry.mBodySize != new_body_size) + { + //already in mHeaderIDMap. + mTexturesSizeMap[entry.mID] = new_body_size; + mTexturesSizeTotal -= entry.mBodySize; + mTexturesSizeTotal += new_body_size; + } + entry.mTime = time(NULL); + entry.mImageSize = new_image_size; + entry.mBodySize = new_body_size; + + writeEntryToHeaderImmediately(idx, entry, update_header); + + if (mTexturesSizeTotal > sCacheMaxTexturesSize) + { + purge = true; + } + + unlockHeaders(); + + if (purge) + { + mDoPurge = TRUE; + } + } + + return false; +} + U32 LLTextureCache::openAndReadEntries(std::vector& entries) { U32 num_entries = mHeaderEntriesInfo.mEntries; @@ -1094,7 +1265,21 @@ U32 LLTextureCache::openAndReadEntries(std::vector& entries) mFreeList.clear(); mTexturesSizeTotal = 0; - LLAPRFile* aprfile = openHeaderEntriesFile(false, (S32)sizeof(EntriesInfo)); + LLAPRFile* aprfile = NULL; + if(mUpdatedEntryMap.empty()) + { + aprfile = openHeaderEntriesFile(true, (S32)sizeof(EntriesInfo)); + } + else //update the header file first. + { + aprfile = openHeaderEntriesFile(false, 0); + updatedHeaderEntriesFile(); + if(!aprfile) + { + return 0; + } + aprfile->seek(APR_SET, (S32)sizeof(EntriesInfo)); + } for (U32 idx=0; idx& entries) } entries.push_back(entry); // llinfos << "ENTRY: " << entry.mTime << " TEX: " << entry.mID << " IDX: " << idx << " Size: " << entry.mImageSize << llendl; - if (entry.mImageSize < 0) - { - mFreeList.insert(idx); - } - else + if(entry.mImageSize > entry.mBodySize) { mHeaderIDMap[entry.mID] = idx; - if (entry.mBodySize > 0) - { mTexturesSizeMap[entry.mID] = entry.mBodySize; mTexturesSizeTotal += entry.mBodySize; } - llassert_always(entry.mImageSize == 0 || entry.mImageSize > entry.mBodySize); + else + { + mFreeList.insert(idx); } } closeHeaderEntriesFile(); @@ -1138,13 +1319,65 @@ void LLTextureCache::writeEntriesAndClose(const std::vector& entries) for (S32 idx=0; idxwrite((void*)(&entries[idx]), (S32)sizeof(Entry)); - llassert_always(bytes_written == sizeof(Entry)); + if(bytes_written != sizeof(Entry)) + { + clearCorruptedCache(); //clear the cache. + return; + } } - mHeaderEntriesMaxWriteIdx = llmax(mHeaderEntriesMaxWriteIdx, num_entries-1); closeHeaderEntriesFile(); } } +void LLTextureCache::writeUpdatedEntries() +{ + lockHeaders(); + if (!mReadOnly && !mUpdatedEntryMap.empty()) + { + openHeaderEntriesFile(false, 0); + updatedHeaderEntriesFile(); + closeHeaderEntriesFile(); + } + unlockHeaders(); +} + +//mHeaderMutex is locked and mHeaderAPRFile is created before calling this. +void LLTextureCache::updatedHeaderEntriesFile() +{ + if (!mReadOnly && !mUpdatedEntryMap.empty() && mHeaderAPRFile) + { + //entriesInfo + mHeaderAPRFile->seek(APR_SET, 0); + S32 bytes_written = mHeaderAPRFile->write((U8*)&mHeaderEntriesInfo, sizeof(EntriesInfo)); + if(bytes_written != sizeof(EntriesInfo)) + { + clearCorruptedCache(); //clear the cache. + return; + } + + //write each updated entry + S32 entry_size = (S32)sizeof(Entry); + S32 prev_idx = -1; + S32 delta_idx; + for (idx_entry_map_t::iterator iter = mUpdatedEntryMap.begin(); iter != mUpdatedEntryMap.end(); ++iter) + { + delta_idx = iter->first - prev_idx - 1; + prev_idx = iter->first; + if(delta_idx) + { + mHeaderAPRFile->seek(APR_CUR, delta_idx * entry_size); + } + + bytes_written = mHeaderAPRFile->write((void*)(&iter->second), entry_size); + if(bytes_written != entry_size) + { + clearCorruptedCache(); //clear the cache. + return; + } + } + mUpdatedEntryMap.clear(); + } +} //---------------------------------------------------------------------------- // Called from either the main thread or the worker thread @@ -1170,13 +1403,12 @@ void LLTextureCache::readHeaderCache() if (num_entries) { U32 empty_entries = 0; - typedef std::pair lru_data_t; + typedef std::pair lru_data_t; std::set lru; - std::set purge_list; + std::set purge_list; for (U32 i=0; i 0) { if (entry.mBodySize > entry.mImageSize) { // Shouldn't happen, failsafe only llwarns << "Bad entry: " << i << ": " << entry.mID << ": BodySize: " << entry.mBodySize << llendl; - purge_list.insert(id); + purge_list.insert(i); } } } } - if (num_entries - empty_entries > sCacheMaxEntries) + if (num_entries > sCacheMaxEntries) { // Special case: cache size was reduced, need to remove entries // Note: After we prune entries, we will call this again and create the LRU - U32 entries_to_purge = (num_entries - empty_entries) - sCacheMaxEntries; + U32 entries_to_purge = (num_entries-empty_entries) - sCacheMaxEntries; llinfos << "Texture Cache Entries: " << num_entries << " Max: " << sCacheMaxEntries << " Empty: " << empty_entries << " Purging: " << entries_to_purge << llendl; - // We can exit the following loop with the given condition, since if we'd reach the end of the lru set we'd have: - // purge_list.size() = lru.size() = num_entries - empty_entries = entries_to_purge + sCacheMaxEntries >= entries_to_purge - for (std::set::iterator iter = lru.begin(); purge_list.size() < entries_to_purge; ++iter) + if (entries_to_purge > 0) { - purge_list.insert(iter->second); + for (std::set::iterator iter = lru.begin(); iter != lru.end(); ++iter) + { + purge_list.insert(iter->second); + if (purge_list.size() >= entries_to_purge) + break; + } } + llassert_always(purge_list.size() >= entries_to_purge); } else { S32 lru_entries = (S32)((F32)sCacheMaxEntries * TEXTURE_CACHE_LRU_SIZE); for (std::set::iterator iter = lru.begin(); iter != lru.end(); ++iter) { - mLRU.insert(iter->second); + mLRU.insert(entries[iter->second].mID); // llinfos << "LRU: " << iter->first << " : " << iter->second << llendl; if (--lru_entries <= 0) break; @@ -1223,9 +1459,10 @@ void LLTextureCache::readHeaderCache() if (purge_list.size() > 0) { - for (std::set::iterator iter = purge_list.begin(); iter != purge_list.end(); ++iter) + for (std::set::iterator iter = purge_list.begin(); iter != purge_list.end(); ++iter) { - removeFromCacheLocked(*iter); + std::string tex_filename = getTextureFileName(entries[*iter].mID); + removeEntry((S32)*iter, entries[*iter], tex_filename); } // If we removed any entries, we need to rebuild the entries list, // write the header, and call this again @@ -1244,7 +1481,7 @@ void LLTextureCache::readHeaderCache() writeEntriesAndClose(new_entries); mHeaderMutex.unlock(); // unlock the mutex before calling again readHeaderCache(); // repeat with new entries file - return; + mHeaderMutex.lock(); } else { @@ -1257,6 +1494,29 @@ void LLTextureCache::readHeaderCache() ////////////////////////////////////////////////////////////////////////////// +//the header mutex is locked before calling this. +void LLTextureCache::clearCorruptedCache() +{ + llwarns << "the texture cache is corrupted, need to be cleared." << llendl; + + closeHeaderEntriesFile();//close possible file handler + purgeAllTextures(false); //clear the cache. + + if (!mReadOnly) //regenerate the directory tree if not exists. + { + LLFile::mkdir(mTexturesDirName); + + const char* subdirs = "0123456789abcdef"; + for (S32 i=0; i<16; i++) + { + std::string dirname = mTexturesDirName + gDirUtilp->getDirDelimiter() + subdirs[i]; + LLFile::mkdir(dirname); + } + } + + return; +} + void LLTextureCache::purgeAllTextures(bool purge_directories) { if (!mReadOnly) @@ -1267,6 +1527,7 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) for (S32 i=0; i<16; i++) { std::string dirname = mTexturesDirName + delem + subdirs[i]; + llinfos << "Deleting files in directory: " << dirname << llendl; gDirUtilp->deleteFilesInDir(dirname,mask); if (purge_directories) { @@ -1275,6 +1536,7 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) } if (purge_directories) { + gDirUtilp->deleteFilesInDir(mTexturesDirName, mask); LLFile::rmdir(mTexturesDirName); } } @@ -1283,15 +1545,31 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) mTexturesSizeTotal = 0; mFreeList.clear(); mTexturesSizeTotal = 0; + mUpdatedEntryMap.clear(); // Info with 0 entries mHeaderEntriesInfo.mVersion = sHeaderCacheVersion; mHeaderEntriesInfo.mEntries = 0; writeEntriesHeader(); + + llinfos << "The entire texture cache is cleared." << llendl; } -void LLTextureCache::purgeTextures(bool validate) +void LLTextureCache::purgeTextures(bool validate, bool force) { + static LLTimer timeout; + const S32 min_purge_count = 20; + const F32 delay_between_passes = 15.0f; + const F32 max_time_per_pass = 0.5f; + static F32 time_per_pass = 0.05f; + + if (!force && timeout.getElapsedTimeF32() <= delay_between_passes) + { + return; + } + mDoPurge = FALSE; + timeout.reset(); + if (mReadOnly) { return; @@ -1312,7 +1590,7 @@ void LLTextureCache::purgeTextures(bool validate) U32 num_entries = openAndReadEntries(entries); if (!num_entries) { - writeEntriesAndClose(entries); + LLAppViewer::instance()->resumeMainloopTimeout(); return; // nothing to purge } @@ -1331,6 +1609,13 @@ void LLTextureCache::purgeTextures(bool validate) time_idx_set.insert(std::make_pair(entries[idx].mTime, idx)); // llinfos << "TIME: " << entries[idx].mTime << " TEX: " << entries[idx].mID << " IDX: " << idx << " Size: " << entries[idx].mImageSize << llendl; } + else + { + LL_WARNS("TextureCache") << "mTexturesSizeMap / mHeaderIDMap corrupted." << LL_ENDL; + clearCorruptedCache(); + LLAppViewer::instance()->resumeMainloopTimeout(); + return; + } } } @@ -1339,17 +1624,31 @@ void LLTextureCache::purgeTextures(bool validate) if (validate) { validate_idx = gSavedSettings.getU32("CacheValidateCounter"); - U32 next_idx = (validate_idx + 1) % 256; + U32 next_idx = (++validate_idx) % 256; gSavedSettings.setU32("CacheValidateCounter", next_idx); LL_DEBUGS("TextureCache") << "TEXTURE CACHE: Validating: " << validate_idx << LL_ENDL; } + F32 overhead = timeout.getElapsedTimeF32(); + if (2.0f * overhead > time_per_pass) + { + time_per_pass = llmin(overhead, max_time_per_pass); + } + timeout.reset(); + S64 cache_size = mTexturesSizeTotal; - S64 purged_cache_size = (sCacheMaxTexturesSize * (S64)((1.f-TEXTURE_CACHE_PURGE_AMOUNT)*100)) / 100; + S64 purged_cache_size = (sCacheMaxTexturesSize * (S64)((1.f - TEXTURE_CACHE_PURGE_AMOUNT) * 100.f)) / 100; S32 purge_count = 0; for (time_idx_set_t::iterator iter = time_idx_set.begin(); iter != time_idx_set.end(); ++iter) { + if (!force && !validate && purge_count >= min_purge_count && timeout.getElapsedTimeF32() > time_per_pass) + { + LL_INFOS("TextureCache") << "Texture cache purge splitted to avoid long hiccup." << LL_ENDL; + mDoPurge = TRUE; + break; + } + S32 idx = iter->second; bool purge_entry = false; std::string filename = getTextureFileName(entries[idx].mID); @@ -1360,7 +1659,7 @@ void LLTextureCache::purgeTextures(bool validate) else if (validate) { // make sure file exists and is the correct size - S32 uuididx = entries[idx].mID.mData[0]; + U32 uuididx = entries[idx].mID.mData[0]; if (uuididx == validate_idx) { LL_DEBUGS("TextureCache") << "Validating: " << filename << "Size: " << entries[idx].mBodySize << LL_ENDL; @@ -1381,38 +1680,76 @@ void LLTextureCache::purgeTextures(bool validate) if (purge_entry) { purge_count++; - LL_DEBUGS("TextureCache") << "PURGING: " << filename << LL_ENDL; - if (entries[idx].mBodySize > 0) - { - LLAPRFile::remove(filename); - } - else if (LLAPRFile::isExist(filename)) // Sanity check. Shouldn't exist. - { - LL_WARNS("TextureCache") << "Entry has zero body size but existing " << filename << ". Deleting file too..." << LL_ENDL; - LLAPRFile::remove(filename); - } + removeEntry(idx, entries[idx], filename); cache_size -= entries[idx].mBodySize; - mTexturesSizeTotal -= entries[idx].mBodySize; - entries[idx].mBodySize = 0; - mTexturesSizeMap.erase(entries[idx].mID); } } + mTimeLastFileDelete.reset(); + LL_DEBUGS("TextureCache") << "TEXTURE CACHE: Writing Entries: " << num_entries << LL_ENDL; + F32 purge_time = timeout.getElapsedTimeF32(); + timeout.reset(); writeEntriesAndClose(entries); - if (!mThreaded) - { + overhead += timeout.getElapsedTimeF32(); + time_per_pass = llmin(overhead, max_time_per_pass); + // *FIX:Mani - watchdog back on. LLAppViewer::instance()->resumeMainloopTimeout(); - } LL_INFOS("TextureCache") << "TEXTURE CACHE:" - << " PURGED: " << purge_count - << " ENTRIES: " << num_entries - << " CACHE SIZE: " << mTexturesSizeTotal / 1024*1024 << " MB" + << " Purged: " << purge_count + << " - Entries: " << num_entries + << " - Cache size: " << mTexturesSizeTotal / (1024 * 1024) << " MB" + << " - Time used for this purge: " << purge_time + overhead << "s (including " << overhead << "s of overhead)." + << LL_ENDL; + + timeout.reset(); +} + +void LLTextureCache::purgeTextureFilesTimeSliced(BOOL force_all) +{ + LLMutexLock lock(&mHeaderMutex); + + F32 delay_between_passes = 1.0f; // seconds + F32 max_time_per_pass = 0.1f; // seconds + + if (!force_all && mTimeLastFileDelete.getElapsedTimeF32() <= delay_between_passes) + { + return; + } + + LLTimer timer; + S32 howmany = 0; + + if (!mFilesToDelete.empty()) + { + LL_INFOS("TEXTURE CACHE") << "purging time sliced with " << mFilesToDelete.size() << " files scheduled for deletion" << llendl; + + for (LLTextureCache::filename_list_t::iterator iter = mFilesToDelete.begin(); iter!=mFilesToDelete.end(); ) + { + LLTextureCache::filename_list_t::iterator iter2 = iter++; + LLAPRFile::remove(*iter2); + mFilesToDelete.erase(iter2); + howmany++; + + if (!force_all && timer.getElapsedTimeF32() > max_time_per_pass) + { + break; + } + } + } + + if (!mFilesToDelete.empty()) + { + LL_INFOS("TEXTURE CACHE") << "purging time sliced with " << howmany << " files deleted (" + << mFilesToDelete.size() << " files left for next pass)" << llendl; + } + + mTimeLastFileDelete.reset(); } ////////////////////////////////////////////////////////////////////////////// @@ -1444,40 +1781,38 @@ LLTextureCacheWorker* LLTextureCache::getWriter(handle_t handle) // Called from work thread // Reads imagesize from the header, updates timestamp -S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, S32& imagesize) +S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, Entry& entry) { LLMutexLock lock(&mHeaderMutex); - Entry entry; S32 idx = openAndReadEntry(id, entry, false); if (idx >= 0) { - imagesize = entry.mImageSize; - writeEntryAndClose(idx, entry); // updates time + updateEntryTimeStamp(idx, entry); // updates time } return idx; } // Writes imagesize to the header, updates timestamp -S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, S32 imagesize) +S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize) { mHeaderMutex.lock(); - llassert_always(imagesize >= 0); - Entry entry; S32 idx = openAndReadEntry(id, entry, true); + mHeaderMutex.unlock(); + if (idx >= 0) { - entry.mImageSize = imagesize; - writeEntryAndClose(idx, entry); - mHeaderMutex.unlock(); + updateEntry(idx, entry, imagesize, datasize); } - else // retry + + if(idx < 0) // retry { - mHeaderMutex.unlock(); readHeaderCache(); // We couldn't write an entry, so refresh the LRU + mHeaderMutex.lock(); llassert_always(!mLRU.empty() || mHeaderEntriesInfo.mEntries < sCacheMaxEntries); mHeaderMutex.unlock(); - idx = setHeaderCacheEntry(id, imagesize); // assert above ensures no inf. recursion + + idx = setHeaderCacheEntry(id, entry, imagesize, datasize); // assert above ensures no inf. recursion } return idx; } @@ -1514,39 +1849,34 @@ LLTextureCache::handle_t LLTextureCache::readFromCache(const LLUUID& id, U32 pri return handle; } -// Return true if the handle is not valid, which is the case -// when the worker was already deleted or is scheduled for deletion. -// -// If the handle exists and a call to worker->complete() returns -// true or abort is true, then the handle is removed and the worker -// scheduled for deletion. + bool LLTextureCache::readComplete(handle_t handle, bool abort) { - lockWorkers(); // Needed for access to mReaders. - + lockWorkers(); handle_map_t::iterator iter = mReaders.find(handle); - bool handle_is_valid = iter != mReaders.end(); - llassert_always(handle_is_valid || abort); LLTextureCacheWorker* worker = NULL; - bool delete_worker = false; - - if (handle_is_valid) + bool complete = false; + if (iter != mReaders.end()) { worker = iter->second; - delete_worker = worker->complete() || abort; - if (delete_worker) + complete = worker->complete(); + + if(!complete && abort) { - mReaders.erase(handle); - handle_is_valid = false; + abortRequest(handle, true); } } - + if (worker && (complete || abort)) + { + mReaders.erase(iter); unlockWorkers(); - - if (delete_worker) worker->scheduleDelete(); - - // Return false if the handle is (still) valid. - return !handle_is_valid; + worker->scheduleDelete(); + } + else + { + unlockWorkers(); + } + return (complete || abort); } LLTextureCache::handle_t LLTextureCache::writeToCache(const LLUUID& id, U32 priority, @@ -1560,12 +1890,11 @@ LLTextureCache::handle_t LLTextureCache::writeToCache(const LLUUID& id, U32 prio } if (mDoPurge) { - // NOTE: This may cause an occasional hiccup, - // but it really needs to be done on the control thread - // (i.e. here) purgeTextures(false); - mDoPurge = FALSE; } + + purgeTextureFilesTimeSliced(); // VWR-3878 - NB - purge textures from cache in a non-hiccup-way + LLMutexLock lock(&mWorkersMutex); LLTextureCacheWorker* worker = new LLTextureCacheRemoteWorker(this, priority, id, data, datasize, 0, @@ -1579,7 +1908,9 @@ bool LLTextureCache::writeComplete(handle_t handle, bool abort) { lockWorkers(); handle_map_t::iterator iter = mWriters.find(handle); - llassert_always(iter != mWriters.end()); + llassert(iter != mWriters.end()); + if (iter != mWriters.end()) + { LLTextureCacheWorker* worker = iter->second; if (worker->complete() || abort) { @@ -1588,11 +1919,9 @@ bool LLTextureCache::writeComplete(handle_t handle, bool abort) worker->scheduleDelete(); return true; } - else - { + } unlockWorkers(); return false; - } } void LLTextureCache::prioritizeWrite(handle_t handle) @@ -1611,46 +1940,63 @@ void LLTextureCache::addCompleted(Responder* responder, bool success) ////////////////////////////////////////////////////////////////////////////// -// Called from MAIN thread (endWork()) - -bool LLTextureCache::removeHeaderCacheEntry(const LLUUID& id) +//called after mHeaderMutex is locked. +void LLTextureCache::removeCachedTexture(const LLUUID& id) { - if (!mReadOnly) + if(mTexturesSizeMap.find(id) != mTexturesSizeMap.end()) { - Entry entry; - S32 idx = openAndReadEntry(id, entry, false); - if (idx >= 0) + mTexturesSizeTotal -= mTexturesSizeMap[id]; + mTexturesSizeMap.erase(id); + } + mHeaderIDMap.erase(id); + std::string filename = getTextureFileName(id); + if (LLAPRFile::isExist(filename)) + { + LLAPRFile::remove(filename); + } +} + +//called after mHeaderMutex is locked. +void LLTextureCache::removeEntry(S32 idx, Entry& entry, std::string& filename) +{ + if(idx >= 0) //valid entry { entry.mImageSize = -1; entry.mBodySize = 0; - writeEntryAndClose(idx, entry); + mHeaderIDMap.erase(entry.mID); + mTexturesSizeMap.erase(entry.mID); + + mTexturesSizeTotal -= entry.mBodySize; mFreeList.insert(idx); - mHeaderIDMap.erase(id); - mTexturesSizeMap.erase(id); - return true; } - } - return false; -} -void LLTextureCache::removeFromCacheLocked(const LLUUID& id) -{ - //llwarns << "Removing texture from cache: " << id << llendl; - if (!mReadOnly) + if (LLAPRFile::isExist(filename)) { - removeHeaderCacheEntry(id); - LLAPRFile::remove(getTextureFileName(id)); + LLAPRFile::remove(filename); } } -void LLTextureCache::removeFromCache(const LLUUID& id) +bool LLTextureCache::removeFromCache(const LLUUID& id) { //llwarns << "Removing texture from cache: " << id << llendl; + bool ret = false; if (!mReadOnly) { - LLMutexLock lock(&mHeaderMutex); - LLTextureCache::removeFromCacheLocked(id); + lockHeaders(); + + Entry entry; + S32 idx = openAndReadEntry(id, entry, false); + std::string tex_filename = getTextureFileName(id); + removeEntry(idx, entry, tex_filename); + if (idx >= 0) + { + writeEntryToHeaderImmediately(idx, entry); + ret = true; + } + + unlockHeaders(); } + return ret; } ////////////////////////////////////////////////////////////////////////////// diff --git a/indra/newview/lltexturecache.h b/indra/newview/lltexturecache.h index c859b9aee..ac9285008 100644 --- a/indra/newview/lltexturecache.h +++ b/indra/newview/lltexturecache.h @@ -40,6 +40,7 @@ #include "llworkerthread.h" +class LLImageFormatted; class LLTextureCacheWorker; class LLTextureCache : public LLWorkerThread @@ -58,10 +59,11 @@ private: }; struct Entry { - Entry() {} + Entry() : mBodySize(0), mImageSize(0), mTime(0) { } Entry(const LLUUID& id, S32 imagesize, S32 bodysize, U32 time) : mID(id), mImageSize(imagesize), mBodySize(bodysize), mTime(time) {} void init(const LLUUID& id, U32 time) { mID = id, mImageSize = 0; mBodySize = 0; mTime = time; } + Entry& operator=(const Entry& entry) { mID = entry.mID, mImageSize = entry.mImageSize; mBodySize = entry.mBodySize; mTime = entry.mTime; return *this; } LLUUID mID; // 16 bytes S32 mImageSize; // total size of image if known S32 mBodySize; // size of body file in body cache @@ -103,7 +105,8 @@ public: /*virtual*/ S32 update(U32 max_time_ms); void purgeCache(ELLPath location); - S64 initCache(ELLPath location, S64 maxsize, BOOL read_only); + void setReadOnly(BOOL read_only) ; + S64 initCache(ELLPath location, S64 maxsize, BOOL texture_cache_mismatch); handle_t readFromCache(const std::string& local_filename, const LLUUID& id, U32 priority, S32 offset, S32 size, ReadResponder* responder); @@ -116,7 +119,7 @@ public: bool writeComplete(handle_t handle, bool abort = false); void prioritizeWrite(handle_t handle); - void removeFromCache(const LLUUID& id); + bool removeFromCache(const LLUUID& id); // For LLTextureCacheWorker::Responder LLTextureCacheWorker* getReader(handle_t handle); @@ -131,10 +134,11 @@ public: S64 getMaxUsage() { return sCacheMaxTexturesSize; } U32 getEntries() { return mHeaderEntriesInfo.mEntries; } U32 getMaxEntries() { return sCacheMaxEntries; }; + BOOL isInCache(const LLUUID& id) ; + BOOL isInLocal(const LLUUID& id) ; protected: // Accessed by LLTextureCacheWorker - bool updateTextureEntryList(const LLUUID& id, S32 size); std::string getLocalFileName(const LLUUID& id); std::string getTextureFileName(const LLUUID& id); void addCompleted(Responder* responder, bool success); @@ -145,20 +149,29 @@ protected: private: void setDirNames(ELLPath location); void readHeaderCache(); + void clearCorruptedCache(); void purgeAllTextures(bool purge_directories); - void purgeTextures(bool validate); + void purgeTextures(bool validate, bool force = false); + void purgeTextureFilesTimeSliced(BOOL force_all = FALSE); // VWR-3878 - NB LLAPRFile* openHeaderEntriesFile(bool readonly, S32 offset); void closeHeaderEntriesFile(); void readEntriesHeader(); void writeEntriesHeader(); S32 openAndReadEntry(const LLUUID& id, Entry& entry, bool create); - void writeEntryAndClose(S32 idx, Entry& entry); + bool updateEntry(S32& idx, Entry& entry, S32 new_image_size, S32 new_body_size); + void updateEntryTimeStamp(S32 idx, Entry& entry) ; U32 openAndReadEntries(std::vector& entries); void writeEntriesAndClose(const std::vector& entries); - S32 getHeaderCacheEntry(const LLUUID& id, S32& imagesize); - S32 setHeaderCacheEntry(const LLUUID& id, S32 imagesize); - bool removeHeaderCacheEntry(const LLUUID& id); - void removeFromCacheLocked(const LLUUID& id); + void readEntryFromHeaderImmediately(S32& idx, Entry& entry) ; + void writeEntryToHeaderImmediately(S32& idx, Entry& entry, bool write_header = false) ; + void removeEntry(S32 idx, Entry& entry, std::string& filename); + void removeCachedTexture(const LLUUID& id) ; + S32 getHeaderCacheEntry(const LLUUID& id, Entry& entry); + S32 setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize); + void writeUpdatedEntries() ; + void updatedHeaderEntriesFile() ; + void lockHeaders() { mHeaderMutex.lock(); } + void unlockHeaders() { mHeaderMutex.unlock(); } private: // Internal @@ -177,6 +190,9 @@ private: typedef std::vector, bool> > responder_list_t; responder_list_t mCompletedList; + typedef std::list filename_list_t; + filename_list_t mFilesToDelete; + LLTimer mTimeLastFileDelete; BOOL mReadOnly; // HEADERS (Include first mip) @@ -195,6 +211,9 @@ private: S64 mTexturesSizeTotal; LLAtomic32 mDoPurge; + typedef std::map idx_entry_map_t; + idx_entry_map_t mUpdatedEntryMap; + // Statics static F32 sHeaderCacheVersion; static U32 sCacheMaxEntries; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 03962c807..429500c9a 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -35,15 +35,16 @@ #include #include "llstl.h" +#include "message.h" #include "lltexturefetch.h" -#include "llappviewer.h" #include "llcurl.h" #include "lldir.h" #include "llhttpclient.h" #include "llhttpstatuscodes.h" #include "llimage.h" +#include "llimagej2c.h" #include "llimageworker.h" #include "llworkerthread.h" @@ -72,13 +73,11 @@ private: } virtual void completed(bool success) { - mFetcher->lockQueue(); LLTextureFetchWorker* worker = mFetcher->getWorker(mID); if (worker) { worker->callbackCacheRead(success, mFormattedImage, mImageSize, mImageLocal); } - mFetcher->unlockQueue(); } private: LLTextureFetch* mFetcher; @@ -94,13 +93,11 @@ private: } virtual void completed(bool success) { - mFetcher->lockQueue(); LLTextureFetchWorker* worker = mFetcher->getWorker(mID); if (worker) { worker->callbackCacheWrite(success); } - mFetcher->unlockQueue(); } private: LLTextureFetch* mFetcher; @@ -116,13 +113,11 @@ private: } virtual void completed(bool success, LLImageRaw* raw, LLImageRaw* aux) { - mFetcher->lockQueue(); LLTextureFetchWorker* worker = mFetcher->getWorker(mID); if (worker) { worker->callbackDecoded(success, raw, aux); } - mFetcher->unlockQueue(); } private: LLTextureFetch* mFetcher; @@ -155,9 +150,9 @@ public: ~LLTextureFetchWorker(); void release() { --mActiveCount; } - void callbackHttpGet(const LLChannelDescriptors& channels, + S32 callbackHttpGet(const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool unsatisfiable, bool success); + bool partial, bool success); void callbackCacheRead(bool success, LLImageFormatted* image, S32 imagesize, BOOL islocal); void callbackCacheWrite(bool success); @@ -165,6 +160,8 @@ public: void setGetStatus(U32 status, const std::string& reason) { + LLMutexLock lock(&mWorkMutex); + mGetStatus = status; mGetReason = reason; } @@ -173,10 +170,8 @@ public: bool getCanUseHTTP()const {return mCanUseHTTP ;} protected: - LLTextureFetchWorker(LLTextureFetch* fetcher, const LLUUID& id, const LLHost& host, - F32 priority, S32 discard, S32 size); LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, - F32 priority, S32 discard, S32 size, bool can_use_http); + F32 priority, S32 discard, S32 size); private: /*virtual*/ void startWork(S32 param); // called from addWork() (MAIN THREAD) @@ -295,8 +290,8 @@ class HTTPGetResponder : public LLCurl::Responder { LOG_CLASS(HTTPGetResponder); public: - HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset) - : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset) + HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) + : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) { } ~HTTPGetResponder() @@ -317,42 +312,38 @@ public: mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); } - LL_DEBUGS("TextureFetch") << "HTTP COMPLETE: " << mID << " with status: " << status << LL_ENDL; - mFetcher->lockQueue(); + LL_DEBUGS("Texture") << "HTTP COMPLETE: " << mID << LL_ENDL; LLTextureFetchWorker* worker = mFetcher->getWorker(mID); if (worker) { bool success = false; bool partial = false; - bool unsatisfiable = false; - if (200 <= status && status < 300) + if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) { success = true; - if (HTTP_PARTIAL_CONTENT == status) // partial information (i.e. last block) + if (HTTP_PARTIAL_CONTENT == status) // partial information { partial = true; } } - else if (status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) - { - LL_DEBUGS("TextureFetch") << "Request was an unsatisfiable range: mRequestedSize=" << mRequestedSize << " mOffset=" << mOffset << " for: " << mID << LL_ENDL; - unsatisfiable = true; - } - if (!success) { worker->setGetStatus(status, reason); // llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; } - mFetcher->removeFromHTTPQueue(mID); - worker->callbackHttpGet(channels, buffer, partial, unsatisfiable, success); + S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); + mFetcher->removeFromHTTPQueue(mID, data_size); } else { mFetcher->removeFromHTTPQueue(mID); llwarns << "Worker not found: " << mID << llendl; } - mFetcher->unlockQueue(); + } + + virtual bool followRedir() + { + return mFollowRedir; } private: @@ -361,6 +352,7 @@ private: U64 mStartTime; S32 mRequestedSize; U32 mOffset; + bool mFollowRedir; }; ////////////////////////////////////////////////////////////////////////////// @@ -473,8 +465,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, const LLHost& host, // Simulator host F32 priority, // Priority S32 discard, // Desired discard - S32 size, // Desired size - bool can_use_http) // Try HTTP first + S32 size) // Desired size : LLWorkerClass(fetcher, "TextureFetch"), mState(INIT), mWriteToCacheState(NOT_WRITE), @@ -495,7 +486,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mBuffer(NULL), mBufferSize(0), mRequestedSize(0), - mDesiredSize(FIRST_PACKET_SIZE), + mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE), mFileSize(0), mCachedSize(0), mLoaded(FALSE), @@ -506,7 +497,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mNeedsAux(FALSE), mHaveAllData(FALSE), mInLocalCache(FALSE), - mCanUseHTTP(can_use_http), + mCanUseHTTP(true), mHTTPFailCount(0), mRetryAttempt(0), mActiveCount(0), @@ -522,14 +513,12 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, calcWorkPriority(); mType = host.isOk() ? LLImageBase::TYPE_AVATAR_BAKE : LLImageBase::TYPE_NORMAL; // llinfos << "Create: " << mID << " mHost:" << host << " Discard=" << discard << llendl; - lockWorkMutex(); if (!mFetcher->mDebugPause) { U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; addWork(0, work_priority ); } setDesiredDiscard(discard, size); - unlockWorkMutex(); } LLTextureFetchWorker::~LLTextureFetchWorker() @@ -540,17 +529,18 @@ LLTextureFetchWorker::~LLTextureFetchWorker() // << " Desired=" << mDesiredDiscard << llendl; llassert_always(!haveWork()); lockWorkMutex(); - if (mCacheReadHandle != LLTextureCache::nullHandle()) + if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) { mFetcher->mTextureCache->readComplete(mCacheReadHandle, true); } - if (mCacheWriteHandle != LLTextureCache::nullHandle()) + if (mCacheWriteHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) { mFetcher->mTextureCache->writeComplete(mCacheWriteHandle, true); } mFormattedImage = NULL; clearPackets(); unlockWorkMutex(); + mFetcher->removeFromHTTPQueue(mID); } void LLTextureFetchWorker::clearPackets() @@ -597,8 +587,8 @@ void LLTextureFetchWorker::setupPacketData() U32 LLTextureFetchWorker::calcWorkPriority() { // llassert_always(mImagePriority >= 0 && mImagePriority <= LLViewerImage::maxDecodePriority()); - F32 priority_scale = (F32)LLWorkerThread::PRIORITY_LOWBITS / LLViewerImage::maxDecodePriority(); - mWorkPriority = (U32)(mImagePriority * priority_scale); + static const F32 PRIORITY_SCALE = (F32)LLWorkerThread::PRIORITY_LOWBITS / LLViewerImage::maxDecodePriority(); + mWorkPriority = llmin((U32)LLWorkerThread::PRIORITY_LOWBITS, (U32)(mImagePriority * PRIORITY_SCALE)); return mWorkPriority; } @@ -629,6 +619,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) mDesiredSize = size; prioritize = true; } + mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); if ((prioritize && mState == INIT) || mState == DONE) { mState = INIT; @@ -678,24 +669,24 @@ bool LLTextureFetchWorker::doWork(S32 param) if ((mFetcher->isQuitting() || getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) { if (mState < DECODE_IMAGE) - { + { return true; // abort } } - if (mImagePriority < 1.0f) + if(mImagePriority < F_ALMOST_ZERO) { if (mState == INIT || mState == LOAD_FROM_NETWORK || mState == LOAD_FROM_SIMULATOR) - { - return true; // abort - } - } - if (mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) + { + return true; // abort + } + } + if(mState > CACHE_POST && !mCanUseNET && !mCanUseHTTP) { //nowhere to get data, abort. return true ; } - if (mFetcher->mDebugPause|| gDisconnected) + if (mFetcher->mDebugPause) { return false; // debug: don't do any work } @@ -719,6 +710,7 @@ bool LLTextureFetchWorker::doWork(S32 param) return true; } + mRawImage = NULL ; mRequestedDiscard = -1; mLoadedDiscard = -1; mDecodedDiscard = -1; @@ -737,6 +729,9 @@ bool LLTextureFetchWorker::doWork(S32 param) mCacheReadHandle = LLTextureCache::nullHandle(); mCacheWriteHandle = LLTextureCache::nullHandle(); mState = LOAD_FROM_TEXTURE_CACHE; + mDesiredSize = llmax(mDesiredSize, TEXTURE_CACHE_ENTRY_SIZE); // min desired size is TEXTURE_CACHE_ENTRY_SIZE + LL_DEBUGS("Texture") << mID << ": Priority: " << llformat("%8.0f",mImagePriority) + << " Desired Discard: " << mDesiredDiscard << " Desired Size: " << mDesiredSize << LL_ENDL; // fall through } @@ -754,18 +749,20 @@ bool LLTextureFetchWorker::doWork(S32 param) } mFileSize = 0; mLoaded = FALSE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); if (mUrl.compare(0, 7, "file://") == 0) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(filename, mID, cache_priority, offset, size, responder); } else if (mUrl.empty()) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); } @@ -808,13 +805,13 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == CACHE_POST) { - mDesiredSize = llmax(mDesiredSize, FIRST_PACKET_SIZE); mCachedSize = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; // Successfully loaded if ((mCachedSize >= mDesiredSize) || mHaveAllData) { // we have enough data, decode it llassert_always(mFormattedImage->getDataSize() > 0); + mLoadedDiscard = mDesiredDiscard; mState = DECODE_IMAGE; mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() @@ -832,11 +829,8 @@ bool LLTextureFetchWorker::doWork(S32 param) // need more data else { - mState = LOAD_FROM_NETWORK; // CACHE_POST --> LOAD_FROM_NETWORK, or SEND_HTTP_REQ see below. - // This is true because mSentRequest is set to UNSENT in INIT and if we get here we went through - // the states INIT --> LOAD_FROM_TEXTURE_CACHE --> CACHE_POST. Therefore either - // mFetcher->addToNetworkQueue(this) is called below, or mState is set to SEND_HTTP_REQ. - llassert(mSentRequest == UNSENT); + LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; + mState = LOAD_FROM_NETWORK; } // fall through } @@ -844,13 +838,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_NETWORK) { - static const LLCachedControl image_pipeline_use_http("ImagePipelineUseHTTP", false); - bool get_url = image_pipeline_use_http; + static const LLCachedControl use_http("ImagePipelineUseHTTP", false); - if (!mUrl.empty()) get_url = false; -// if (mHost != LLHost::invalid) get_url = false; - - if ( get_url && mCanUseHTTP && mUrl.empty())//get http url. +// if (mHost != LLHost::invalid) use_http = false; + if (use_http && mCanUseHTTP && mUrl.empty()) // get http url. { LLViewerRegion* region = NULL; if (mHost == LLHost::invalid) @@ -891,7 +882,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } // don't return, fall through to next state } - else if (mSentRequest == UNSENT) + else if (mCanUseNET && mSentRequest == UNSENT) { // Add this to the network queue and sit here. // LLTextureFetch::update() will send off a request which will change our state @@ -917,12 +908,12 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (processSimulatorPackets()) { + LL_DEBUGS("Texture") << mID << ": Loaded from Sim. Bytes: " << mFormattedImage->getDataSize() << LL_ENDL; mFetcher->removeFromNetworkQueue(this, false); if (mFormattedImage.isNull() || !mFormattedImage->getDataSize()) { // processSimulatorPackets() failed // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; - // FIXME: Don't we need a mState change? return true; // failed } setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); @@ -931,6 +922,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { + mFetcher->addToNetworkQueue(this); // failsafe setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); } return false; @@ -940,19 +932,16 @@ bool LLTextureFetchWorker::doWork(S32 param) { if(mCanUseHTTP) { - const S32 HTTP_QUEUE_MAX_SIZE = 16; // *TODO: Integrate this with llviewerthrottle // Note: LLViewerThrottle uses dynamic throttling which makes sense for UDP, // but probably not for Textures. // Set the throttle to the entire bandwidth, assuming UDP packets will get priority // when they are needed - F32 max_bandwidth = mFetcher->mMaxBandwidth; - if ((mFetcher->getNumHTTPRequests() >= HTTP_QUEUE_MAX_SIZE) || - (mFetcher->getTextureBandwidth() > max_bandwidth)) + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 32; + if ((mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) || + (mFetcher->getTextureBandwidth() > mFetcher->mMaxBandwidth)) { - // Make normal priority and return (i.e. wait until there is room in the queue) - setPriority(LLWorkerThread::PRIORITY_NORMAL | mWorkPriority); - return false; + return false ; //wait. } S32 cur_size = 0; @@ -967,7 +956,6 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } } - mRequestedSize = mDesiredSize; mRequestedDiscard = mDesiredDiscard; mRequestedSize -= cur_size; @@ -981,11 +969,10 @@ bool LLTextureFetchWorker::doWork(S32 param) mLoaded = FALSE; mGetStatus = 0; mGetReason.clear(); - LL_DEBUGS("TextureFetch") << "HTTP GET: " << mID << " Offset: " << offset + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset << " Bytes: " << mRequestedSize - << " Range: " << offset << "-" << offset+mRequestedSize-1 - << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << max_bandwidth - << LL_ENDL; + << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() + << "/" << mFetcher->mMaxBandwidth << LL_ENDL; setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; @@ -994,7 +981,7 @@ bool LLTextureFetchWorker::doWork(S32 param) std::vector headers; headers.push_back("Accept: image/x-j2c"); res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset)); + new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); } if (!res) { @@ -1005,6 +992,10 @@ bool LLTextureFetchWorker::doWork(S32 param) } // fall through } + else //can not use http fetch. + { + return true ; //abort + } } if (mState == WAIT_HTTP_REQ) @@ -1028,31 +1019,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llwarns << "Texture server busy (503): " << mUrl << LL_ENDL; SGHostBlackList::add(mUrl, 60.0, mGetStatus); } - - //roll back to try UDP - if(mCanUseNET) - { - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; - } - else - { - // UDP is not an option, we are dead - resetFormattedData(); - return true; // failed - } } - //else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) - //{ - // // *TODO: Should probably introduce a timer here to delay future HTTP requsts - // // for a short time (~1s) to ease server load? Ideally the server would queue - // // requests instead of returning 503... we already limit the number pending. - // ++mHTTPFailCount; - // max_attempts = mHTTPFailCount+1; // Keep retrying - // LL_INFOS_ONCE("Texture") << "Texture server busy (503): " << mUrl << LL_ENDL; - //} else { const S32 HTTP_MAX_RETRY_COUNT = 1; @@ -1065,32 +1032,29 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mHTTPFailCount >= max_attempts) { - // Make max_attempts attempt at decoding what data we have, then bail forever on this image - if (cur_size > 0 && (mHTTPFailCount < (max_attempts+1)) ) - { - // Use available data - mLoadedDiscard = mFormattedImage->getDiscardLevel(); - mState = DECODE_IMAGE; - return false; - } - else - { - //roll back to try UDP - if(mCanUseNET) + if (mCanUseNET) { + llinfos << "Falling back to SIM fetch for texture " << mID << llendl; + resetFormattedData(); mState = INIT ; mCanUseHTTP = false ; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false ; } + if (cur_size > 0) + { + // Use available data + mLoadedDiscard = mFormattedImage->getDiscardLevel(); + mState = DECODE_IMAGE; + return false; + } else { - // UDP is not an option, we are dead resetFormattedData(); + mState = DONE; return true; // failed } } - } else { mState = SEND_HTTP_REQ; @@ -1098,6 +1062,17 @@ bool LLTextureFetchWorker::doWork(S32 param) } } + llassert_always(mBufferSize == cur_size + mRequestedSize); + if(!mBufferSize)//no data received. + { + delete[] mBuffer; + mBuffer = NULL; + + //abort. + mState = DONE; + return true; + } + if (mFormattedImage.isNull()) { // For now, create formatted image based on extension @@ -1109,7 +1084,6 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - llassert_always(mBufferSize == cur_size + mRequestedSize); if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. { mFileSize = mBufferSize; @@ -1118,7 +1092,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded. } - + U8* buffer = new U8[mBufferSize]; if (cur_size > 0) { @@ -1149,7 +1123,14 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE) { - llassert_always(mFormattedImage->getDataSize() > 0); + if (mDesiredDiscard < 0 || mFormattedImage->getDataSize() <= 0 || mLoadedDiscard < 0) + { + // We aborted, or decode entered with invalid mFormattedImage, + // or decode entered with invalid mLoadedDiscard: don't decode. + mState = DONE; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return true; + } setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it mRawImage = NULL; mAuxImage = NULL; @@ -1158,6 +1139,8 @@ bool LLTextureFetchWorker::doWork(S32 param) U32 image_priority = LLWorkerThread::PRIORITY_NORMAL | mWorkPriority; mDecoded = FALSE; mState = DECODE_IMAGE_UPDATE; + LL_DEBUGS("Texture") << mID << ": Decoding. Bytes: " << mFormattedImage->getDataSize() << " Discard: " << discard + << " All Data: " << mHaveAllData << LL_ENDL; mDecodeHandle = mFetcher->mImageDecodeThread->decodeImage(mFormattedImage, image_priority, discard, mNeedsAux, new DecodeResponder(mFetcher, mID, this)); // fall though @@ -1169,6 +1152,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { if (mDecodedDiscard < 0) { + LL_DEBUGS("Texture") << mID << ": Failed to Decode." << LL_ENDL; if (mCachedSize > 0 && !mInLocalCache && mRetryAttempt == 0) { // Cache file should be deleted, try again @@ -1188,6 +1172,9 @@ bool LLTextureFetchWorker::doWork(S32 param) } else { + llassert_always(mRawImage.notNull()); + LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard + << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = WRITE_TO_CACHE; } @@ -1209,6 +1196,17 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } S32 datasize = mFormattedImage->getDataSize(); + if (mFileSize < datasize) // This could happen when http fetching and sim fetching mixed. + { + if (mHaveAllData) + { + mFileSize = datasize; + } + else + { + mFileSize = datasize + 1; // flag not fully loaded. + } + } llassert_always(datasize); setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; @@ -1255,7 +1253,7 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); if(mDecodedDiscard<=0) - { + { return true; } else @@ -1325,10 +1323,9 @@ bool LLTextureFetchWorker::deleteOK() } } - if ((haveWork() && + if (haveWork() && // not ok to delete from these states - ((mState >= SEND_HTTP_REQ && mState <= WAIT_HTTP_REQ) || - (mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE)))) + mState >= WRITE_TO_CACHE && mState <= WAIT_ON_WRITE) { delete_ok = false; } @@ -1407,112 +1404,63 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// -void LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, +S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, const LLIOPipe::buffer_ptr_t& buffer, - bool partial, - bool unsatisfiable, - bool success) + bool partial, bool success) { + S32 data_size = 0 ; + LLMutexLock lock(&mWorkMutex); if (mState != WAIT_HTTP_REQ) { llwarns << "callbackHttpGet for unrequested fetch worker: " << mID << " req=" << mSentRequest << " state= " << mState << llendl; - return; + return data_size; } if (mLoaded) { llwarns << "Duplicate callback for " << mID.asString() << llendl; - return; // ignore duplicate callback + return data_size; // ignore duplicate callback } - - S32 data_size = 0; if (success) { // get length of stream: data_size = buffer->countAfter(channels.in(), NULL); - - gImageList.sTextureBits += data_size * 8; // Approximate - does not include header bits - LL_DEBUGS("TextureFetch") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << " mRequestedSize: " << mRequestedSize << LL_ENDL; - + LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; if (data_size > 0) { - bool clean_data = false; - bool done = false; - if (!partial) - { - // we got the whole image in one go - done = true; - clean_data = true; - } - else if (data_size < mRequestedSize) - { - // we have the whole image - done = true; - } - else if (data_size == mRequestedSize) - { - if (mRequestedDiscard <= 0) - { - done = true; - } - else - { - // this is the normal case where we get the data we requested, - // but still need to request more data. - } - } - else if (data_size > mRequestedSize) - { - // *TODO: This shouldn't be happening any more - llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; - done = true; - clean_data = true; - llassert_always(mDecodeHandle == 0); - } - - if (clean_data) - { - resetFormattedData(); // discard any previous data we had - llassert(mBufferSize == 0); - } - if (done) - { - mHaveAllData = TRUE; - mRequestedDiscard = 0; - } - // *TODO: set the formatted image data here directly to avoid the copy mBuffer = new U8[data_size]; buffer->readAfter(channels.in(), NULL, mBuffer, data_size); + if (data_size < mRequestedSize && mRequestedDiscard == 0) + { + mHaveAllData = TRUE; + } + else if (data_size > mRequestedSize) + { + LL_DEBUGS("Texture") << "Extra data received: got " << data_size << " bytes (requested: " << mRequestedSize << " ) for texture: " << mID.asString() << LL_ENDL; + data_size = mRequestedSize; + } mBufferSize += data_size; - mRequestedSize = data_size; } + else + { + // We requested data but received none (and no error), + // so presumably we have all of it + mHaveAllData = TRUE; + } + mRequestedSize = data_size; } else { mRequestedSize = -1; // error } - - if ((success && (data_size == 0)) || unsatisfiable) - { - if (mFormattedImage.notNull() && mFormattedImage->getDataSize() > 0) - { - // we already have some data, so we'll assume we have it all - mRequestedSize = 0; - mRequestedDiscard = 0; - mHaveAllData = TRUE; - } - else - { - mRequestedSize = -1; // treat this fetch as if it failed. - } - } - mLoaded = TRUE; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + + return data_size ; } ////////////////////////////////////////////////////////////////////////////// @@ -1574,20 +1522,22 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag mDecodeHandle = 0; if (success) { + llassert_always(raw); mRawImage = raw; mAuxImage = aux; mDecodedDiscard = mFormattedImage->getDiscardLevel(); -// llinfos << mID << " : DECODE FINISHED. DISCARD: " << mDecodedDiscard << llendl; + LL_DEBUGS("Texture") << mID << ": Decode Finished. Discard: " << mDecodedDiscard + << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; } else { if (mFormattedImage.notNull()) { - LL_WARNS("http-texture") << "DECODE FAILED: id = " << mID << ", Discard = " << (S32)mFormattedImage->getDiscardLevel() << LL_ENDL; + LL_WARNS("Texture") << "DECODE FAILED: id = " << mID << ", Discard = " << (S32)mFormattedImage->getDiscardLevel() << LL_ENDL; } else { - LL_WARNS("http-texture") << "DECODE FAILED: id = " << mID << ", mFormattedImage is Null!" << LL_ENDL; + LL_WARNS("Texture") << "DECODE FAILED: id = " << mID << ", mFormattedImage is Null!" << LL_ENDL; } removeFromCache(); mDecodedDiscard = -1; // Redundant, here for clarity and paranoia @@ -1595,10 +1545,6 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - // Set the decode flag at the end of the callback or we trigger race conditions between the fetch thread and the - // decode threads that's calling this callback. The fetch thread might set mFormattedImage to NULL before we - // have time here to call getDiscardLevel() which causes crashes - mDecoded = TRUE; } ////////////////////////////////////////////////////////////////////////////// @@ -1640,6 +1586,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureCache(cache), mImageDecodeThread(imagedecodethread), mTextureBandwidth(0), + mHTTPTextureBits(0), mCurlGetRequest(NULL) { mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); @@ -1648,6 +1595,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image LLTextureFetch::~LLTextureFetch() { + clearDeleteList() ; // ~LLQueuedThread() called here } @@ -1659,12 +1607,9 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con return false; } - LLTextureFetchWorker* worker = NULL; - LLMutexLock lock(&mQueueMutex); - map_t::iterator iter = mRequestMap.find(id); - if (iter != mRequestMap.end()) + LLTextureFetchWorker* worker = getWorker(id); + if (worker) { - worker = iter->second; if (worker->mHost != host) { llwarns << "LLTextureFetch::createRequest " << id << " called with multiple hosts: " @@ -1682,6 +1627,7 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con // Only do partial requests for J2C at the moment //llinfos << "Merov : LLTextureFetch::createRequest(), blocking fetch on " << url << llendl; desired_size = MAX_IMAGE_DATA_SIZE; + desired_discard = 0; } else if (desired_discard == 0) { @@ -1700,7 +1646,7 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con } else { - desired_size = FIRST_PACKET_SIZE; + desired_size = TEXTURE_CACHE_ENTRY_SIZE; desired_discard = MAX_DISCARD_LEVEL; } @@ -1712,48 +1658,53 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con return false; // need to wait for previous aborted request to complete } worker->lockWorkMutex(); + worker->mActiveCount++; + worker->mNeedsAux = needs_aux; worker->setImagePriority(priority); worker->setDesiredDiscard(desired_discard, desired_size); - worker->setCanUseHTTP(can_use_http); - worker->unlockWorkMutex(); + worker->setCanUseHTTP(can_use_http) ; if (!worker->haveWork()) { - worker->lockWorkMutex(); - if (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK || worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) - { - removeFromNetworkQueue(worker, true); - } worker->mState = LLTextureFetchWorker::INIT; worker->unlockWorkMutex(); worker->addWork(0, LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); } + else + { + worker->unlockWorkMutex(); + } } else { - worker = new LLTextureFetchWorker(this, url, id, host, priority, desired_discard, desired_size, can_use_http); + worker = new LLTextureFetchWorker(this, url, id, host, priority, desired_discard, desired_size); + lockQueue() ; mRequestMap[id] = worker; - } + unlockQueue() ; + + worker->lockWorkMutex(); worker->mActiveCount++; worker->mNeedsAux = needs_aux; + worker->setCanUseHTTP(can_use_http) ; + worker->unlockWorkMutex(); + } + // llinfos << "REQUESTED: " << id << " Discard: " << desired_discard << llendl; return true; } -void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) -{ - LLMutexLock lock(&mQueueMutex); - LLTextureFetchWorker* worker = getWorker(id); - if (worker) - { - removeRequest(worker, cancel); - } -} - // protected void LLTextureFetch::addToNetworkQueue(LLTextureFetchWorker* worker) { + lockQueue(); + bool in_request_map = (mRequestMap.find(worker->mID) != mRequestMap.end()); + unlockQueue(); LLMutexLock lock(&mNetworkQueueMutex); + if (in_request_map) + { + // only add to the queue if in the request map + // i.e. a delete has not been requested mNetworkQueue.insert(worker->mID); + } for (cancel_queue_t::iterator iter1 = mCancelQueue.begin(); iter1 != mCancelQueue.end(); ++iter1) { iter1->second.erase(worker->mID); @@ -1777,16 +1728,41 @@ void LLTextureFetch::addToHTTPQueue(const LLUUID& id) mHTTPTextureQueue.insert(id); } -void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id) +void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) { LLMutexLock lock(&mNetworkQueueMutex); mHTTPTextureQueue.erase(id); + mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits +} + +void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) +{ + lockQueue() ; + LLTextureFetchWorker* worker = getWorkerAfterLock(id); + if (worker) + { + size_t erased_1 = mRequestMap.erase(worker->mID); + unlockQueue() ; + + llassert_always(erased_1 > 0) ; + + removeFromNetworkQueue(worker, cancel); + llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; + + worker->scheduleDelete(); + } + else + { + unlockQueue() ; + } } -// call lockQueue() first! void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) { + lockQueue() ; size_t erased_1 = mRequestMap.erase(worker->mID); + unlockQueue() ; + llassert_always(erased_1 > 0) ; removeFromNetworkQueue(worker, cancel); llassert_always(!(worker->getFlags(LLWorkerClass::WCF_DELETE_REQUESTED))) ; @@ -1794,8 +1770,26 @@ void LLTextureFetch::removeRequest(LLTextureFetchWorker* worker, bool cancel) worker->scheduleDelete(); } +S32 LLTextureFetch::getNumRequests() +{ + lockQueue() ; + S32 size = (S32)mRequestMap.size(); + unlockQueue() ; + + return size ; +} + +S32 LLTextureFetch::getNumHTTPRequests() +{ + mNetworkQueueMutex.lock() ; + S32 size = (S32)mHTTPTextureQueue.size(); + mNetworkQueueMutex.unlock() ; + + return size ; +} + // call lockQueue() first! -LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) +LLTextureFetchWorker* LLTextureFetch::getWorkerAfterLock(const LLUUID& id) { LLTextureFetchWorker* res = NULL; map_t::iterator iter = mRequestMap.find(id); @@ -1806,12 +1800,18 @@ LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) return res; } +LLTextureFetchWorker* LLTextureFetch::getWorker(const LLUUID& id) +{ + LLMutexLock lock(&mQueueMutex) ; + + return getWorkerAfterLock(id) ; +} + bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, LLPointer& raw, LLPointer& aux) { bool res = false; - LLMutexLock lock(&mQueueMutex); LLTextureFetchWorker* worker = getWorker(id); if (worker) { @@ -1830,10 +1830,13 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, } else if (worker->checkWork()) { + worker->lockWorkMutex(); discard_level = worker->mDecodedDiscard; - raw = worker->mRawImage; worker->mRawImage = NULL; - aux = worker->mAuxImage; worker->mAuxImage = NULL; + raw = worker->mRawImage; + aux = worker->mAuxImage; res = true; + LL_DEBUGS("Texture") << id << ": Request Finished. State: " << worker->mState << " Discard: " << discard_level << LL_ENDL; + worker->unlockWorkMutex(); } else { @@ -1844,8 +1847,8 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, { // Not finished, but data is ready discard_level = worker->mDecodedDiscard; - if (worker->mRawImage) raw = worker->mRawImage; - if (worker->mAuxImage) aux = worker->mAuxImage; + raw = worker->mRawImage; + aux = worker->mAuxImage; } worker->unlockWorkMutex(); } @@ -1860,7 +1863,6 @@ bool LLTextureFetch::getRequestFinished(const LLUUID& id, S32& discard_level, bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) { bool res = false; - LLMutexLock lock(&mQueueMutex); LLTextureFetchWorker* worker = getWorker(id); if (worker) { @@ -1878,20 +1880,57 @@ bool LLTextureFetch::updateRequestPriority(const LLUUID& id, F32 priority) //virtual S32 LLTextureFetch::update(U32 max_time_ms) { - S32 res; + { + mNetworkQueueMutex.lock() ; + static const LLCachedControl max_bandwidth("ThrottleBandwidthKBPS", 2000); + mMaxBandwidth = max_bandwidth; - mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); + gImageList.sTextureBits += mHTTPTextureBits; + mHTTPTextureBits = 0 ; + + mNetworkQueueMutex.unlock() ; + } - res = LLWorkerThread::update(max_time_ms); + S32 res = LLWorkerThread::update(max_time_ms); if (!mDebugPause) { sendRequestListToSimulators(); } + if (!mThreaded) + { + // Update Curl on same thread as mCurlGetRequest was constructed + S32 processed = mCurlGetRequest->process(); + if (processed > 0) + { + LL_DEBUGS("Texture") << "processed: " << processed << " messages." << LL_ENDL; + } + } + return res; } +//called in the MAIN thread after the TextureCacheThread shuts down. +void LLTextureFetch::shutDownTextureCacheThread() +{ + if(mTextureCache) + { + llassert_always(mTextureCache->isQuitting() || mTextureCache->isStopped()) ; + mTextureCache = NULL ; + } +} + +//called in the MAIN thread after the ImageDecodeThread shuts down. +void LLTextureFetch::shutDownImageDecodeThread() +{ + if(mImageDecodeThread) + { + llassert_always(mImageDecodeThread->isQuitting() || mImageDecodeThread->isStopped()) ; + mImageDecodeThread = NULL ; + } +} + // WORKER THREAD void LLTextureFetch::startThread() { @@ -1925,7 +1964,7 @@ void LLTextureFetch::threadedUpdate() S32 processed = mCurlGetRequest->process(); if (processed > 0) { - lldebugs << "processed: " << processed << " messages." << llendl; + LL_DEBUGS("Texture") << "processed: " << processed << " messages." << LL_ENDL; } #if 0 @@ -1966,8 +2005,6 @@ void LLTextureFetch::sendRequestListToSimulators() } timer.reset(); - LLMutexLock lock(&mQueueMutex); - // Send requests typedef std::set request_list_t; typedef std::map< LLHost, request_list_t > work_request_map_t; @@ -1985,12 +2022,11 @@ void LLTextureFetch::sendRequestListToSimulators() mNetworkQueue.erase(curiter); continue; } - llassert(req->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK || LLTextureFetchWorker::LOAD_FROM_SIMULATOR); if ((req->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK) && (req->mState != LLTextureFetchWorker::LOAD_FROM_SIMULATOR)) { - // We really should never ever get here anymore. - llwarns << "SNOW-119 failure: Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; + // We already received our URL, remove from the queue + llwarns << "Worker: " << req->mID << " in mNetworkQueue but in wrong state: " << req->mState << llendl; mNetworkQueue.erase(curiter); continue; } @@ -2179,23 +2215,17 @@ bool LLTextureFetchWorker::insertPacket(S32 index, U8* data, S32 size) bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data) { - LLMutexLock lock(&mQueueMutex); LLTextureFetchWorker* worker = getWorker(id); + bool res = true; ++mPacketCount; if (!worker) { // llwarns << "Received header for non active worker: " << id << llendl; - ++mBadPacketCount; - LLMutexLock lock2(&mNetworkQueueMutex); - mCancelQueue[host].insert(id); - return false; + res = false; } - - bool res = true; - worker->lockWorkMutex(); - if (worker->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK || + else if (worker->mState != LLTextureFetchWorker::LOAD_FROM_NETWORK || worker->mSentRequest != LLTextureFetchWorker::SENT_SIM) { // llwarns << "receiveImageHeader for worker: " << id @@ -2217,11 +2247,13 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 if (!res) { ++mBadPacketCount; - LLMutexLock lock2(&mNetworkQueueMutex); + mNetworkQueueMutex.lock() ; mCancelQueue[host].insert(id); + mNetworkQueueMutex.unlock() ; + return false; } - else - { + worker->lockWorkMutex(); + // Copy header data into image object worker->mImageCodec = codec; worker->mTotalPackets = packets; @@ -2231,14 +2263,12 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 res = worker->insertPacket(0, data, data_size); worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; - } worker->unlockWorkMutex(); return res; } bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data) { - LLMutexLock lock(&mQueueMutex); LLTextureFetchWorker* worker = getWorker(id); bool res = true; @@ -2262,8 +2292,9 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 if (!res) { ++mBadPacketCount; - LLMutexLock lock2(&mNetworkQueueMutex); + mNetworkQueueMutex.lock() ; mCancelQueue[host].insert(id); + mNetworkQueueMutex.unlock() ; return false; } @@ -2281,6 +2312,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 { // llwarns << "receiveImagePacket " << packet_num << "/" << worker->mLastPacket << " for worker: " << id // << " in state: " << LLTextureFetchWorker::sStateDescs[worker->mState] << llendl; + removeFromNetworkQueue(worker, true); // failsafe } if(packet_num >= (worker->mTotalPackets - 1)) @@ -2298,9 +2330,23 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 } ////////////////////////////////////////////////////////////////////////////// +BOOL LLTextureFetch::isFromLocalCache(const LLUUID& id) +{ + BOOL from_cache = FALSE; + + LLTextureFetchWorker* worker = getWorker(id); + if (worker) + { + worker->lockWorkMutex(); + from_cache = worker->mInLocalCache; + worker->unlockWorkMutex(); + } + + return from_cache; +} S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& requested_priority_p, - U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p) + U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http) { S32 state = LLTextureFetchWorker::INVALID; F32 data_progress = 0.0f; @@ -2309,7 +2355,6 @@ S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& r F32 request_dtime = 999999.f; U32 fetch_priority = 0; - LLMutexLock lock(&mQueueMutex); LLTextureFetchWorker* worker = getWorker(id); if (worker && worker->haveWork()) { @@ -2339,6 +2384,7 @@ S32 LLTextureFetch::getFetchState(const LLUUID& id, F32& data_progress_p, F32& r requested_priority = worker->mImagePriority; } fetch_priority = worker->getPriority(); + can_use_http = worker->getCanUseHTTP() ; worker->unlockWorkMutex(); } data_progress_p = data_progress; @@ -2364,3 +2410,4 @@ void LLTextureFetch::dump() << llendl; } } + diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 18fa89df7..e22283f05 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -58,9 +58,11 @@ public: ~LLTextureFetch(); /*virtual*/ S32 update(U32 max_time_ms); + void shutDownTextureCacheThread() ; //called in the main thread after the TextureCacheThread shuts down. + void shutDownImageDecodeThread() ; //called in the main thread after the ImageDecodeThread shuts down. bool createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority, - S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool use_http); + S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http); void deleteRequest(const LLUUID& id, bool cancel); bool getRequestFinished(const LLUUID& id, S32& discard_level, LLPointer& raw, LLPointer& aux); @@ -73,24 +75,26 @@ public: F32 getTextureBandwidth() { return mTextureBandwidth; } // Debug + BOOL isFromLocalCache(const LLUUID& id); S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p, - U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p); + U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http); void dump(); - S32 getNumRequests() const { LLMutexLock lock(&mQueueMutex); return mRequestMap.size(); } - S32 getNumHTTPRequests() const { LLMutexLock lock(&mNetworkQueueMutex); return mHTTPTextureQueue.size(); } + S32 getNumRequests(); + S32 getNumHTTPRequests(); // Public for access by callbacks void lockQueue() { mQueueMutex.lock(); } void unlockQueue() { mQueueMutex.unlock(); } LLTextureFetchWorker* getWorker(const LLUUID& id); + LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id); LLTextureInfo* getTextureInfo() { return &mTextureInfo; } - + protected: void addToNetworkQueue(LLTextureFetchWorker* worker); void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel); void addToHTTPQueue(const LLUUID& id); - void removeFromHTTPQueue(const LLUUID& id); + void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0); void removeRequest(LLTextureFetchWorker* worker, bool cancel); // Called from worker thread (during doWork) void processCurlRequests(); @@ -109,8 +113,8 @@ public: S32 mBadPacketCount; private: - mutable LLMutex mQueueMutex; - mutable LLMutex mNetworkQueueMutex; + LLMutex mQueueMutex; //to protect mRequestMap only + LLMutex mNetworkQueueMutex; //to protect mNetworkQueue, mHTTPTextureQueue and mCancelQueue. LLTextureCache* mTextureCache; LLImageDecodeThread* mImageDecodeThread; @@ -129,6 +133,8 @@ private: F32 mTextureBandwidth; F32 mMaxBandwidth; LLTextureInfo mTextureInfo; + + U32 mHTTPTextureBits; }; #endif // LL_LLTEXTUREFETCH_H diff --git a/indra/newview/llviewerimage.cpp b/indra/newview/llviewerimage.cpp index d7d38d476..227404e84 100644 --- a/indra/newview/llviewerimage.cpp +++ b/indra/newview/llviewerimage.cpp @@ -1026,8 +1026,7 @@ bool LLViewerImage::updateFetch() S32 current_discard = getDiscardLevel(); S32 desired_discard = getDesiredDiscardLevel(); F32 decode_priority = getDecodePriority(); - decode_priority = llmax(decode_priority, 0.0f); - decode_priority = llmin(decode_priority, maxDecodePriority()); + decode_priority = llclamp(decode_priority, 0.0f, maxDecodePriority()); if (mIsFetching) { @@ -1053,7 +1052,7 @@ bool LLViewerImage::updateFetch() else { mFetchState = LLAppViewer::getTextureFetch()->getFetchState(mID, mDownloadProgress, mRequestedDownloadPriority, - mFetchPriority, mFetchDeltaTime, mRequestDeltaTime); + mFetchPriority, mFetchDeltaTime, mRequestDeltaTime, mCanUseHTTP); } // We may have data ready regardless of whether or not we are finished (e.g. waiting on write) @@ -1200,7 +1199,7 @@ bool LLViewerImage::updateFetch() mRequestedDiscardLevel = desired_discard; mFetchState = LLAppViewer::getTextureFetch()->getFetchState(mID, mDownloadProgress, mRequestedDownloadPriority, - mFetchPriority, mFetchDeltaTime, mRequestDeltaTime); + mFetchPriority, mFetchDeltaTime, mRequestDeltaTime, mCanUseHTTP); } // if createRequest() failed, we're finishing up a request for this UUID, @@ -1282,7 +1281,7 @@ BOOL LLViewerImage::forceFetch() mRequestedDiscardLevel = desired_discard ; mFetchState = LLAppViewer::getTextureFetch()->getFetchState(mID, mDownloadProgress, mRequestedDownloadPriority, - mFetchPriority, mFetchDeltaTime, mRequestDeltaTime); + mFetchPriority, mFetchDeltaTime, mRequestDeltaTime, mCanUseHTTP); } return mIsFetching ? true : false; diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index 399081ed3..7d31d4085 100644 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -1492,10 +1492,8 @@ void init_debug_rendering_menu(LLMenuGL* menu) item = new LLMenuItemCheckGL("Disable Textures", menu_toggle_variable, NULL, menu_check_variable, (void*)&LLViewerImage::sDontLoadVolumeTextures); menu->append(item); -#if 1 //ndef LL_RELEASE_FOR_DOWNLOAD item = new LLMenuItemCheckGL("HTTP Get Textures", menu_toggle_control, NULL, menu_check_control, (void*)"ImagePipelineUseHTTP"); menu->append(item); -#endif item = new LLMenuItemCheckGL("Run Multiple Threads", menu_toggle_control, NULL, menu_check_control, (void*)"RunMultipleThreads"); menu->append(item); diff --git a/indra/newview/llviewerwindow.cpp b/indra/newview/llviewerwindow.cpp index 44a702619..21eff8a44 100644 --- a/indra/newview/llviewerwindow.cpp +++ b/indra/newview/llviewerwindow.cpp @@ -4358,7 +4358,14 @@ BOOL LLViewerWindow::rawSnapshot(LLImageRaw *raw, S32 image_width, S32 image_hei image_buffer_x = llfloor(snapshot_width*scale_factor) ; image_buffer_y = llfloor(snapshot_height *scale_factor) ; } + if (image_buffer_x > 0 && image_buffer_y > 0) + { raw->resize(image_buffer_x, image_buffer_y, 3); + } + else + { + return FALSE ; + } if(raw->isBufferInvalid()) { return FALSE ; diff --git a/indra/newview/llvotree.cpp b/indra/newview/llvotree.cpp index 03ea8d7db..3d52a4a6c 100644 --- a/indra/newview/llvotree.cpp +++ b/indra/newview/llvotree.cpp @@ -446,23 +446,36 @@ void LLVOTree::render(LLAgent &agent) void LLVOTree::setPixelAreaAndAngle(LLAgent &agent) { - // First calculate values as for any other object (for mAppAngle) - LLViewerObject::setPixelAreaAndAngle(agent); - - // Re-calculate mPixelArea accurately + LLVector3 center = getPositionAgent();//center of tree. + LLVector3 viewer_pos_agent = gAgent.getCameraPositionAgent(); + LLVector3 lookAt = center - viewer_pos_agent; + F32 dist = lookAt.normVec() ; + F32 cos_angle_to_view_dir = lookAt * LLViewerCamera::getInstance()->getXAxis() ; - // This should be the camera's center, as soon as we move to all region-local. - LLVector3 relative_position = getPositionAgent() - agent.getCameraPositionAgent(); - F32 range = relative_position.length(); // ugh, square root + F32 range = dist - getMinScale()/2; + if (range < F_ALMOST_ZERO) // range == zero + { + range = 0; + mAppAngle = 180.f; + } + else + { + mAppAngle = (F32) atan2( getMaxScale(), range) * RAD_TO_DEG; + } F32 max_scale = mBillboardScale * getMaxScale(); F32 area = max_scale * (max_scale*mBillboardRatio); - // Compute pixels per meter at the given range - F32 pixels_per_meter = LLViewerCamera::getInstance()->getViewHeightInPixels() / - (tan(LLViewerCamera::getInstance()->getView()) * range); + F32 pixels_per_meter = LLViewerCamera::getInstance()->getViewHeightInPixels() / (tan(LLViewerCamera::getInstance()->getView()) * dist); + mPixelArea = pixels_per_meter * pixels_per_meter * area ; + + F32 importance = LLFace::calcImportanceToCamera(cos_angle_to_view_dir, dist) ; + mPixelArea = LLFace::adjustPixelArea(importance, mPixelArea) ; + if (mPixelArea > LLViewerCamera::getInstance()->getScreenPixelArea()) + { + mAppAngle = 180.f; + } - mPixelArea = (pixels_per_meter) * (pixels_per_meter) * area; #if 0 // mAppAngle is a bit of voodoo; // use the one calculated LLViewerObject::setPixelAreaAndAngle above diff --git a/indra/newview/llvovolume.cpp b/indra/newview/llvovolume.cpp index b4d0dd3cd..281e30394 100644 --- a/indra/newview/llvovolume.cpp +++ b/indra/newview/llvovolume.cpp @@ -465,6 +465,27 @@ void LLVOVolume::updateTextureVirtualSize() else { vsize = face->getTextureVirtualSize(); + if (isAttachment()) + { + // Rez attachments faster and at full details ! + if (permYouOwner()) + { + // Our attachments must really rez fast and fully: + // we shouldn't have to zoom on them to get the textures + // fully loaded ! + imagep->setBoostLevel(LLViewerImageBoostLevel::BOOST_HUD); + imagep->dontDiscard(); + } + else + { + // Others' can get their texture discarded to avoid + // filling up the video buffers in crowded areas... + imagep->setBoostLevel(LLViewerImageBoostLevel::BOOST_SELECTED); + imagep->setAdditionalDecodePriority(1.5f); + vsize = (F32) LLViewerCamera::getInstance()->getScreenPixelArea(); + face->setPixelArea(vsize); // treat as full screen + } + } } mPixelArea = llmax(mPixelArea, face->getPixelArea());