From a559a092171f80fff7054fa059cba66499d3efe1 Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Wed, 9 Oct 2019 16:52:46 -0400 Subject: [PATCH] Fix a ton of small bugs in 64bit memory value handling --- indra/llcommon/llmemory.cpp | 164 ++++++++------------------ indra/llcommon/llmemory.h | 11 +- indra/llcommon/llsys.cpp | 103 ++++++---------- indra/llcommon/llsys.h | 5 - indra/newview/llappviewer.cpp | 6 +- indra/newview/llfeaturemanager.cpp | 2 +- indra/newview/llviewerdisplay.cpp | 4 +- indra/newview/llviewertexturelist.cpp | 8 +- indra/newview/llviewerwindow.cpp | 2 +- 9 files changed, 99 insertions(+), 206 deletions(-) diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index f843f3c96..1b73507a1 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -102,39 +102,47 @@ void LLMemory::initMaxHeapSizeGB(F32Gigabytes max_heap_size, BOOL prevent_heap_f //static void LLMemory::updateMemoryInfo() { -#if LL_WINDOWS - HANDLE self = GetCurrentProcess(); - PROCESS_MEMORY_COUNTERS counters; - - if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) +#if LL_WINDOWS + PROCESS_MEMORY_COUNTERS_EX counters; + counters.cb = sizeof(counters); + + if (!GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*) &counters, sizeof(counters))) { LL_WARNS() << "GetProcessMemoryInfo failed" << LL_ENDL; - return ; + return; } - sAllocatedMemInKB = (U32Bytes)(counters.WorkingSetSize) ; - sAllocatedPageSizeInKB = (U32Bytes)(counters.PagefileUsage) ; + sAllocatedMemInKB = U64Bytes(counters.WorkingSetSize); + sAllocatedPageSizeInKB = (counters.PagefileUsage != 0) ? U64Bytes(counters.PagefileUsage) : U64Bytes(counters.PrivateUsage); - U32Kilobytes avail_phys, avail_virtual; - LLMemoryInfo::getAvailableMemoryKB(avail_phys, avail_virtual) ; - sMaxPhysicalMemInKB = llmin(avail_phys + sAllocatedMemInKB, sMaxHeapSizeInKB); - - if(sMaxPhysicalMemInKB > sAllocatedMemInKB) + MEMORYSTATUSEX memorystat; + memorystat.dwLength = sizeof(memorystat); + if (!GlobalMemoryStatusEx(&memorystat)) { - sAvailPhysicalMemInKB = sMaxPhysicalMemInKB - sAllocatedMemInKB ; + LL_WARNS() << "GlobalMemoryStatusEx failed" << LL_ENDL; + return; + } +#if (defined(_WIN64) || defined(__amd64__) || defined(__x86_64__)) + sMaxPhysicalMemInKB = U64Bytes(memorystat.ullTotalPhys); + sAvailPhysicalMemInKB = U64Bytes(memorystat.ullAvailPhys); +#else + sMaxPhysicalMemInKB = llmin(U32Kilobytes(U64Bytes(memorystat.ullTotalPhys)), sMaxHeapSizeInKB); + if (sMaxPhysicalMemInKB > sAllocatedMemInKB) + { + sAvailPhysicalMemInKB = U64Bytes(memorystat.ullAvailPhys);; } else { sAvailPhysicalMemInKB = U32Kilobytes(0); } -#else - //not valid for other systems for now. - sAllocatedMemInKB = (U32Bytes)LLMemory::getCurrentRSS(); - sMaxPhysicalMemInKB = (U32Bytes)U32_MAX ; - sAvailPhysicalMemInKB = (U32Bytes)U32_MAX ; #endif - return ; +#else + //not valid for other systems for now. + sAllocatedMemInKB = U64Bytes(LLMemory::getCurrentRSS()); + sMaxPhysicalMemInKB = U64Bytes(U32_MAX); + sAvailPhysicalMemInKB = U64Bytes(U32_MAX); +#endif } // @@ -158,7 +166,7 @@ void* LLMemory::tryToAlloc(void* address, U32 size) return address ; #else return (void*)0x01 ; //skip checking -#endif +#endif } //static @@ -233,31 +241,31 @@ bool LLMemory::isMemoryPoolLow() //static U32Kilobytes LLMemory::getAvailableMemKB() { - return sAvailPhysicalMemInKB ; + return sAvailPhysicalMemInKB; } //static U32Kilobytes LLMemory::getMaxMemKB() { - return sMaxPhysicalMemInKB ; + return sMaxPhysicalMemInKB; } //static U32Kilobytes LLMemory::getAllocatedMemKB() { - return sAllocatedMemInKB ; + return sAllocatedMemInKB; } //---------------------------------------------------------------------------- #if defined(LL_WINDOWS) +//static U64 LLMemory::getCurrentRSS() { - HANDLE self = GetCurrentProcess(); PROCESS_MEMORY_COUNTERS counters; - - if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) + + if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) { LL_WARNS() << "GetProcessMemoryInfo failed" << LL_ENDL; return 0; @@ -266,35 +274,7 @@ U64 LLMemory::getCurrentRSS() return counters.WorkingSetSize; } -//static -U32 LLMemory::getWorkingSetSize() -{ - PROCESS_MEMORY_COUNTERS pmc ; - U32 ret = 0 ; - - if (GetProcessMemoryInfo( GetCurrentProcess(), &pmc, sizeof(pmc)) ) - { - ret = pmc.WorkingSetSize ; - } - - return ret ; -} - #elif defined(LL_DARWIN) - -/* - The API used here is not capable of dealing with 64-bit memory sizes, but is available before 10.4. - - Once we start requiring 10.4, we can use the updated API, which looks like this: - - task_basic_info_64_data_t basicInfo; - mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_64_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) - - Of course, this doesn't gain us anything unless we start building the viewer as a 64-bit executable, since that's the only way - for our memory allocation to exceed 2^32. -*/ - // if (sysctl(ctl, 2, &page_size, &size, NULL, 0) == -1) // { // LL_WARNS() << "Couldn't get page size" << LL_ENDL; @@ -307,16 +287,15 @@ U32 LLMemory::getWorkingSetSize() U64 LLMemory::getCurrentRSS() { U64 residentSize = 0; - task_basic_info_data_t basicInfo; - mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) + mach_task_basic_info_data_t basicInfo; + mach_msg_type_number_t basicInfoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) { - residentSize = basicInfo.resident_size; - - // If we ever wanted it, the process virtual size is also available as: - // virtualSize = basicInfo.virtual_size; - -// LL_INFOS() << "resident size is " << residentSize << LL_ENDL; +// residentSize = basicInfo.resident_size; + // Although this method is defined to return the "resident set size," + // in fact what callers want from it is the total virtual memory + // consumed by the application. + residentSize = basicInfo.virtual_size; } else { @@ -326,11 +305,6 @@ U64 LLMemory::getCurrentRSS() return residentSize; } -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - #elif defined(LL_LINUX) U64 LLMemory::getCurrentRSS() @@ -342,7 +316,7 @@ U64 LLMemory::getCurrentRSS() if (fp == NULL) { LL_WARNS() << "couldn't open " << statPath << LL_ENDL; - goto bail; + return rss; } // Eee-yew! See Documentation/filesystems/proc.txt in your @@ -361,49 +335,8 @@ U64 LLMemory::getCurrentRSS() fclose(fp); -bail: return rss; } - -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - -#elif LL_SOLARIS -#include -#include -#include -#define _STRUCTURED_PROC 1 -#include - -U64 LLMemory::getCurrentRSS() -{ - char path [LL_MAX_PATH]; /* Flawfinder: ignore */ - - sprintf(path, "/proc/%d/psinfo", (int)getpid()); - int proc_fd = -1; - if((proc_fd = open(path, O_RDONLY)) == -1){ - LL_WARNS() << "LLmemory::getCurrentRSS() unable to open " << path << ". Returning 0 RSS!" << LL_ENDL; - return 0; - } - psinfo_t proc_psinfo; - if(read(proc_fd, &proc_psinfo, sizeof(psinfo_t)) != sizeof(psinfo_t)){ - LL_WARNS() << "LLmemory::getCurrentRSS() Unable to read from " << path << ". Returning 0 RSS!" << LL_ENDL; - close(proc_fd); - return 0; - } - - close(proc_fd); - - return((U64)proc_psinfo.pr_rssize * 1024); -} - -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - #else U64 LLMemory::getCurrentRSS() @@ -411,11 +344,6 @@ U64 LLMemory::getCurrentRSS() return 0; } -U32 LLMemory::getWorkingSetSize() -{ - return 0; -} - #endif //-------------------------------------------------------------------------------------------------- @@ -427,7 +355,7 @@ LLMemTracker* LLMemTracker::sInstance = NULL ; LLMemTracker::LLMemTracker() { - mLastAllocatedMem = LLMemory::getWorkingSetSize() ; + mLastAllocatedMem = LLMemory::getCurrentRSS() ; mCapacity = 128 ; mCurIndex = 0 ; mCounter = 0 ; @@ -480,7 +408,7 @@ void LLMemTracker::track(const char* function, const int line) return ; } - U32 allocated_mem = LLMemory::getWorkingSetSize() ; + U64 allocated_mem = LLMemory::getCurrentRSS() ; LLMutexLock lock(mMutexp) ; diff --git a/indra/llcommon/llmemory.h b/indra/llcommon/llmemory.h index efd61590a..18af01f65 100644 --- a/indra/llcommon/llmemory.h +++ b/indra/llcommon/llmemory.h @@ -133,11 +133,15 @@ template T* LL_NEXT_ALIGNED_ADDRESS_64(T* address) #if defined(LL_WINDOWS) return _aligned_malloc(size, align); #else + char* aligned = NULL; void* mem = malloc( size + (align - 1) + sizeof(void*) ); - char* aligned = ((char*)mem) + sizeof(void*); - aligned += align - ((uintptr_t)aligned & (align - 1)); + if (mem) + { + aligned = ((char*)mem) + sizeof(void*); + aligned += align - ((uintptr_t)aligned & (align - 1)); - ((void**)aligned)[-1] = mem; + ((void**)aligned)[-1] = mem; + } return aligned; #endif } @@ -365,7 +369,6 @@ public: // Return the resident set size of the current process, in bytes. // Return value is zero if not known. static U64 getCurrentRSS(); - static U32 getWorkingSetSize(); static void* tryToAlloc(void* address, U32 size); static void initMaxHeapSizeGB(F32Gigabytes max_heap_size, BOOL prevent_heap_failure); static void updateMemoryInfo() ; diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index c1249984f..ba69408a7 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -702,28 +702,10 @@ LLMemoryInfo::LLMemoryInfo() refresh(); } -#if LL_WINDOWS -static U32Kilobytes LLMemoryAdjustKBResult(U32Kilobytes inKB) -{ - // Moved this here from llfloaterabout.cpp - - //! \bug - // For some reason, the reported amount of memory is always wrong. - // The original adjustment assumes it's always off by one meg, however - // errors of as much as 2520 KB have been observed in the value - // returned from the GetMemoryStatusEx function. Here we keep the - // original adjustment from llfoaterabout.cpp until this can be - // fixed somehow. - inKB += U32Megabytes(1); - - return inKB; -} -#endif - U32Kilobytes LLMemoryInfo::getPhysicalMemoryKB() const { #if LL_WINDOWS - return LLMemoryAdjustKBResult(U32Kilobytes(mStatsMap["Total Physical KB"].asInteger())); + return U32Kilobytes(mStatsMap["Total Physical KB"].asInteger()); #elif LL_DARWIN // This might work on Linux as well. Someone check... @@ -740,43 +722,22 @@ U32Kilobytes LLMemoryInfo::getPhysicalMemoryKB() const phys = (U64)(getpagesize()) * (U64)(get_phys_pages()); return U64Bytes(phys); -#elif LL_SOLARIS - U64 phys = 0; - phys = (U64)(getpagesize()) * (U64)(sysconf(_SC_PHYS_PAGES)); - return U64Bytes(phys); - #else return 0; #endif } -U32Bytes LLMemoryInfo::getPhysicalMemoryClamped() const -{ - // Return the total physical memory in bytes, but clamp it - // to no more than U32_MAX - - U32Kilobytes phys_kb = getPhysicalMemoryKB(); - if (phys_kb >= U32Gigabytes(4)) - { - return U32Bytes(U32_MAX); - } - else - { - return phys_kb; - } -} - //static void LLMemoryInfo::getAvailableMemoryKB(U32Kilobytes& avail_physical_mem_kb, U32Kilobytes& avail_virtual_mem_kb) { #if LL_WINDOWS - // Sigh, this shouldn't be a static method, then we wouldn't have to - // reload this data separately from refresh() - LLSD statsMap(loadStatsMap()); + MEMORYSTATUSEX state; + state.dwLength = sizeof(state); + GlobalMemoryStatusEx(&state); - avail_physical_mem_kb = (U32Kilobytes)statsMap["Avail Physical KB"].asInteger(); - avail_virtual_mem_kb = (U32Kilobytes)statsMap["Avail Virtual KB"].asInteger(); + avail_physical_mem_kb = U64Bytes(state.ullAvailPhys); + avail_virtual_mem_kb = U64Bytes(state.ullAvailVirtual); #elif LL_DARWIN // mStatsMap is derived from vm_stat, look for (e.g.) "kb free": @@ -924,7 +885,7 @@ LLSD LLMemoryInfo::loadStatsMap() DWORDLONG div = 1024; - stats.add("Percent Memory use", state.dwMemoryLoad/div); + stats.add("Percent Memory use", state.dwMemoryLoad); stats.add("Total Physical KB", state.ullTotalPhys/div); stats.add("Avail Physical KB", state.ullAvailPhys/div); stats.add("Total page KB", state.ullTotalPageFile/div); @@ -973,27 +934,33 @@ LLSD LLMemoryInfo::loadStatsMap() stats.add("PrivateUsage KB", pmem.PrivateUsage/div); #elif LL_DARWIN - - const vm_size_t pagekb(vm_page_size / 1024); + vm_size_t page_size_kb; + if (host_page_size(mach_host_self(), &page_size_kb) != KERN_SUCCESS) + { + LL_WARNS() << "Unable to get host page size. Using default value." << LL_ENDL; + page_size_kb = 4096; + } + page_size_kb = page_size_kb / 1024; + // // Collect the vm_stat's // { - vm_statistics_data_t vmstat; - mach_msg_type_number_t vmstatCount = HOST_VM_INFO_COUNT; + vm_statistics64_data_t vmstat; + mach_msg_type_number_t vmstatCount = HOST_VM_INFO64_COUNT; - if (host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vmstat, &vmstatCount) != KERN_SUCCESS) + if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t) &vmstat, &vmstatCount) != KERN_SUCCESS) { LL_WARNS("LLMemoryInfo") << "Unable to collect memory information" << LL_ENDL; } else { - stats.add("Pages free KB", pagekb * vmstat.free_count); - stats.add("Pages active KB", pagekb * vmstat.active_count); - stats.add("Pages inactive KB", pagekb * vmstat.inactive_count); - stats.add("Pages wired KB", pagekb * vmstat.wire_count); + stats.add("Pages free KB", page_size_kb * vmstat.free_count); + stats.add("Pages active KB", page_size_kb * vmstat.active_count); + stats.add("Pages inactive KB", page_size_kb * vmstat.inactive_count); + stats.add("Pages wired KB", page_size_kb * vmstat.wire_count); stats.add("Pages zero fill", vmstat.zero_fill_count); stats.add("Page reactivations", vmstat.reactivations); @@ -1042,20 +1009,20 @@ LLSD LLMemoryInfo::loadStatsMap() // { - task_basic_info_64_data_t taskinfo; - unsigned taskinfoSize = sizeof(taskinfo); - - if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t) &taskinfo, &taskinfoSize) != KERN_SUCCESS) + mach_task_basic_info_data_t taskinfo; + mach_msg_type_number_t task_count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t) &taskinfo, &task_count) != KERN_SUCCESS) { - LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; - } - else - { - stats.add("Basic suspend count", taskinfo.suspend_count); - stats.add("Basic virtual memory KB", taskinfo.virtual_size / 1024); - stats.add("Basic resident memory KB", taskinfo.resident_size / 1024); - stats.add("Basic new thread policy", taskinfo.policy); - } + LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; + } + else + { + stats.add("Basic virtual memory KB", taskinfo.virtual_size / 1024); + stats.add("Basic resident memory KB", taskinfo.resident_size / 1024); + stats.add("Basic max resident memory KB", taskinfo.resident_size_max / 1024); + stats.add("Basic new thread policy", taskinfo.policy); + stats.add("Basic suspend count", taskinfo.suspend_count); + } } #elif LL_SOLARIS diff --git a/indra/llcommon/llsys.h b/indra/llcommon/llsys.h index accdbb5d2..d96319941 100644 --- a/indra/llcommon/llsys.h +++ b/indra/llcommon/llsys.h @@ -114,11 +114,6 @@ public: U32Kilobytes getPhysicalMemoryKB() const; - /*! Memory size in bytes, if total memory is >= 4GB then U32_MAX will - ** be returned. - */ - U32Bytes getPhysicalMemoryClamped() const; ///< Memory size in clamped bytes - //get the available memory infomation in KiloBytes. static void getAvailableMemoryKB(U32Kilobytes& avail_physical_mem_kb, U32Kilobytes& avail_virtual_mem_kb); diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index bbeb286a0..aa368032d 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -990,7 +990,7 @@ bool LLAppViewer::init() minSpecs += "\n"; unsupported = true; } - if(gSysMemory.getPhysicalMemoryClamped() < minRAM) + if(gSysMemory.getPhysicalMemoryKB() < minRAM) { minSpecs += LLNotificationTemplates::instance().getGlobalString("UnsupportedRAM"); minSpecs += "\n"; @@ -2692,8 +2692,8 @@ void LLAppViewer::writeSystemInfo() gDebugInfo["CPUInfo"]["CPUSSE"] = gSysCPU.hasSSE(); gDebugInfo["CPUInfo"]["CPUSSE2"] = gSysCPU.hasSSE2(); - gDebugInfo["RAMInfo"]["Physical"] = (LLSD::Integer)(gSysMemory.getPhysicalMemoryKB().value()); - gDebugInfo["RAMInfo"]["Allocated"] = (LLSD::Integer)(gMemoryAllocated.valueInUnits()); + gDebugInfo["RAMInfo"]["Physical"] = LLSD::Integer(gSysMemory.getPhysicalMemoryKB().value()); + gDebugInfo["RAMInfo"]["Allocated"] = LLSD::Integer(gMemoryAllocated.valueInUnits()); gDebugInfo["OSInfo"] = getOSInfo().getOSStringSimple(); // The user is not logged on yet, but record the current grid choice login url diff --git a/indra/newview/llfeaturemanager.cpp b/indra/newview/llfeaturemanager.cpp index 096df5cde..8ecf9f8c6 100644 --- a/indra/newview/llfeaturemanager.cpp +++ b/indra/newview/llfeaturemanager.cpp @@ -679,7 +679,7 @@ void LLFeatureManager::applyBaseMasks() maskFeatures(gpustr); // now mask cpu type ones - if (gSysMemory.getPhysicalMemoryClamped() <= U32Megabytes(256)) + if (gSysMemory.getPhysicalMemoryKB() <= U32Megabytes(256)) { maskFeatures("RAM256MB"); } diff --git a/indra/newview/llviewerdisplay.cpp b/indra/newview/llviewerdisplay.cpp index 5aa8239d4..6d694ca3b 100644 --- a/indra/newview/llviewerdisplay.cpp +++ b/indra/newview/llviewerdisplay.cpp @@ -238,9 +238,9 @@ void display_stats() F32 mem_log_freq = gSavedSettings.getF32("MemoryLogFrequency"); if (mem_log_freq > 0.f && gRecentMemoryTime.getElapsedTimeF32() >= mem_log_freq) { - gMemoryAllocated = (U64Bytes)LLMemory::getCurrentRSS(); + gMemoryAllocated = U64Bytes(LLMemory::getCurrentRSS()); U32Megabytes memory = gMemoryAllocated; - LL_INFOS() << llformat("MEMORY: %d MB", memory.value()) << LL_ENDL; + LL_INFOS() << "MEMORY: " << memory << LL_ENDL; LL_INFOS() << "THREADS: "<< LLThread::getCount() << LL_ENDL; LL_INFOS() << "MALLOC: " << SGMemStat::getPrintableStat() < LLViewerTextureList::convertToUploadFile(LLPointer S32Megabytes(1500)) ? S32Megabytes(64) : gMinVideoRam ; } @@ -1325,15 +1325,15 @@ S32Megabytes LLViewerTextureList::getMaxVideoRamSetting(bool get_recommended, fl { if (!get_recommended) { - max_texmem = (S32Megabytes)512; + max_texmem = S32Megabytes(512); } else if (gSavedSettings.getBOOL("NoHardwareProbe")) //did not do hardware detection at startup { - max_texmem = (S32Megabytes)512; + max_texmem = S32Megabytes(512); } else { - max_texmem = (S32Megabytes)128; + max_texmem = S32Megabytes(128); } LL_WARNS() << "VRAM amount not detected, defaulting to " << max_texmem << " MB" << LL_ENDL; diff --git a/indra/newview/llviewerwindow.cpp b/indra/newview/llviewerwindow.cpp index 5854038ab..608a4529a 100644 --- a/indra/newview/llviewerwindow.cpp +++ b/indra/newview/llviewerwindow.cpp @@ -385,7 +385,7 @@ public: static const LLCachedControl debug_show_memory("DebugShowMemory"); if (debug_show_memory) { - addText(xpos, ypos, llformat("Memory: %d (KB)", LLMemory::getWorkingSetSize() / 1024)); + addText(xpos, ypos, llformat("Memory: %d (KB)", LLMemory::getCurrentRSS() / 1024)); ypos += y_inc; } #endif