Index: ps/trunk/source/lib/sysdep/arch/x86_x64/apic.cpp =================================================================== --- ps/trunk/source/lib/sysdep/arch/x86_x64/apic.cpp (revision 24325) +++ ps/trunk/source/lib/sysdep/arch/x86_x64/apic.cpp (revision 24326) @@ -1,155 +1,162 @@ -/* Copyright (C) 2011 Wildfire Games. +/* Copyright (C) 2020 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "precompiled.h" #include "lib/sysdep/arch/x86_x64/apic.h" #include "lib/bits.h" #include "lib/module_init.h" #include "lib/sysdep/cpu.h" // ERR::CPU_FEATURE_MISSING #include "lib/sysdep/os_cpu.h" #include "lib/sysdep/arch/x86_x64/x86_x64.h" ApicId GetApicId() { x86_x64::CpuidRegs regs = { 0 }; regs.eax = 1; // note: CPUID function 1 is always supported, but only processors with // an xAPIC (e.g. P4/Athlon XP) will return a nonzero ID. bool ok = x86_x64::cpuid(®s); ASSERT(ok); UNUSED2(ok); const u8 apicId = (u8)bits(regs.ebx, 24, 31); return apicId; } static size_t numIds; static ApicId processorApicIds[os_cpu_MaxProcessors]; static ApicId sortedApicIds[os_cpu_MaxProcessors]; static Status GetAndValidateApicIds() { numIds = os_cpu_NumProcessors(); struct StoreEachProcessorsApicId { static void Callback(size_t processor, uintptr_t UNUSED(data)) { processorApicIds[processor] = GetApicId(); } }; // (can fail due to restrictions on our process affinity or lack of // support for affinity masks in OS X.) RETURN_STATUS_IF_ERR(os_cpu_CallByEachCPU(StoreEachProcessorsApicId::Callback, 0)); std::copy(processorApicIds, processorApicIds+numIds, sortedApicIds); std::sort(sortedApicIds, sortedApicIds+numIds); ApicId* const end = std::unique(sortedApicIds, sortedApicIds+numIds); const size_t numUnique = end-sortedApicIds; // all IDs are zero - system lacks an xAPIC. // (NB: we exclude single-processor systems in this test - // having one zero-valued ID is legitimate) if(numUnique == 1 && sortedApicIds[0] == 0 && numIds != 1) { debug_printf("APIC: all zero\n"); return ERR::CPU_FEATURE_MISSING; // NOWARN } // not all unique - probably running in a VM whose emulation is // imperfect or doesn't allow access to all processors. if(numUnique != numIds) { debug_printf("APIC: not unique\n"); return ERR::FAIL; // NOWARN } return INFO::OK; } static Status InitApicIds() { const Status status = GetAndValidateApicIds(); if(status < 0) // failed { // generate fake but legitimate APIC IDs for(size_t processor = 0; processor < numIds; processor++) processorApicIds[processor] = sortedApicIds[processor] = (ApicId)processor; } return status; } static ModuleInitState apicInitState; bool AreApicIdsReliable() { ModuleInit(&apicInitState, InitApicIds); if(apicInitState < 0) return false; return true; } +bool IsProcessorKnown(ApicId apicId) +{ + ModuleInit(&apicInitState, InitApicIds); + + const ApicId* pos = std::find(processorApicIds, processorApicIds+numIds, apicId); + return pos != processorApicIds+numIds; +} static size_t IndexFromApicId(const ApicId* apicIds, ApicId apicId) { ModuleInit(&apicInitState, InitApicIds); const ApicId* pos = std::find(apicIds, apicIds+numIds, apicId); if(pos == apicIds+numIds) { DEBUG_WARN_ERR(ERR::LOGIC); return 0; } const size_t index = pos - apicIds; return index; } size_t ProcessorFromApicId(ApicId apicId) { return IndexFromApicId(processorApicIds, apicId); } size_t ContiguousIdFromApicId(ApicId apicId) { return IndexFromApicId(sortedApicIds, apicId); } static ApicId ApicIdFromIndex(const ApicId* apicIds, size_t index) { ModuleInit(&apicInitState, InitApicIds); ASSERT(index < numIds); return apicIds[index]; } ApicId ApicIdFromProcessor(size_t processor) { return ApicIdFromIndex(processorApicIds, processor); } ApicId ApicIdFromContiguousId(size_t contiguousId) { return ApicIdFromIndex(sortedApicIds, contiguousId); } Index: ps/trunk/source/lib/sysdep/arch/x86_x64/apic.h =================================================================== --- ps/trunk/source/lib/sysdep/arch/x86_x64/apic.h (revision 24325) +++ ps/trunk/source/lib/sysdep/arch/x86_x64/apic.h (revision 24326) @@ -1,49 +1,52 @@ /* Copyright (C) 2020 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef INCLUDED_X86_X64_APIC #define INCLUDED_X86_X64_APIC typedef u8 ApicId; // not necessarily contiguous values /** * @return APIC ID of the currently executing processor or zero if the * platform does not have an xAPIC (i.e. 7th generation x86 or below). * * rationale: the alternative of accessing the APIC mmio registers is not * feasible. We also don't want to interfere with the OS's constant use of * the APIC registers. **/ LIB_API ApicId GetApicId(); // if this returns false, apicId = contiguousId = processor. // otherwise, there are unspecified but bijective mappings between // apicId<->contiguousId and apicId<->processor. LIB_API bool AreApicIdsReliable(); +// we may get the apicId of a processor we don't have access to. +LIB_API bool IsProcessorKnown(ApicId apicId); + LIB_API size_t ProcessorFromApicId(ApicId apicId); LIB_API size_t ContiguousIdFromApicId(ApicId apicId); LIB_API ApicId ApicIdFromProcessor(size_t contiguousId); LIB_API ApicId ApicIdFromContiguousId(size_t contiguousId); #endif // #ifndef INCLUDED_X86_X64_APIC Index: ps/trunk/source/lib/sysdep/os/win/wnuma.cpp =================================================================== --- ps/trunk/source/lib/sysdep/os/win/wnuma.cpp (revision 24325) +++ ps/trunk/source/lib/sysdep/os/win/wnuma.cpp (revision 24326) @@ -1,518 +1,524 @@ -/* Copyright (C) 2018 Wildfire Games. +/* Copyright (C) 2020 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "precompiled.h" #include "lib/sysdep/numa.h" #include "lib/bits.h" // PopulationCount #include "lib/alignment.h" #include "lib/lib.h" #include "lib/timer.h" #include "lib/module_init.h" #include "lib/sysdep/vm.h" #include "lib/sysdep/acpi.h" #include "lib/sysdep/os_cpu.h" #include "lib/sysdep/os/win/win.h" #include "lib/sysdep/os/win/wutil.h" #include "lib/sysdep/os/win/wcpu.h" #include #include #if ARCH_X86_X64 #include "lib/sysdep/arch/x86_x64/apic.h" // ProcessorFromApicId #endif //----------------------------------------------------------------------------- // nodes struct Node // POD { // (Windows doesn't guarantee node numbers are contiguous, so // we associate them with contiguous indices in nodes[]) UCHAR nodeNumber; u32 proximityDomainNumber; uintptr_t processorMask; }; static Node nodes[os_cpu_MaxProcessors]; static size_t numNodes; static Node* AddNode() { ENSURE(numNodes < ARRAY_SIZE(nodes)); return &nodes[numNodes++]; } static Node* FindNodeWithProcessorMask(uintptr_t processorMask) { for(size_t node = 0; node < numNodes; node++) { if(nodes[node].processorMask == processorMask) return &nodes[node]; } return 0; } static Node* FindNodeWithProcessor(size_t processor) { for(size_t node = 0; node < numNodes; node++) { if(IsBitSet(nodes[node].processorMask, processor)) return &nodes[node]; } return 0; } //----------------------------------------------------------------------------- // Windows topology static UCHAR HighestNodeNumber() { WUTIL_FUNC(pGetNumaHighestNodeNumber, BOOL, (PULONG)); WUTIL_IMPORT_KERNEL32(GetNumaHighestNodeNumber, pGetNumaHighestNodeNumber); if(!pGetNumaHighestNodeNumber) return 0; // NUMA not supported => only one node ULONG highestNodeNumber; const BOOL ok = pGetNumaHighestNodeNumber(&highestNodeNumber); WARN_IF_FALSE(ok); return (UCHAR)highestNodeNumber; } static void PopulateNodes() { WUTIL_FUNC(pGetNumaNodeProcessorMask, BOOL, (UCHAR, PULONGLONG)); WUTIL_IMPORT_KERNEL32(GetNumaNodeProcessorMask, pGetNumaNodeProcessorMask); if(!pGetNumaNodeProcessorMask) return; DWORD_PTR processAffinity, systemAffinity; { const BOOL ok = GetProcessAffinityMask(GetCurrentProcess(), &processAffinity, &systemAffinity); WARN_IF_FALSE(ok); } ENSURE(PopulationCount(processAffinity) <= PopulationCount(systemAffinity)); for(UCHAR nodeNumber = 0; nodeNumber <= HighestNodeNumber(); nodeNumber++) { ULONGLONG affinity; { const BOOL ok = pGetNumaNodeProcessorMask(nodeNumber, &affinity); WARN_IF_FALSE(ok); } if(!affinity) continue; // empty node, skip Node* node = AddNode(); node->nodeNumber = nodeNumber; node->processorMask = wcpu_ProcessorMaskFromAffinity(processAffinity, (DWORD_PTR)affinity); } } //----------------------------------------------------------------------------- // ACPI SRAT topology #if ARCH_X86_X64 #pragma pack(push, 1) // fields common to Affinity* structures struct AffinityHeader { u8 type; u8 length; // size [bytes], including this header }; struct AffinityAPIC { static const u8 type = 0; AffinityHeader header; u8 proximityDomainNumber0; u8 apicId; u32 flags; u8 sapicId; u8 proximityDomainNumber123[3]; u32 clockDomain; u32 ProximityDomainNumber() const { // (this is the apparent result of backwards compatibility, ugh.) u32 proximityDomainNumber; memcpy(&proximityDomainNumber, &proximityDomainNumber123[0]-1, sizeof(proximityDomainNumber)); proximityDomainNumber &= ~0xFF; proximityDomainNumber |= proximityDomainNumber0; return proximityDomainNumber; } }; struct AffinityMemory { static const u8 type = 1; AffinityHeader header; u32 proximityDomainNumber; u16 reserved1; u64 baseAddress; u64 length; u32 reserved2; u32 flags; u64 reserved3; }; // AffinityX2APIC omitted, since the APIC ID is sufficient for our purposes // Static Resource Affinity Table struct SRAT { AcpiTable header; u32 reserved1; u8 reserved2[8]; AffinityHeader affinities[1]; }; #pragma pack(pop) template static const Affinity* DynamicCastFromHeader(const AffinityHeader* header) { if(header->type != Affinity::type) return 0; // sanity check: ensure no padding was inserted ENSURE(header->length == sizeof(Affinity)); const Affinity* affinity = (const Affinity*)header; if(!IsBitSet(affinity->flags, 0)) // not enabled return 0; return affinity; } struct ProximityDomain { uintptr_t processorMask; // (AffinityMemory's fields are not currently needed) }; typedef std::map ProximityDomains; static ProximityDomains ExtractProximityDomainsFromSRAT(const SRAT* srat) { ProximityDomains proximityDomains; for(const AffinityHeader* header = srat->affinities; header < (const AffinityHeader*)(uintptr_t(srat)+srat->header.size); header = (const AffinityHeader*)(uintptr_t(header) + header->length)) { const AffinityAPIC* affinityAPIC = DynamicCastFromHeader(header); - if(affinityAPIC) + if(!affinityAPIC) + continue; + + if(!IsProcessorKnown(affinityAPIC->apicId)) { - const size_t processor = ProcessorFromApicId(affinityAPIC->apicId); - const u32 proximityDomainNumber = affinityAPIC->ProximityDomainNumber(); - ProximityDomain& proximityDomain = proximityDomains[proximityDomainNumber]; - proximityDomain.processorMask |= Bit(processor); + debug_printf("Processor with APIC ID %d found in SRAT but not known to CPUID.", affinityAPIC->apicId); + continue; } + + const size_t processor = ProcessorFromApicId(affinityAPIC->apicId); + const u32 proximityDomainNumber = affinityAPIC->ProximityDomainNumber(); + ProximityDomain& proximityDomain = proximityDomains[proximityDomainNumber]; + proximityDomain.processorMask |= Bit(processor); } return proximityDomains; } static void PopulateNodesFromProximityDomains(const ProximityDomains& proximityDomains) { for(ProximityDomains::const_iterator it = proximityDomains.begin(); it != proximityDomains.end(); ++it) { const u32 proximityDomainNumber = it->first; const ProximityDomain& proximityDomain = it->second; Node* node = FindNodeWithProcessorMask(proximityDomain.processorMask); if(!node) node = AddNode(); // (we don't know Windows' nodeNumber; it has hopefully already been set) node->proximityDomainNumber = proximityDomainNumber; node->processorMask = proximityDomain.processorMask; } } #endif // #if ARCH_X86_X64 //----------------------------------------------------------------------------- static ModuleInitState initState; static Status InitTopology() { PopulateNodes(); #if ARCH_X86_X64 const SRAT* srat = (const SRAT*)acpi_GetTable("SRAT"); if(srat && AreApicIdsReliable()) { const ProximityDomains proximityDomains = ExtractProximityDomainsFromSRAT(srat); PopulateNodesFromProximityDomains(proximityDomains); } #endif // neither OS nor ACPI information is available if(numNodes == 0) { // add dummy node that contains all system processors Node* node = AddNode(); node->nodeNumber = 0; node->proximityDomainNumber = 0; node->processorMask = os_cpu_ProcessorMask(); } return INFO::OK; } size_t numa_NumNodes() { UNUSED2(ModuleInit(&initState, InitTopology)); return numNodes; } size_t numa_NodeFromProcessor(size_t processor) { UNUSED2(ModuleInit(&initState, InitTopology)); ENSURE(processor < os_cpu_NumProcessors()); Node* node = FindNodeWithProcessor(processor); ENSURE(node); return nodes-node; } uintptr_t numa_ProcessorMaskFromNode(size_t node) { UNUSED2(ModuleInit(&initState, InitTopology)); ENSURE(node < numNodes); return nodes[node].processorMask; } static UCHAR NodeNumberFromNode(size_t node) { UNUSED2(ModuleInit(&initState, InitTopology)); ENSURE(node < numa_NumNodes()); return nodes[node].nodeNumber; } //----------------------------------------------------------------------------- // memory info size_t numa_AvailableMemory(size_t node) { // note: it is said that GetNumaAvailableMemoryNode sometimes incorrectly // reports zero bytes. the actual cause may however be unexpected // RAM configuration, e.g. not all slots filled. WUTIL_FUNC(pGetNumaAvailableMemoryNode, BOOL, (UCHAR, PULONGLONG)); WUTIL_IMPORT_KERNEL32(GetNumaAvailableMemoryNode, pGetNumaAvailableMemoryNode); if(pGetNumaAvailableMemoryNode) { const UCHAR nodeNumber = NodeNumberFromNode(node); ULONGLONG availableBytes; const BOOL ok = pGetNumaAvailableMemoryNode(nodeNumber, &availableBytes); WARN_IF_FALSE(ok); const size_t availableMiB = size_t(availableBytes / MiB); return availableMiB; } // NUMA not supported - return available system memory else return os_cpu_MemoryAvailable(); } #pragma pack(push, 1) // ACPI System Locality Information Table // (System Locality == Proximity Domain) struct SLIT { AcpiTable header; u64 numSystemLocalities; u8 entries[1]; // numSystemLocalities*numSystemLocalities entries }; #pragma pack(pop) static double ReadRelativeDistanceFromSLIT(const SLIT* slit) { const size_t n = slit->numSystemLocalities; ENSURE(slit->header.size == sizeof(SLIT)-sizeof(slit->entries)+n*n); // diagonals are specified to be 10 for(size_t i = 0; i < n; i++) ENSURE(slit->entries[i*n+i] == 10); // entries = relativeDistance * 10 return *std::max_element(slit->entries, slit->entries+n*n) / 10.0; } // @return ratio between max/min time required to access one node's // memory from each processor. static double MeasureRelativeDistance() { const size_t size = 32*MiB; void* mem = vm::Allocate(size); ASSUME_ALIGNED(mem, pageSize); const uintptr_t previousProcessorMask = os_cpu_SetThreadAffinityMask(os_cpu_ProcessorMask()); double minTime = 1e10, maxTime = 0.0; for(size_t node = 0; node < numa_NumNodes(); node++) { const uintptr_t processorMask = numa_ProcessorMaskFromNode(node); os_cpu_SetThreadAffinityMask(processorMask); const double startTime = timer_Time(); memset(mem, 0, size); const double elapsedTime = timer_Time() - startTime; minTime = std::min(minTime, elapsedTime); maxTime = std::max(maxTime, elapsedTime); } UNUSED2(os_cpu_SetThreadAffinityMask(previousProcessorMask)); vm::Free(mem, size); return maxTime / minTime; } static double relativeDistance; static Status InitRelativeDistance() { // early-out for non-NUMA systems (saves some time) if(numa_NumNodes() == 1) { relativeDistance = 1.0; return INFO::OK; } // trust values reported by the BIOS, if available const SLIT* slit = (const SLIT*)acpi_GetTable("SLIT"); if(slit) relativeDistance = ReadRelativeDistanceFromSLIT(slit); else relativeDistance = MeasureRelativeDistance(); ENSURE(relativeDistance >= 1.0); ENSURE(relativeDistance <= 4.0); return INFO::OK; } double numa_Factor() { static ModuleInitState _initState; UNUSED2(ModuleInit(&_initState, InitRelativeDistance)); return relativeDistance; } static bool IsMemoryInterleaved() { if(numa_NumNodes() == 1) return false; if(!acpi_GetTable("FACP")) // no ACPI tables available return false; // indeterminate, assume not interleaved if(acpi_GetTable("SRAT")) // present iff not interleaved return false; return true; } static bool isMemoryInterleaved; static Status InitMemoryInterleaved() { isMemoryInterleaved = IsMemoryInterleaved(); return INFO::OK; } bool numa_IsMemoryInterleaved() { static ModuleInitState _initState; UNUSED2(ModuleInit(&_initState, InitMemoryInterleaved)); return isMemoryInterleaved; } //----------------------------------------------------------------------------- #if 0 static bool VerifyPages(void* mem, size_t size, size_t pageSize, size_t node) { WUTIL_FUNC(pQueryWorkingSetEx, BOOL, (HANDLE, PVOID, DWORD)); WUTIL_IMPORT_KERNEL32(QueryWorkingSetEx, pQueryWorkingSetEx); if(!pQueryWorkingSetEx) return true; // can't do anything #if WINVER >= 0x600 size_t largePageSize = os_cpu_LargePageSize(); ENSURE(largePageSize != 0); // this value is needed for later // retrieve attributes of all pages constituting mem const size_t numPages = (size + pageSize-1) / pageSize; PSAPI_WORKING_SET_EX_INFORMATION* wsi = new PSAPI_WORKING_SET_EX_INFORMATION[numPages]; for(size_t i = 0; i < numPages; i++) wsi[i].VirtualAddress = (u8*)mem + i*pageSize; pQueryWorkingSetEx(GetCurrentProcess(), wsi, DWORD(sizeof(PSAPI_WORKING_SET_EX_INFORMATION)*numPages)); // ensure each is valid and allocated on the correct node for(size_t i = 0; i < numPages; i++) { const PSAPI_WORKING_SET_EX_BLOCK& attributes = wsi[i].VirtualAttributes; if(!attributes.Valid) return false; if((attributes.LargePage != 0) != (pageSize == largePageSize)) { debug_printf("NUMA: is not a large page\n"); return false; } if(attributes.Node != node) { debug_printf("NUMA: allocated from remote node\n"); return false; } } delete[] wsi; #else UNUSED2(mem); UNUSED2(size); UNUSED2(pageSize); UNUSED2(node); #endif return true; } #endif