From a82350bb774f70fcbd41c9987bf67a3775409963 Mon Sep 17 00:00:00 2001 From: LotP Date: Fri, 17 Jul 2026 12:57:07 +0000 Subject: [PATCH] mono-jit-v2 (#173) second attempt at single layer address tables (first attempt https://git.ryujinx.app/projects/Ryubing/pulls/167). should work with all games now. Co-authored-by: LotP1 <68976644+LotP1@users.noreply.github.com> Reviewed-on: https://git.ryujinx.app/projects/Ryubing/pulls/173 --- src/ARMeilleure/Common/AddressTablePresets.cs | 40 +-- src/ARMeilleure/Common/AddressTableType.cs | 8 + src/ARMeilleure/Common/BitUtils.cs | 9 + src/ARMeilleure/Common/IAddressTable.cs | 14 +- .../Instructions/InstEmitFlowHelper.cs | 27 +- src/ARMeilleure/Translation/PTC/Ptc.cs | 2 +- src/ARMeilleure/Translation/Translator.cs | 4 +- src/Ryujinx.Cpu/AddressTable.cs | 327 ++---------------- src/Ryujinx.Cpu/Jit/JitCpuContext.cs | 10 +- src/Ryujinx.Cpu/LightningJit/AarchCompiler.cs | 2 +- .../LightningJit/Arm32/A32Compiler.cs | 2 +- .../Arm32/Target/Arm64/Compiler.cs | 6 +- .../Arm32/Target/Arm64/InstEmitFlow.cs | 24 +- .../LightningJit/Arm64/A64Compiler.cs | 2 +- .../Arm64/Target/Arm64/Compiler.cs | 6 +- .../Arm64/Target/Arm64/InstEmitSystem.cs | 26 +- .../LightningJit/LightningJitCpuContext.cs | 10 +- src/Ryujinx.Cpu/LightningJit/Translator.cs | 5 +- src/Ryujinx.Cpu/SparseAddressTable.cs | 247 +++++++++++++ src/Ryujinx.Memory/SparseMemoryBlock.cs | 26 +- src/Ryujinx.Tests/Cpu/CpuContext.cs | 5 +- src/Ryujinx.Tests/Cpu/EnvironmentTests.cs | 4 +- src/Ryujinx.Tests/Memory/PartialUnmaps.cs | 2 +- 23 files changed, 401 insertions(+), 407 deletions(-) create mode 100644 src/ARMeilleure/Common/AddressTableType.cs create mode 100644 src/Ryujinx.Cpu/SparseAddressTable.cs diff --git a/src/ARMeilleure/Common/AddressTablePresets.cs b/src/ARMeilleure/Common/AddressTablePresets.cs index fd786fc7e..ce45c892a 100644 --- a/src/ARMeilleure/Common/AddressTablePresets.cs +++ b/src/ARMeilleure/Common/AddressTablePresets.cs @@ -20,45 +20,21 @@ namespace ARMeilleure.Common new( 1, 6) ]; - private static readonly AddressTableLevel[] _levels64BitSparseTiny = + private static readonly AddressTableLevel[] _levels64BitMono = [ - new( 11, 28), - new( 2, 9) + new( 2, 37) ]; - private static readonly AddressTableLevel[] _levels32BitSparseTiny = + private static readonly AddressTableLevel[] _levels32BitMono = [ - new( 10, 22), - new( 1, 9) + new( 1, 31) ]; - - private static readonly AddressTableLevel[] _levels64BitSparseGiant = - [ - new( 38, 1), - new( 2, 36) - ]; - - private static readonly AddressTableLevel[] _levels32BitSparseGiant = - [ - new( 31, 1), - new( 1, 30) - ]; - - //high power will run worse on DDR3 systems and some DDR4 systems due to the higher ram utilization - //low power will never run worse than non-sparse, but for most systems it won't be necessary - //high power is always used, but I've left low power in here for future reference - public static AddressTableLevel[] GetArmPreset(bool for64Bits, bool sparse, bool lowPower = false) + + public static AddressTableLevel[] GetArmPreset(bool for64Bits, bool mono) { - if (sparse) + if (mono) { - if (lowPower) - { - return for64Bits ? _levels64BitSparseTiny : _levels32BitSparseTiny; - } - else - { - return for64Bits ? _levels64BitSparseGiant : _levels32BitSparseGiant; - } + return for64Bits ? _levels64BitMono : _levels32BitMono; } else { diff --git a/src/ARMeilleure/Common/AddressTableType.cs b/src/ARMeilleure/Common/AddressTableType.cs new file mode 100644 index 000000000..b85118224 --- /dev/null +++ b/src/ARMeilleure/Common/AddressTableType.cs @@ -0,0 +1,8 @@ +namespace ARMeilleure.Common +{ + public enum AddressTableType + { + Default, + Sparse + } +} diff --git a/src/ARMeilleure/Common/BitUtils.cs b/src/ARMeilleure/Common/BitUtils.cs index 7e2ed7f60..17a00ae57 100644 --- a/src/ARMeilleure/Common/BitUtils.cs +++ b/src/ARMeilleure/Common/BitUtils.cs @@ -53,5 +53,14 @@ namespace ARMeilleure.Common { return (bits >> shift) | (bits << (size - shift)); } + + public static T AlignUp(T value, T size) where T : IBinaryInteger + => (value + (size - T.One)) & -size; + + public static T AlignDown(T value, T size) where T : IBinaryInteger + => value & -size; + + public static T DivRoundUp(T value, T dividend) where T : IBinaryInteger + => (value + (dividend - T.One)) / dividend; } } diff --git a/src/ARMeilleure/Common/IAddressTable.cs b/src/ARMeilleure/Common/IAddressTable.cs index 65077ec43..487f7dd31 100644 --- a/src/ARMeilleure/Common/IAddressTable.cs +++ b/src/ARMeilleure/Common/IAddressTable.cs @@ -5,11 +5,10 @@ namespace ARMeilleure.Common public interface IAddressTable : IDisposable where TEntry : unmanaged { /// - /// True if the address table's bottom level is sparsely mapped. - /// This also ensures the second bottom level is filled with a dummy page rather than 0. + /// Gets the of the instance. /// - bool Sparse { get; } - + AddressTableType TableType { get; } + /// /// Gets the bits used by the of the instance. /// @@ -31,6 +30,13 @@ namespace ARMeilleure.Common /// instance was disposed nint Base { get; } + /// + /// Signal that the given code range exists. + /// + /// Guest code range address + /// Guest code range size + void SignalCodeRange(ulong address, ulong size); + /// /// Determines if the specified is in the range of the /// . diff --git a/src/ARMeilleure/Instructions/InstEmitFlowHelper.cs b/src/ARMeilleure/Instructions/InstEmitFlowHelper.cs index 74866f982..dfd46d872 100644 --- a/src/ARMeilleure/Instructions/InstEmitFlowHelper.cs +++ b/src/ARMeilleure/Instructions/InstEmitFlowHelper.cs @@ -5,7 +5,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; using ARMeilleure.Translation; using ARMeilleure.Translation.PTC; - +using System.Linq; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.IntermediateRepresentation.Operand.Factory; @@ -236,9 +236,9 @@ namespace ARMeilleure.Instructions hostAddress = context.Load(OperandType.I64, hostAddressAddr); } - else if (table.Sparse) + else if (table.TableType == AddressTableType.Sparse) { - // Inline table lookup. Only enabled when the sparse function table is enabled with 2 levels. + // Inline table lookup. Only enabled when the sparse function table is enabled with 1 level. // Deliberately attempts to avoid branches. Operand tableBase = !context.HasPtc ? @@ -247,24 +247,21 @@ namespace ARMeilleure.Instructions hostAddress = tableBase; - for (int i = 0; i < table.Levels.Length; i++) - { - AddressTableLevel level = table.Levels[i]; - int clearBits = 64 - (level.Index + level.Length); + AddressTableLevel level = table.Levels.Last(); + int clearBits = 64 - (level.Index + level.Length); - Operand index = context.ShiftLeft( - context.ShiftRightUI(context.ShiftLeft(guestAddress, Const(clearBits)), Const(clearBits + level.Index)), - Const(3) - ); + Operand index = context.ShiftLeft( + context.ShiftRightUI(context.ShiftLeft(guestAddress, Const(clearBits)), Const(clearBits + level.Index)), + Const(3) + ); - hostAddress = context.Load(OperandType.I64, context.Add(hostAddress, index)); - } + hostAddress = context.Load(OperandType.I64, context.Add(hostAddress, index)); } else { hostAddress = !context.HasPtc ? - Const((long)context.Stubs.DispatchStub) : - Const((long)context.Stubs.DispatchStub, Ptc.DispatchStubSymbol); + Const(context.Stubs.DispatchStub) : + Const(context.Stubs.DispatchStub, Ptc.DispatchStubSymbol); } if (isJump) diff --git a/src/ARMeilleure/Translation/PTC/Ptc.cs b/src/ARMeilleure/Translation/PTC/Ptc.cs index 00152f744..de402809f 100644 --- a/src/ARMeilleure/Translation/PTC/Ptc.cs +++ b/src/ARMeilleure/Translation/PTC/Ptc.cs @@ -34,7 +34,7 @@ namespace ARMeilleure.Translation.PTC private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - private const uint InternalVersion = 7019; //! To be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 7020; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; diff --git a/src/ARMeilleure/Translation/Translator.cs b/src/ARMeilleure/Translation/Translator.cs index e73ecc85e..1c66af046 100644 --- a/src/ARMeilleure/Translation/Translator.cs +++ b/src/ARMeilleure/Translation/Translator.cs @@ -28,7 +28,7 @@ namespace ARMeilleure.Translation private readonly Ptc _ptc; internal TranslatorCache Functions { get; } - internal IAddressTable FunctionTable { get; } + public IAddressTable FunctionTable { get; } internal EntryTable CountTable { get; } internal TranslatorStubs Stubs { get; } internal TranslatorQueue Queue { get; } @@ -53,7 +53,9 @@ namespace ARMeilleure.Translation CountTable = new EntryTable(); Functions = new TranslatorCache(); + FunctionTable = functionTable; + Stubs = new TranslatorStubs(JitCache, FunctionTable); FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub; diff --git a/src/Ryujinx.Cpu/AddressTable.cs b/src/Ryujinx.Cpu/AddressTable.cs index 19e405491..bc77767ab 100644 --- a/src/Ryujinx.Cpu/AddressTable.cs +++ b/src/Ryujinx.Cpu/AddressTable.cs @@ -1,15 +1,9 @@ -using ARMeilleure.Memory; -using Ryujinx.Common; -using Ryujinx.Cpu.Signal; -using Ryujinx.Memory; +using ARMeilleure.Common; using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.InteropServices; -using System.Threading; -using static Ryujinx.Cpu.MemoryEhMeilleure; -namespace ARMeilleure.Common +namespace Ryujinx.Cpu { /// /// Represents a table of guest address to a value. @@ -17,84 +11,12 @@ namespace ARMeilleure.Common /// Type of the value public unsafe class AddressTable : IAddressTable where TEntry : unmanaged { - /// - /// Represents a page of the address table. - /// - private readonly struct AddressTablePage - { - /// - /// True if the allocation belongs to a sparse block, false otherwise. - /// - public readonly bool IsSparse; - - /// - /// Base address for the page. - /// - public readonly nint Address; - - public AddressTablePage(bool isSparse, nint address) - { - IsSparse = isSparse; - Address = address; - } - } - - /// - /// A sparsely mapped block of memory with a signal handler to map pages as they're accessed. - /// - private readonly struct TableSparseBlock : IDisposable - { - public readonly SparseMemoryBlock Block; - private readonly TrackingEventDelegate _trackingEvent; - - public TableSparseBlock(ulong size, Action ensureMapped, PageInitDelegate pageInit) - { - SparseMemoryBlock block = new(size, pageInit, null); - - _trackingEvent = (address, size, write) => - { - ulong pointer = (ulong)block.Block.Pointer + address; - ensureMapped((nint)pointer); - return pointer; - }; - - bool added = NativeSignalHandler.AddTrackedRegion( - (nuint)block.Block.Pointer, - (nuint)(block.Block.Pointer + (nint)block.Block.Size), - Marshal.GetFunctionPointerForDelegate(_trackingEvent)); - - if (!added) - { - throw new InvalidOperationException("Number of allowed tracked regions exceeded."); - } - - Block = block; - } - - public void Dispose() - { - NativeSignalHandler.RemoveTrackedRegion((nuint)Block.Block.Pointer); - - Block.Dispose(); - } - } - private bool _disposed; private TEntry** _table; - private readonly List _pages; - private TEntry _fill; + private readonly List _pages; - private readonly MemoryBlock _sparseFill; - private readonly SparseMemoryBlock _fillBottomLevel; - private readonly TEntry* _fillBottomLevelPtr; - - private readonly List _sparseReserved; - private readonly ReaderWriterLockSlim _sparseLock; - - private ulong _sparseBlockSize; - private ulong _sparseReservedOffset; - - public bool Sparse { get; } + /// + public AddressTableType TableType => AddressTableType.Default; /// public ulong Mask { get; } @@ -103,17 +25,7 @@ namespace ARMeilleure.Common public AddressTableLevel[] Levels { get; } /// - public TEntry Fill - { - get - { - return _fill; - } - set - { - UpdateFill(value); - } - } + public TEntry Fill { get; set; } /// public nint Base @@ -131,17 +43,16 @@ namespace ARMeilleure.Common /// /// Constructs a new instance of the class with the specified list of - /// . + /// . /// /// Levels for the address table - /// True if the bottom page should be sparsely mapped /// is null /// Length of is less than 2 - public AddressTable(AddressTableLevel[] levels, bool sparse) + public AddressTable(AddressTableLevel[] levels) { ArgumentNullException.ThrowIfNull(levels); - _pages = new List(capacity: 16); + _pages = new List(capacity: 16); Levels = levels; Mask = 0; @@ -150,25 +61,6 @@ namespace ARMeilleure.Common { Mask |= level.Mask; } - - Sparse = sparse; - - if (sparse) - { - // If the address table is sparse, allocate a fill block - - _sparseFill = new MemoryBlock(268435456ul, MemoryAllocationFlags.Mirrorable); //low Power TC uses size: 65536ul - - ulong bottomLevelSize = (1ul << levels.Last().Length) * (ulong)sizeof(TEntry); - - _fillBottomLevel = new SparseMemoryBlock(bottomLevelSize, null, _sparseFill); - _fillBottomLevelPtr = (TEntry*)_fillBottomLevel.Block.Pointer; - - _sparseReserved = []; - _sparseLock = new ReaderWriterLockSlim(); - - _sparseBlockSize = bottomLevelSize; - } } /// @@ -176,29 +68,10 @@ namespace ARMeilleure.Common /// Selects the best table structure for A32/A64, taking into account the selected memory manager type. /// /// True if the guest is A64, false otherwise - /// Memory manager type /// An for ARM function lookup - public static AddressTable CreateForArm(bool for64Bits, MemoryManagerType type) + public static AddressTable CreateForArm(bool for64Bits) { - // Assume software memory means that we don't want to use any signal handlers. - bool sparse = type is not MemoryManagerType.SoftwareMmu and not MemoryManagerType.SoftwarePageTable; - - return new AddressTable(AddressTablePresets.GetArmPreset(for64Bits, sparse), sparse); - } - - /// - /// Update the fill value for the bottom level of the table. - /// - /// New fill value - private void UpdateFill(TEntry fillValue) - { - if (_sparseFill != null) - { - Span span = _sparseFill.GetSpan(0, (int)_sparseFill.Size); - MemoryMarshal.Cast(span).Fill(fillValue); - } - - _fill = fillValue; + return new AddressTable(AddressTablePresets.GetArmPreset(for64Bits, false)); } /// @@ -206,17 +79,7 @@ namespace ARMeilleure.Common /// /// /// - public void SignalCodeRange(ulong address, ulong size) - { - AddressTableLevel bottom = Levels.Last(); - ulong bottomLevelEntries = 1ul << bottom.Length; - - ulong entryIndex = address >> bottom.Index; - ulong entries = size >> bottom.Index; - entries += entryIndex - BitUtils.AlignDown(entryIndex, bottomLevelEntries); - - _sparseBlockSize = Math.Max(_sparseBlockSize, BitUtils.AlignUp(entries, bottomLevelEntries) * (ulong)sizeof(TEntry)); - } + public void SignalCodeRange(ulong address, ulong size) { } /// public bool IsValid(ulong address) @@ -240,8 +103,6 @@ namespace ARMeilleure.Common long index = Levels[^1].GetValue(address); - EnsureMapped((nint)(page + index)); - return ref page[index]; } } @@ -258,19 +119,19 @@ namespace ARMeilleure.Common for (int i = 0; i < Levels.Length - 1; i++) { ref AddressTableLevel level = ref Levels[i]; - ref TEntry* nextPage = ref page[level.GetValue(address)]; + ref TEntry* nextPage = ref page![level.GetValue(address)]; - if (nextPage == null || nextPage == _fillBottomLevelPtr) + if (nextPage == null) { ref AddressTableLevel nextLevel = ref Levels[i + 1]; if (i == Levels.Length - 2) { - nextPage = (TEntry*)Allocate(1 << nextLevel.Length, Fill, leaf: true); + nextPage = (TEntry*)Allocate(1 << nextLevel.Length, Fill); } else { - nextPage = (TEntry*)Allocate(1 << nextLevel.Length, GetFillValue(i), leaf: false); + nextPage = (TEntry*)Allocate(1 << nextLevel.Length, nint.Zero); } } @@ -280,57 +141,6 @@ namespace ARMeilleure.Common return (TEntry*)page; } - /// - /// Ensure the given pointer is mapped in any overlapping sparse reservations. - /// - /// Pointer to be mapped - private void EnsureMapped(nint ptr) - { - if (Sparse) - { - // Check sparse allocations to see if the pointer is in any of them. - // Ensure the page is committed if there's a match. - - _sparseLock.EnterReadLock(); - - try - { - foreach (TableSparseBlock reserved in _sparseReserved) - { - SparseMemoryBlock sparse = reserved.Block; - - if (ptr >= sparse.Block.Pointer && ptr < sparse.Block.Pointer + (nint)sparse.Block.Size) - { - sparse.EnsureMapped((ulong)(ptr - sparse.Block.Pointer)); - - break; - } - } - } - finally - { - _sparseLock.ExitReadLock(); - } - } - } - - /// - /// Get the fill value for a non-leaf level of the table. - /// - /// Level to get the fill value for - /// The fill value - private nint GetFillValue(int level) - { - if (_fillBottomLevel != null && level == Levels.Length - 2) - { - return (nint)_fillBottomLevelPtr; - } - else - { - return nint.Zero; - } - } - /// /// Lazily initialize and get the root page of the . /// @@ -340,91 +150,35 @@ namespace ARMeilleure.Common if (_table == null) { if (Levels.Length == 1) - _table = (TEntry**)Allocate(1 << Levels[0].Length, Fill, leaf: true); + _table = (TEntry**)Allocate(1 << Levels[0].Length, Fill); else - _table = (TEntry**)Allocate(1 << Levels[0].Length, GetFillValue(0), leaf: false); + _table = (TEntry**)Allocate(1 << Levels[0].Length, nint.Zero); } return _table; } - /// - /// Initialize a leaf page with the fill value. - /// - /// Page to initialize - private void InitLeafPage(Span page) - { - MemoryMarshal.Cast(page).Fill(_fill); - } - - /// - /// Reserve a new sparse block, and add it to the list. - /// - /// The new sparse block that was added - private TableSparseBlock ReserveNewSparseBlock() - { - TableSparseBlock block = new(_sparseBlockSize, EnsureMapped, InitLeafPage); - - _sparseReserved.Add(block); - _sparseReservedOffset = 0; - - return block; - } - /// /// Allocates a block of memory of the specified type and length. /// /// Type of elements /// Number of elements /// Fill value - /// if leaf; otherwise /// Allocated block - private nint Allocate(int length, T fill, bool leaf) where T : unmanaged + private nint Allocate(int length, T fill) where T : unmanaged { int size = sizeof(T) * length; - AddressTablePage page; + nint address = (nint)NativeAllocator.Instance.Allocate((uint)size); - if (Sparse && leaf) - { - _sparseLock.EnterWriteLock(); + Span span = new((void*)address, length); + span.Fill(fill); - SparseMemoryBlock block; - - if (_sparseReserved.Count == 0) - { - block = ReserveNewSparseBlock().Block; - } - else - { - block = _sparseReserved.Last().Block; - - if (_sparseReservedOffset == block.Block.Size) - { - block = ReserveNewSparseBlock().Block; - } - } - - page = new AddressTablePage(true, block.Block.Pointer + (nint)_sparseReservedOffset); - - _sparseReservedOffset += (ulong)size; - - _sparseLock.ExitWriteLock(); - } - else - { - nint address = (nint)NativeAllocator.Instance.Allocate((uint)size); - page = new AddressTablePage(false, address); - - Span span = new((void*)page.Address, length); - span.Fill(fill); - } - - _pages.Add(page); + _pages.Add(address); //TranslatorEventSource.Log.AddressTableAllocated(size, leaf); - return page.Address; + return address; } /// @@ -443,32 +197,17 @@ namespace ARMeilleure.Common /// to dispose managed resources also; otherwise just unmanaged resouces protected virtual void Dispose(bool disposing) { - if (!_disposed) + if (_disposed) { - foreach (AddressTablePage page in _pages) - { - if (!page.IsSparse) - { - Marshal.FreeHGlobal(page.Address); - } - } - - if (Sparse) - { - foreach (TableSparseBlock block in _sparseReserved) - { - block.Dispose(); - } - - _sparseReserved.Clear(); - - _fillBottomLevel.Dispose(); - _sparseFill.Dispose(); - _sparseLock.Dispose(); - } - - _disposed = true; + return; } + + foreach (nint page in _pages) + { + Marshal.FreeHGlobal(page); + } + + _disposed = true; } /// diff --git a/src/Ryujinx.Cpu/Jit/JitCpuContext.cs b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs index 9b2bf7015..e967b799a 100644 --- a/src/Ryujinx.Cpu/Jit/JitCpuContext.cs +++ b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs @@ -9,13 +9,15 @@ namespace Ryujinx.Cpu.Jit { private readonly ITickSource _tickSource; private readonly Translator _translator; - private readonly AddressTable _functionTable; public JitCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit) { _tickSource = tickSource; - _functionTable = AddressTable.CreateForArm(for64Bit, memory.Type); - _translator = new Translator(new JitMemoryAllocator(forJit: true), memory, _functionTable); + + bool sparse = memory.Type is not MemoryManagerType.SoftwareMmu and not MemoryManagerType.SoftwarePageTable; + IAddressTable functionTable = sparse ? SparseAddressTable.CreateForArm(for64Bit) : AddressTable.CreateForArm(for64Bit); + + _translator = new Translator(new JitMemoryAllocator(forJit: true), memory, functionTable); if (memory.Type.IsHostMappedOrTracked) { @@ -57,7 +59,7 @@ namespace Ryujinx.Cpu.Jit /// public void PrepareCodeRange(ulong address, ulong size) { - _functionTable.SignalCodeRange(address, size); + _translator.FunctionTable.SignalCodeRange(address, size); _translator.PrepareCodeRange(address, size); } diff --git a/src/Ryujinx.Cpu/LightningJit/AarchCompiler.cs b/src/Ryujinx.Cpu/LightningJit/AarchCompiler.cs index 68dff624c..82a492b8f 100644 --- a/src/Ryujinx.Cpu/LightningJit/AarchCompiler.cs +++ b/src/Ryujinx.Cpu/LightningJit/AarchCompiler.cs @@ -13,7 +13,7 @@ namespace Ryujinx.Cpu.LightningJit CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, - AddressTable funcTable, + IAddressTable funcTable, nint dispatchStubPtr, ExecutionMode executionMode, Architecture targetArch) diff --git a/src/Ryujinx.Cpu/LightningJit/Arm32/A32Compiler.cs b/src/Ryujinx.Cpu/LightningJit/Arm32/A32Compiler.cs index 0fe42b923..f7283741d 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm32/A32Compiler.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm32/A32Compiler.cs @@ -12,7 +12,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32 CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, - AddressTable funcTable, + IAddressTable funcTable, nint dispatchStubPtr, bool isThumb, Architecture targetArch) diff --git a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/Compiler.cs b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/Compiler.cs index ff11cbd41..c7d025927 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/Compiler.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/Compiler.cs @@ -22,7 +22,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 public readonly RegisterAllocator RegisterAllocator; public readonly MemoryManagerType MemoryManagerType; public readonly TailMerger TailMerger; - public readonly AddressTable FuncTable; + public readonly IAddressTable FuncTable; public readonly nint DispatchStubPointer; private readonly RegisterSaveRestore _registerSaveRestore; @@ -33,7 +33,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 RegisterAllocator registerAllocator, MemoryManagerType mmType, TailMerger tailMerger, - AddressTable funcTable, + IAddressTable funcTable, RegisterSaveRestore registerSaveRestore, nint dispatchStubPointer, nint pageTablePointer) @@ -225,7 +225,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 } } - public static CompiledFunction Compile(CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, AddressTable funcTable, nint dispatchStubPtr, bool isThumb) + public static CompiledFunction Compile(CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, IAddressTable funcTable, nint dispatchStubPtr, bool isThumb) { MultiBlock multiBlock = Decoder.DecodeMulti(cpuPreset, memoryManager, address, isThumb); diff --git a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitFlow.cs b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitFlow.cs index 943bc6897..ff1c2a942 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitFlow.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitFlow.cs @@ -3,6 +3,7 @@ using Ryujinx.Cpu.LightningJit.CodeGen; using Ryujinx.Cpu.LightningJit.CodeGen.Arm64; using System; using System.Diagnostics; +using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; @@ -132,7 +133,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 RegisterAllocator regAlloc, TailMerger tailMerger, Action writeEpilogue, - AddressTable funcTable, + IAddressTable funcTable, nint funcPtr, int spillBaseOffset, uint nextAddress, @@ -143,7 +144,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 int tempGuestAddress = -1; bool inlineLookup = guestAddress.Kind != OperandKind.Constant && - funcTable is { Sparse: true }; + funcTable.TableType == AddressTableType.Sparse; if (guestAddress.Kind == OperandKind.Constant) { @@ -189,7 +190,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 } else if (inlineLookup) { - // Inline table lookup. Only enabled when the sparse function table is enabled with 2 levels. + // Inline table lookup. Only enabled when the sparse function table is enabled with 1 level. Operand indexReg = Register(NextFreeRegister(tempRegister + 1, tempGuestAddress)); @@ -203,18 +204,15 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 // Index into the table. asm.Mov(rn, tableBase); - for (int i = 0; i < funcTable.Levels.Length; i++) - { - AddressTableLevel level = funcTable.Levels[i]; - asm.Ubfx(indexReg, guestAddress, level.Index, level.Length); - asm.Lsl(indexReg, indexReg, Const(3)); + AddressTableLevel level = funcTable.Levels.Last(); + asm.Ubfx(indexReg, guestAddress, level.Index, level.Length); + asm.Lsl(indexReg, indexReg, Const(3)); - // Index into the page. - asm.Add(rn, rn, indexReg); + // Index into the page. + asm.Add(rn, rn, indexReg); - // Load the page address. - asm.LdrRiUn(rn, rn, 0); - } + // Load the page address. + asm.LdrRiUn(rn, rn, 0); if (tempGuestAddress != -1) { diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/A64Compiler.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/A64Compiler.cs index 44de4cd0d..8a253c544 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm64/A64Compiler.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm64/A64Compiler.cs @@ -12,7 +12,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64 CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, - AddressTable funcTable, + IAddressTable funcTable, nint dispatchStubPtr, Architecture targetArch) { diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs index ac389f4ce..e8a75f294 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs @@ -18,7 +18,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 public readonly CodeWriter Writer; public readonly RegisterAllocator RegisterAllocator; public readonly TailMerger TailMerger; - public readonly AddressTable FuncTable; + public readonly IAddressTable FuncTable; public readonly nint DispatchStubPointer; private readonly MultiBlock _multiBlock; @@ -31,7 +31,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 TailMerger tailMerger, RegisterSaveRestore registerSaveRestore, MultiBlock multiBlock, - AddressTable funcTable, + IAddressTable funcTable, nint dispatchStubPointer, nint pageTablePointer) { @@ -303,7 +303,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 } } - public static CompiledFunction Compile(CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, AddressTable funcTable, nint dispatchStubPtr) + public static CompiledFunction Compile(CpuPreset cpuPreset, IMemoryManager memoryManager, ulong address, IAddressTable funcTable, nint dispatchStubPtr) { MultiBlock multiBlock = Decoder.DecodeMulti(cpuPreset, memoryManager, address); diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitSystem.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitSystem.cs index 1bd69bc4b..3c1f1a246 100644 --- a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitSystem.cs +++ b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitSystem.cs @@ -3,6 +3,7 @@ using Ryujinx.Cpu.LightningJit.CodeGen; using Ryujinx.Cpu.LightningJit.CodeGen.Arm64; using System; using System.Diagnostics; +using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -214,7 +215,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 RegisterAllocator regAlloc, TailMerger tailMerger, Action writeEpilogue, - AddressTable funcTable, + IAddressTable funcTable, nint dispatchStubPtr, InstName name, ulong pc, @@ -298,7 +299,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 RegisterAllocator regAlloc, TailMerger tailMerger, Action writeEpilogue, - AddressTable funcTable, + IAddressTable funcTable, nint funcPtr, int spillBaseOffset, ulong pc, @@ -309,7 +310,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 int tempGuestAddress = -1; bool inlineLookup = guestAddress.Kind != OperandKind.Constant && - funcTable is { Sparse: true }; + funcTable.TableType == AddressTableType.Sparse; if (guestAddress.Kind == OperandKind.Constant) { @@ -355,7 +356,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 } else if (inlineLookup) { - // Inline table lookup. Only enabled when the sparse function table is enabled with 2 levels. + // Inline table lookup. Only enabled when the sparse function table is enabled with 1 level. Operand indexReg = Register(NextFreeRegister(tempRegister + 1, tempGuestAddress)); @@ -369,18 +370,15 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64 // Index into the table. asm.Mov(rn, tableBase); - for (int i = 0; i < funcTable.Levels.Length; i++) - { - AddressTableLevel level = funcTable.Levels[i]; - asm.Ubfx(indexReg, guestAddress, level.Index, level.Length); - asm.Lsl(indexReg, indexReg, Const(3)); + AddressTableLevel level = funcTable.Levels.Last(); + asm.Ubfx(indexReg, guestAddress, level.Index, level.Length); + asm.Lsl(indexReg, indexReg, Const(3)); - // Index into the page. - asm.Add(rn, rn, indexReg); + // Index into the page. + asm.Add(rn, rn, indexReg); - // Load the page address. - asm.LdrRiUn(rn, rn, 0); - } + // Load the page address. + asm.LdrRiUn(rn, rn, 0); if (tempGuestAddress != -1) { diff --git a/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs b/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs index 0f47ffb15..185a296e2 100644 --- a/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs +++ b/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs @@ -9,15 +9,15 @@ namespace Ryujinx.Cpu.LightningJit { private readonly ITickSource _tickSource; private readonly Translator _translator; - private readonly AddressTable _functionTable; public LightningJitCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit) { _tickSource = tickSource; - _functionTable = AddressTable.CreateForArm(for64Bit, memory.Type); - - _translator = new Translator(memory, _functionTable); + bool sparse = memory.Type is not MemoryManagerType.SoftwareMmu and not MemoryManagerType.SoftwarePageTable; + IAddressTable functionTable = sparse ? SparseAddressTable.CreateForArm(for64Bit) : AddressTable.CreateForArm(for64Bit); + + _translator = new Translator(memory, functionTable); memory.UnmapEvent += UnmapHandler; } @@ -54,7 +54,7 @@ namespace Ryujinx.Cpu.LightningJit /// public void PrepareCodeRange(ulong address, ulong size) { - _functionTable.SignalCodeRange(address, size); + _translator.FunctionTable.SignalCodeRange(address, size); } public void Dispose() diff --git a/src/Ryujinx.Cpu/LightningJit/Translator.cs b/src/Ryujinx.Cpu/LightningJit/Translator.cs index db77cf058..f3b80c972 100644 --- a/src/Ryujinx.Cpu/LightningJit/Translator.cs +++ b/src/Ryujinx.Cpu/LightningJit/Translator.cs @@ -24,11 +24,11 @@ namespace Ryujinx.Cpu.LightningJit private bool _disposed; internal TranslatorCache Functions { get; } - internal AddressTable FunctionTable { get; } + internal IAddressTable FunctionTable { get; } internal TranslatorStubs Stubs { get; } internal IMemoryManager Memory { get; } - public Translator(IMemoryManager memory, AddressTable functionTable) + public Translator(IMemoryManager memory, IAddressTable functionTable) { Memory = memory; @@ -45,6 +45,7 @@ namespace Ryujinx.Cpu.LightningJit Functions = new TranslatorCache(); FunctionTable = functionTable; + Stubs = new TranslatorStubs(_jitCache, FunctionTable, _noWxCache); FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub; diff --git a/src/Ryujinx.Cpu/SparseAddressTable.cs b/src/Ryujinx.Cpu/SparseAddressTable.cs new file mode 100644 index 000000000..cf915cb03 --- /dev/null +++ b/src/Ryujinx.Cpu/SparseAddressTable.cs @@ -0,0 +1,247 @@ +using ARMeilleure.Common; +using Ryujinx.Common; +using Ryujinx.Cpu.Signal; +using Ryujinx.Memory; +using System; +using System.Linq; +using System.Runtime.InteropServices; +using static Ryujinx.Cpu.MemoryEhMeilleure; + +namespace Ryujinx.Cpu +{ + /// + /// Represents a table of guest address to a value. + /// + /// Type of the value + public unsafe class SparseAddressTable : IAddressTable where TEntry : unmanaged + { + /// + /// A sparsely mapped block of memory with a signal handler to map pages as they're accessed. + /// + private readonly struct TableSparseBlock : IDisposable + { + public readonly SparseMemoryBlock Block; + private readonly TrackingEventDelegate _trackingEvent; + + public TableSparseBlock(ulong size, Action ensureMapped, PageInitDelegate pageInit, MemoryBlock fill) + { + SparseMemoryBlock block = new(size, pageInit, fill); + + _trackingEvent = (address, _, _) => + { + ulong pointer = (ulong)block.Block.Pointer + address; + ensureMapped((nint)pointer); + return pointer; + }; + + bool added = NativeSignalHandler.AddTrackedRegion( + (nuint)block.Block.Pointer, + (nuint)(block.Block.Pointer + (nint)block.Block.Size), + Marshal.GetFunctionPointerForDelegate(_trackingEvent)); + + if (!added) + { + throw new InvalidOperationException("Number of allowed tracked regions exceeded."); + } + + Block = block; + } + + public void Dispose() + { + NativeSignalHandler.RemoveTrackedRegion((nuint)Block.Block.Pointer); + + Block.Dispose(); + } + } + + private bool _disposed; + private TEntry* _table; + private TEntry _fill; + + private readonly TableSparseBlock _block; + private readonly MemoryBlock _fillBlock; + + /// + public AddressTableType TableType => AddressTableType.Sparse; + + /// + public ulong Mask { get; } + + /// + public AddressTableLevel[] Levels { get; } + + /// + public TEntry Fill + { + get + { + return _fill; + } + set + { + UpdateFill(value); + } + } + + /// + public nint Base + { + get + { + ObjectDisposedException.ThrowIf(_disposed, this); + + return (nint)_table; + } + } + + /// + /// Constructs a new instance of the class with the specified list of + /// . + /// + /// Levels for the address table + /// is null + /// Length of is less than 2 + public SparseAddressTable(AddressTableLevel[] levels) + { + ArgumentNullException.ThrowIfNull(levels); + + Levels = levels; + Mask = 0; + + foreach (AddressTableLevel level in Levels) + { + Mask |= level.Mask; + } + + _fillBlock = new MemoryBlock(MemoryBlock.GetPageSize() << 10, MemoryAllocationFlags.Mirrorable); + + // We need to use the full size, as some games dynamically expand the code range (e.g. SSBU) + // Limiting the size to only the requested code range will cause crashes + // This should not be an issue tho, as the SparseBlock dynamically allocates memory as needed, and + // Falls back to the fill block in case guest code tries to call an unmapped function. + ulong bottomLevelSize = (1ul << Levels.Last().Length) * (ulong)sizeof(TEntry); + + _block = new TableSparseBlock(bottomLevelSize, EnsureMapped, InitLeafPage, _fillBlock); + + _table = (TEntry*)_block.Block.Block.Pointer; + } + + /// + /// Create an instance for an ARM function table. + /// Selects the best table structure for A32/A64, taking into account the selected memory manager type. + /// + /// True if the guest is A64, false otherwise + /// An for ARM function lookup + public static SparseAddressTable CreateForArm(bool for64Bits) + { + return new SparseAddressTable(AddressTablePresets.GetArmPreset(for64Bits, true)); + } + + /// + /// Update the fill value for the bottom level of the table. + /// + /// New fill value + private void UpdateFill(TEntry fillValue) + { + if (_fillBlock != null) + { + Span span = _fillBlock.GetSpan(0, (int)_fillBlock.Size); + MemoryMarshal.Cast(span).Fill(fillValue); + } + + _fill = fillValue; + } + + /// + /// Signal that the given code range exists. + /// + /// + /// + public void SignalCodeRange(ulong address, ulong size) + { + + } + + /// + public bool IsValid(ulong address) + { + return (address & ~Mask) == 0; + } + + /// + public ref TEntry GetValue(ulong address) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (!IsValid(address)) + { + throw new ArgumentException($"Address 0x{address:X} is not mapped onto the table.", nameof(address)); + } + + long index = Levels.Last().GetValue(address); + + EnsureMapped((nint)(_table + index)); + + return ref _table[index]; + } + + /// + /// Ensure the given pointer is mapped in any overlapping sparse reservations. + /// + /// Pointer to be mapped + private void EnsureMapped(nint ptr) + { + SparseMemoryBlock sparse = _block.Block; + + if (ptr >= sparse.Block.Pointer && ptr < sparse.Block.Pointer + (nint)sparse.Block.Size) + { + sparse.EnsureMapped((ulong)(ptr - sparse.Block.Pointer)); + } + } + + /// + /// Initialize a leaf page with the fill value. + /// + /// Page to initialize + private void InitLeafPage(Span page) + { + MemoryMarshal.Cast(page).Fill(_fill); + } + + /// + /// Releases all resources used by the instance. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases all unmanaged and optionally managed resources used by the + /// instance. + /// + /// to dispose managed resources also; otherwise just unmanaged resouces + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + _block.Dispose(); + _fillBlock.Dispose(); + + _disposed = true; + } + + /// + /// Frees resources used by the instance. + /// + ~SparseAddressTable() + { + Dispose(false); + } + } +} diff --git a/src/Ryujinx.Memory/SparseMemoryBlock.cs b/src/Ryujinx.Memory/SparseMemoryBlock.cs index 7e46370d0..bed046b83 100644 --- a/src/Ryujinx.Memory/SparseMemoryBlock.cs +++ b/src/Ryujinx.Memory/SparseMemoryBlock.cs @@ -21,6 +21,7 @@ namespace Ryujinx.Memory private ulong _mappedBlockUsage; private readonly ulong[] _mappedPageBitmap; + private readonly MemoryBlock _fill; public MemoryBlock Block => _reservedBlock; @@ -34,23 +35,24 @@ namespace Ryujinx.Memory int pages = (int)BitUtils.DivRoundUp(size, _pageSize); int bitmapEntries = BitUtils.DivRoundUp(pages, 64); _mappedPageBitmap = new ulong[bitmapEntries]; + _fill = fill; - if (fill != null) + if (_fill is not null) { + if (_fill.Size % _pageSize !=0) + { + throw new ArgumentException("Fill memory block should be page sized.", nameof(_fill)); + } + // Fill the block with mappings from the fill block. - if (fill.Size % _pageSize != 0) - { - throw new ArgumentException("Fill memory block should be page aligned.", nameof(fill)); - } - - int repeats = (int)BitUtils.DivRoundUp(size, fill.Size); + int repeats = (int)BitUtils.DivRoundUp(size, _fill.Size); ulong offset = 0; for (int i = 0; i < repeats; i++) { - _reservedBlock.MapView(fill, 0, offset, Math.Min(fill.Size, size - offset)); - offset += fill.Size; + _reservedBlock.MapView(_fill, 0, offset, Math.Min(_fill.Size, size - offset)); + offset += _fill.Size; } } @@ -75,6 +77,12 @@ namespace Ryujinx.Memory } _pageInit(block.GetSpan(_mappedBlockUsage, (int)_pageSize)); + + if (_fill is not null) + { + _reservedBlock.UnmapView(_fill, pageOffset, _pageSize); + } + _reservedBlock.MapView(block, _mappedBlockUsage, pageOffset, _pageSize); _mappedBlockUsage += _pageSize; diff --git a/src/Ryujinx.Tests/Cpu/CpuContext.cs b/src/Ryujinx.Tests/Cpu/CpuContext.cs index 81e8ba8c9..d703f8035 100644 --- a/src/Ryujinx.Tests/Cpu/CpuContext.cs +++ b/src/Ryujinx.Tests/Cpu/CpuContext.cs @@ -13,7 +13,10 @@ namespace Ryujinx.Tests.Cpu public CpuContext(IMemoryManager memory, bool for64Bit) { - _translator = new Translator(new JitMemoryAllocator(), memory, AddressTable.CreateForArm(for64Bit, memory.Type)); + bool sparse = memory.Type is not MemoryManagerType.SoftwareMmu and not MemoryManagerType.SoftwarePageTable; + IAddressTable functionTable = sparse ? SparseAddressTable.CreateForArm(for64Bit) : AddressTable.CreateForArm(for64Bit); + + _translator = new Translator(new JitMemoryAllocator(), memory, functionTable); memory.UnmapEvent += UnmapHandler; } diff --git a/src/Ryujinx.Tests/Cpu/EnvironmentTests.cs b/src/Ryujinx.Tests/Cpu/EnvironmentTests.cs index a6014ed56..f5d15b690 100644 --- a/src/Ryujinx.Tests/Cpu/EnvironmentTests.cs +++ b/src/Ryujinx.Tests/Cpu/EnvironmentTests.cs @@ -1,7 +1,7 @@ using ARMeilleure.Common; -using ARMeilleure.Memory; using ARMeilleure.Translation; using NUnit.Framework; +using Ryujinx.Cpu; using Ryujinx.Cpu.Jit; using Ryujinx.Tests.Memory; using System; @@ -20,7 +20,7 @@ namespace Ryujinx.Tests.Cpu _translator ??= new Translator( new JitMemoryAllocator(), new MockMemoryManager(), - AddressTable.CreateForArm(true, MemoryManagerType.SoftwarePageTable)); + AddressTable.CreateForArm(true)); } [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] diff --git a/src/Ryujinx.Tests/Memory/PartialUnmaps.cs b/src/Ryujinx.Tests/Memory/PartialUnmaps.cs index f80a7519f..279a0dca3 100644 --- a/src/Ryujinx.Tests/Memory/PartialUnmaps.cs +++ b/src/Ryujinx.Tests/Memory/PartialUnmaps.cs @@ -60,7 +60,7 @@ namespace Ryujinx.Tests.Memory _translator ??= new Translator( new JitMemoryAllocator(), new MockMemoryManager(), - AddressTable.CreateForArm(true, MemoryManagerType.SoftwarePageTable)); + AddressTable.CreateForArm(true)); NativeSignalHandler.InitializeSignalHandler(); }