diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs new file mode 100644 index 00000000000..b2827495ba5 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/KeccakCacheTests.cs @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers.Binary; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using NUnit.Framework; + +namespace Nethermind.Core.Test +{ + [TestFixture] + public class KeccakCacheTests + { + [Test] + public void Multiple() + { + const int spins = 10; + + var random = new Random(13); + var bytes = new byte[31]; // misaligned length + random.NextBytes(bytes); + + ValueHash256 expected = ValueKeccak.Compute(bytes); + + for (int i = 0; i < spins; i++) + { + ValueHash256 actual = KeccakCache.Compute(bytes); + actual.Equals(expected).Should().BeTrue(); + } + } + + [Test] + public void Empty() + { + ReadOnlySpan span = ReadOnlySpan.Empty; + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + + [Test] + public void Very_long() + { + ReadOnlySpan span = new byte[192]; + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + + private string[] GetBucketCollisions() + { + var random = new Random(13); + Span span = stackalloc byte[32]; + string[] collisions = new string[4]; + + random.NextBytes(span); + var bucket = KeccakCache.GetBucket(span); + + Console.WriteLine(span.ToHexString()); + + collisions[0] = span.ToHexString(); + var found = 1; + + ulong iterations = 0; + while (found < 4) + { + random.NextBytes(span); + if (KeccakCache.GetBucket(span) == bucket) + { + collisions[found] = span.ToHexString(); + Console.WriteLine(span.ToHexString()); + found++; + } + iterations++; + } + + Console.WriteLine($"{iterations} iterations to find"); + return collisions; + } + + [Test] + public void Collision() + { + var colliding = GetBucketCollisions(); + + var collisions = colliding.Length; + var array = colliding.Select(c => Bytes.FromHexString(c)).ToArray(); + var values = array.Select(a => ValueKeccak.Compute(a)).ToArray(); + + var bucket = KeccakCache.GetBucket(array[0]); + + for (int i = 1; i < collisions; i++) + { + var input = array[i]; + bucket.Should().Be(KeccakCache.GetBucket(input)); + KeccakCache.Compute(input).Should().Be(values[i]); + } + + Parallel.ForEach(array, (a, state, index) => + { + ValueHash256 v = values[index]; + + for (int i = 0; i < 100_000; i++) + { + KeccakCache.Compute(a).Should().Be(v); + } + }); + } + + [Test] + public void Spin_through_all() + { + Span span = stackalloc byte[4]; + for (int i = 0; i < KeccakCache.Count; i++) + { + BinaryPrimitives.WriteInt32LittleEndian(span, i); + KeccakCache.Compute(span).Should().Be(ValueKeccak.Compute(span)); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index ff233e8dd50..f09c9e5b76d 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -21,11 +21,6 @@ namespace Nethermind.Core [TypeConverter(typeof(AddressTypeConverter))] public class Address : IEquatable
, IComparable
{ - // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on - // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade - // the performance of the network as a whole. - private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); - public const int Size = 20; private const int HexCharsCount = 2 * Size; // 5a4eab120fb44eb6684e5e32785702ff45ea344d private const int PrefixedHexCharsCount = 2 + HexCharsCount; // 0x5a4eab120fb44eb6684e5e32785702ff45ea344d @@ -201,14 +196,7 @@ public override bool Equals(object? obj) return obj.GetType() == GetType() && Equals((Address)obj); } - public override int GetHashCode() - { - uint hash = s_instanceRandom; - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(Bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(ulong)))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Bytes), sizeof(long) * 2))); - return (int)hash; - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public static bool operator ==(Address? a, Address? b) { @@ -243,7 +231,7 @@ public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destina destinationType == typeof(string) || base.CanConvertTo(context, destinationType); } - public Hash256 ToAccountPath => Keccak.Compute(Bytes); + public Hash256 ToAccountPath => KeccakCache.Compute(Bytes); [SkipLocalsInit] public ValueHash256 ToHash() diff --git a/src/Nethermind/Nethermind.Core/Bloom.cs b/src/Nethermind/Nethermind.Core/Bloom.cs index f980e1b4a0e..05d3f4e5d9f 100644 --- a/src/Nethermind/Nethermind.Core/Bloom.cs +++ b/src/Nethermind/Nethermind.Core/Bloom.cs @@ -114,10 +114,7 @@ public override bool Equals(object? obj) return Equals((Bloom)obj); } - public override int GetHashCode() - { - return Bytes.GetSimplifiedHashCode(); - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public void Add(LogEntry[] logEntries, Bloom? blockBloom = null) { @@ -278,10 +275,7 @@ public override readonly bool Equals(object? obj) return Equals((Bloom)obj); } - public override readonly int GetHashCode() - { - return Core.Extensions.Bytes.GetSimplifiedHashCode(Bytes); - } + public override readonly int GetHashCode() => Bytes.FastHash(); public readonly bool Matches(LogEntry logEntry) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs index 0f223c05197..808a262d22b 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs @@ -66,19 +66,9 @@ public ValueHash256(ReadOnlySpan bytes) public bool Equals(Hash256? other) => _bytes.Equals(other?.ValueHash256._bytes ?? default); - public override int GetHashCode() - { - return GetChainedHashCode(s_instanceRandom); - } + public override int GetHashCode() => GetChainedHashCode(s_instanceRandom); - public int GetChainedHashCode(uint previousHash) - { - uint hash = BitOperations.Crc32C(previousHash, Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 1)); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 2)); - hash = BitOperations.Crc32C(hash, Unsafe.Add(ref Unsafe.As, ulong>(ref Unsafe.AsRef(in _bytes)), 3)); - return (int)hash; - } + public int GetChainedHashCode(uint previousHash) => Bytes.FastHash() ^ (int)previousHash; public int CompareTo(ValueHash256 other) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs new file mode 100644 index 00000000000..ca34856205e --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Threading; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Crypto; + +/// +/// This is a minimalistic one-way set associative cache for Keccak values. +/// +/// It allocates only 12MB of memory to store 128k of entries. +/// No misaligned reads. Everything is aligned to both cache lines as well as to boundaries so no torn reads. +/// Requires a single CAS to lock and to unlock. +/// On a lock failure, it just moves on with execution. +/// When a potential hit happens, the value of the result and the value of the key are copied on the stack to release the lock ASAP. +/// +public static unsafe class KeccakCache +{ + /// + /// Count is defined as a +1 over bucket mask. In the future, just change the mask as the main parameter. + /// + public const int Count = BucketMask + 1; + + private const int BucketMask = 0x0001_FFFF; + private const uint HashMask = unchecked((uint)~BucketMask); + private const uint LockMarker = 0x0000_8000; + + private const int InputLengthOfKeccak = ValueHash256.MemorySize; + private const int InputLengthOfAddress = Address.Size; + + private static readonly Entry* Memory; + + static KeccakCache() + { + const UIntPtr size = Count * Entry.Size; + + // Aligned, so that no torn reads if fields of Entry are properly aligned. + Memory = (Entry*)NativeMemory.AlignedAlloc(size, BitOperations.RoundUpToPowerOf2(Entry.Size)); + NativeMemory.Clear(Memory, size); + GC.AddMemoryPressure((long)size); + } + + [SkipLocalsInit] + public static ValueHash256 Compute(ReadOnlySpan input) + { + ComputeTo(input, out ValueHash256 keccak256); + return keccak256; + } + + [SkipLocalsInit] + public static void ComputeTo(ReadOnlySpan input, out ValueHash256 keccak256) + { + // Special cases jump forward as unpredicted + if (input.Length == 0 || input.Length > Entry.MaxPayloadLength) + { + goto Uncommon; + } + + int hashCode = input.FastHash(); + uint index = (uint)hashCode & BucketMask; + + Debug.Assert(index < Count); + + ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef(Memory), index); + + // Half the hash his encoded in the bucket so we only need half of it and can use other half for length. + // This allows to create a combined value that represents a part of the hash, the input's length and the lock marker. + uint combined = (HashMask & (uint)hashCode) | (uint)input.Length; + + // Compare with volatile read and then try to lock with CAS + if (Volatile.Read(ref e.Combined) == combined && + Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, combined) == combined) + { + // Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal. + + // Take local copy of the payload and hash, to release the lock as soon as possible and make a key comparison without holding it. + // Local copy of 8+16+64 payload bytes. + Payload copy = e.Value; + // Copy Keccak256 directly to the local return variable, since we will overwrite if no match anyway. + keccak256 = e.Keccak256; + + // Release the lock, by setting back to the combined value. + Volatile.Write(ref e.Combined, combined); + + // Lengths are equal, the input length can be used without any additional operation. + if (input.Length == InputLengthOfKeccak) + { + // Hashing UInt256 or Hash256 which is Vector256 + if (Unsafe.As>(ref copy.Aligned32) == + Unsafe.As>(ref MemoryMarshal.GetReference(input))) + { + // Current keccak256 is correct hash. + return; + } + } + else if (input.Length == InputLengthOfAddress) + { + // Hashing Address + ref byte bytes1 = ref MemoryMarshal.GetReference(input); + // 20 bytes which is uint+Vector128 + if (Unsafe.As(ref copy.Start) == Unsafe.As(ref bytes1) && + Unsafe.As>(ref copy.Aligned32) == + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint)))) + { + // Current keccak256 is correct hash. + return; + } + } + else if (MemoryMarshal.CreateReadOnlySpan(ref copy.Start, input.Length).SequenceEqual(input)) + { + // Non 32 byte or 20 byte input; call SequenceEqual. + // Current keccak256 is correct hash. + return; + } + } + + keccak256 = ValueKeccak.Compute(input); + + uint existing = Volatile.Read(ref e.Combined); + + // Try to set to the combined locked state, if not already locked. + if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing) + { + e.Keccak256 = keccak256; + + // Fast copy for 2 common sizes + if (input.Length == InputLengthOfKeccak) + { + // UInt256 or Hash256 which is Vector256 + Unsafe.As>(ref e.Value.Aligned32) = + Unsafe.As>(ref MemoryMarshal.GetReference(input)); + } + else if (input.Length == InputLengthOfAddress) + { + // Address + ref byte bytes1 = ref MemoryMarshal.GetReference(input); + // 20 bytes which is uint+Vector128 + Unsafe.As(ref e.Value.Start) = Unsafe.As(ref bytes1); + Unsafe.As>(ref e.Value.Aligned32) = + Unsafe.As>(ref Unsafe.Add(ref bytes1, sizeof(uint))); + } + else + { + // Non 32 byte or 20 byte input; call CopyTo + input.CopyTo(MemoryMarshal.CreateSpan(ref e.Value.Start, input.Length)); + } + + // Release the lock, by setting back to the combined value. + Volatile.Write(ref e.Combined, combined); + } + + return; + + Uncommon: + if (input.Length == 0) + { + keccak256 = ValueKeccak.OfAnEmptyString; + } + else + { + keccak256 = ValueKeccak.Compute(input); + } + } + + /// + /// Gets the bucket for tests. + /// + public static uint GetBucket(ReadOnlySpan input) => (uint)input.FastHash() & BucketMask; + + /// + /// An entry to cache keccak + /// + [StructLayout(LayoutKind.Explicit, Size = Size)] + private struct Entry + { + /// + /// The size will make it 1.5 CPU cache entry or 0.75 which may result in some collisions. + /// Still, it's better to save these 32 bytes per entry and have a bigger cache. + /// + public const int Size = 96; + + private const int PayloadStart = sizeof(uint); + private const int ValueStart = Size - ValueHash256.MemorySize; + public const int MaxPayloadLength = ValueStart - PayloadStart; + + /// + /// Represents a combined value for: hash, length and a potential . + /// + [FieldOffset(0)] public uint Combined; + + /// + /// The actual value + /// + [FieldOffset(PayloadStart)] public Payload Value; + + /// + /// The Keccak of the Value + /// + [FieldOffset(ValueStart)] public ValueHash256 Keccak256; + } + + [StructLayout(LayoutKind.Explicit, Size = Entry.MaxPayloadLength)] + private struct Payload + { + private const int AlignedStart = Entry.MaxPayloadLength - 32; + + [FieldOffset(0)] public byte Start; + [FieldOffset(AlignedStart)] public byte Aligned32; + } +} diff --git a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs index c8ab15b9fdd..19db37e7914 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs @@ -16,11 +16,6 @@ namespace Nethermind.Core.Crypto [JsonConverter(typeof(PublicKeyConverter))] public class PublicKey : IEquatable { - // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on - // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade - // the performance of the network as a whole. - private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); - public const int PrefixedLengthInBytes = 65; public const int LengthInBytes = 64; private Address? _address; @@ -112,19 +107,7 @@ public override int GetHashCode() return _hashCode; } - private static int GetHashCode(byte[] bytes) - { - uint hash = s_instanceRandom; - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(bytes))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long)))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 2))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 3))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 4))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 5))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 6))); - hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 7))); - return (int)hash; - } + private static int GetHashCode(byte[] bytes) => new ReadOnlySpan(bytes).FastHash(); public override string ToString() { diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 4e547fcaee9..9957bd1e912 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -37,10 +37,7 @@ public override bool Equals(byte[]? x, byte[]? y) return AreEqual(x, y); } - public override int GetHashCode(byte[] obj) - { - return obj.GetSimplifiedHashCode(); - } + public override int GetHashCode(byte[] obj) => new ReadOnlySpan(obj).FastHash(); } private class NullableBytesEqualityComparer : EqualityComparer @@ -50,17 +47,14 @@ public override bool Equals(byte[]? x, byte[]? y) return AreEqual(x, y); } - public override int GetHashCode(byte[]? obj) - { - return obj?.GetSimplifiedHashCode() ?? 0; - } + public override int GetHashCode(byte[]? obj) => new ReadOnlySpan(obj).FastHash(); } private class SpanBytesEqualityComparer : ISpanEqualityComparer { public bool Equals(ReadOnlySpan x, ReadOnlySpan y) => AreEqual(x, y); - public int GetHashCode(ReadOnlySpan obj) => GetSimplifiedHashCode(obj); + public int GetHashCode(ReadOnlySpan obj) => obj.FastHash(); } public class BytesComparer : Comparer @@ -1130,32 +1124,6 @@ private static byte[] FromHexString(ReadOnlySpan hexString) return isSuccess ? result : throw new FormatException("Incorrect hex string"); } - [SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")] - public static int GetSimplifiedHashCode(this byte[] bytes) - { - const int fnvPrime = 0x01000193; - - if (bytes.Length == 0) - { - return 0; - } - - return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53); - } - - [SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")] - public static int GetSimplifiedHashCode(this ReadOnlySpan bytes) - { - const int fnvPrime = 0x01000193; - - if (bytes.Length == 0) - { - return 0; - } - - return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53); - } - public static void ChangeEndianness8(Span bytes) { if (bytes.Length % 16 != 0) diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 99b4b7645fe..1d6f49d57ee 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Nethermind.Core.Collections; @@ -13,6 +14,11 @@ namespace Nethermind.Core.Extensions { public static class SpanExtensions { + // Ensure that hashes are different for every run of the node and every node, so if are any hash collisions on + // one node they will not be the same on another node or across a restart so hash collision cannot be used to degrade + // the performance of the network as a whole. + private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue); + public static string ToHexString(this in ReadOnlySpan span, bool withZeroX) { return ToHexString(span, withZeroX, false, false); @@ -166,5 +172,80 @@ public static ArrayPoolList ToPooledList(this in ReadOnlySpan span) newList.AddRange(span); return newList; } + + [SkipLocalsInit] + public static int FastHash(this ReadOnlySpan input) + { + // Very fast hardware accelerated non-cryptographic hash function + var length = input.Length; + if (length == 0) return 0; + + ref var b = ref MemoryMarshal.GetReference(input); + uint hash = s_instanceRandom + (uint)length; + if (length < sizeof(long)) + { + goto Short; + } + + // Start with instance random, length and first ulong as seed + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + + // Calculate misalignment and move by it if needed. + // If no misalignment, advance by the size of ulong + uint misaligned = (uint)length & 7; + if (misaligned != 0) + { + // Align by moving by the misaligned count + b = ref Unsafe.Add(ref b, misaligned); + length -= (int)misaligned; + } + else + { + // Already Crc'd first ulong so skip it + b = ref Unsafe.Add(ref b, sizeof(ulong)); + length -= sizeof(ulong); + } + + // Length is fully aligned here and b is set in place + while (length >= sizeof(ulong) * 3) + { + // Crc32C is 3 cycle latency, 1 cycle throughput + // So we us same initial 3 times to not create a dependency chain + uint hash0 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + uint hash1 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong)))); + uint hash2 = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref Unsafe.Add(ref b, sizeof(ulong) * 2))); + b = ref Unsafe.Add(ref b, sizeof(ulong) * 3); + length -= sizeof(ulong) * 3; + // Combine the 3 hashes; performing the shift on first crc to calculate + hash = BitOperations.Crc32C(hash1, ((ulong)hash0 << (sizeof(uint) * 8)) | hash2); + } + + while (length > 0) + { + hash = BitOperations.Crc32C(hash, Unsafe.ReadUnaligned(ref b)); + b = ref Unsafe.Add(ref b, sizeof(ulong)); + length -= sizeof(ulong); + } + + return (int)hash; + Short: + ulong data = 0; + if ((length & sizeof(byte)) != 0) + { + data = b; + b = ref Unsafe.Add(ref b, sizeof(byte)); + } + if ((length & sizeof(ushort)) != 0) + { + data = (data << (sizeof(ushort) * 8)) | Unsafe.ReadUnaligned(ref b); + b = ref Unsafe.Add(ref b, sizeof(ushort)); + } + if ((length & sizeof(uint)) != 0) + { + data = (data << (sizeof(uint) * 8)) | Unsafe.ReadUnaligned(ref b); + } + + return (int)BitOperations.Crc32C(hash, data); + } } } diff --git a/src/Nethermind/Nethermind.Core/StorageCell.cs b/src/Nethermind/Nethermind.Core/StorageCell.cs index 62e73e89156..a35bc2eda97 100644 --- a/src/Nethermind/Nethermind.Core/StorageCell.cs +++ b/src/Nethermind/Nethermind.Core/StorageCell.cs @@ -3,9 +3,11 @@ using System; using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Int256; namespace Nethermind.Core @@ -17,17 +19,16 @@ namespace Nethermind.Core private readonly bool _isHash; public Address Address { get; } + public bool IsHash => _isHash; public UInt256 Index => _index; public ValueHash256 Hash => _isHash ? Unsafe.As(ref Unsafe.AsRef(in _index)) : GetHash(); - public bool IsHash => _isHash; - private ValueHash256 GetHash() { Span key = stackalloc byte[32]; Index.ToBigEndian(key); - return ValueKeccak.Compute(key); + return KeccakCache.Compute(key); } public StorageCell(Address address, in UInt256 index) @@ -43,7 +44,10 @@ public StorageCell(Address address, ValueHash256 hash) _isHash = true; } - public bool Equals(StorageCell other) => Address.Equals(other.Address) && Index.Equals(other.Index); + public bool Equals(StorageCell other) => + _isHash == other._isHash && + Unsafe.As>(ref Unsafe.AsRef(in _index)) == Unsafe.As>(ref Unsafe.AsRef(in other._index)) && + Address.Equals(other.Address); public override bool Equals(object? obj) { @@ -57,12 +61,8 @@ public override bool Equals(object? obj) public override int GetHashCode() { - uint hash = (uint)Address.GetHashCode(); - hash = BitOperations.Crc32C(hash, _index.u0); - hash = BitOperations.Crc32C(hash, _index.u1); - hash = BitOperations.Crc32C(hash, _index.u2); - hash = BitOperations.Crc32C(hash, _index.u3); - return (int)hash; + int hash = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in _index), 1)).FastHash(); + return hash ^ Address.GetHashCode(); } public override string ToString() diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index c642ea2e433..358c1247d82 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -1132,7 +1132,9 @@ private CallResult ExecuteCode(ref stack.PushBytesRef())); break; } case Instruction.ADDRESS: diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 8859089addb..e786e96fd12 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -1602,10 +1602,7 @@ public override bool Equals(object? other) return Equals(other as Rlp); } - public override int GetHashCode() - { - return Bytes is not null ? Bytes.GetSimplifiedHashCode() : 0; - } + public override int GetHashCode() => new ReadOnlySpan(Bytes).FastHash(); public bool Equals(Rlp? other) { diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index 9469208c731..af65595b304 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -2,10 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Collections; @@ -38,7 +40,7 @@ internal sealed class PersistentStorageProvider : PartialStorageProviderBase private readonly Dictionary _originalValues = new(); private readonly HashSet _committedThisRound = new(); - private readonly Dictionary> _blockCache = new(4_096); + private readonly Dictionary> _blockCache = new(4_096); private readonly ConcurrentDictionary? _preBlockCache; private readonly Func _loadFromTree; @@ -303,10 +305,10 @@ private void SaveToTree(HashSet toUpdateRoots, Change change) toUpdateRoots.Add(change.StorageCell.Address); tree.Set(change.StorageCell.Index, change.Value); - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, change.StorageCell.Address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, change.StorageCell.Address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } dict[change.StorageCell.Index] = change.Value; @@ -360,10 +362,10 @@ public void WarmUp(in StorageCell storageCell, bool isEmpty) private ReadOnlySpan LoadFromTree(in StorageCell storageCell) { - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell.Address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell.Address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } ref byte[]? value = ref dict.GetValueRefOrAddDefault(storageCell.Index, out exists); @@ -455,10 +457,10 @@ public override void ClearStorage(Address address) { base.ClearStorage(address); - ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, address, out bool exists); + ref SelfDestructDictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, address, out bool exists); if (!exists) { - dict = new SelfDestructDictionary(StorageTree.EmptyBytes); + dict = new SelfDestructDictionary(StorageTree.EmptyBytes); } dict.SelfDestruct(); @@ -477,10 +479,10 @@ public StorageTree Create(Address address, IScopedTrieStore trieStore, Hash256 s => new(trieStore, storageRoot, logManager); } - private class SelfDestructDictionary(TValue destructedValue) where TKey : notnull + private sealed class SelfDestructDictionary(TValue destructedValue) { private bool _selfDestruct; - private readonly Dictionary _dictionary = new(); + private readonly Dictionary _dictionary = new(Comparer.Instance); public void SelfDestruct() { @@ -488,7 +490,7 @@ public void SelfDestruct() _dictionary.Clear(); } - public ref TValue? GetValueRefOrAddDefault(TKey storageCellIndex, out bool exists) + public ref TValue? GetValueRefOrAddDefault(UInt256 storageCellIndex, out bool exists) { ref TValue value = ref CollectionsMarshal.GetValueRefOrAddDefault(_dictionary, storageCellIndex, out exists); if (!exists && _selfDestruct) @@ -499,10 +501,23 @@ public void SelfDestruct() return ref value; } - public TValue? this[TKey key] + public TValue? this[UInt256 key] { set => _dictionary[key] = value; } + + private sealed class Comparer : IEqualityComparer + { + public static Comparer Instance { get; } = new(); + + private Comparer() { } + + public bool Equals(UInt256 x, UInt256 y) + => Unsafe.As>(ref x) == Unsafe.As>(ref y); + + public int GetHashCode([DisallowNull] UInt256 obj) + => MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(in obj, 1)).FastHash(); + } } } } diff --git a/src/Nethermind/Nethermind.State/PreBlockCaches.cs b/src/Nethermind/Nethermind.State/PreBlockCaches.cs index 5c75b47ad5b..696afcb1603 100644 --- a/src/Nethermind/Nethermind.State/PreBlockCaches.cs +++ b/src/Nethermind/Nethermind.State/PreBlockCaches.cs @@ -6,6 +6,8 @@ using System.Numerics; using System.Runtime.InteropServices; using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Collections; using Nethermind.Trie; @@ -58,31 +60,6 @@ public readonly struct PrecompileCacheKey(Address address, ReadOnlyMemory private ReadOnlyMemory Data { get; } = data; public bool Equals(PrecompileCacheKey other) => Address == other.Address && Data.Span.SequenceEqual(other.Data.Span); public override bool Equals(object? obj) => obj is PrecompileCacheKey other && Equals(other); - public override int GetHashCode() - { - uint crc = (uint)Address.GetHashCode(); - ReadOnlySpan span = Data.Span; - var longSize = span.Length / sizeof(ulong) * sizeof(ulong); - if (longSize > 0) - { - foreach (ulong ul in MemoryMarshal.Cast(span[..longSize])) - { - crc = BitOperations.Crc32C(crc, ul); - } - foreach (byte b in span[longSize..]) - { - crc = BitOperations.Crc32C(crc, b); - } - } - else - { - foreach (byte b in span) - { - crc = BitOperations.Crc32C(crc, b); - } - } - - return (int)crc; - } + public override int GetHashCode() => Data.Span.FastHash() ^ Address.GetHashCode(); } } diff --git a/src/Nethermind/Nethermind.State/StateTree.cs b/src/Nethermind/Nethermind.State/StateTree.cs index 9d32c8e28ab..19cc6ca2a96 100644 --- a/src/Nethermind/Nethermind.State/StateTree.cs +++ b/src/Nethermind/Nethermind.State/StateTree.cs @@ -42,14 +42,14 @@ public StateTree(ITrieStore? store, ILogManager? logManager) [DebuggerStepThrough] public Account? Get(Address address, Hash256? rootHash = null) { - ReadOnlySpan bytes = Get(ValueKeccak.Compute(address.Bytes).BytesAsSpan, rootHash); + ReadOnlySpan bytes = Get(KeccakCache.Compute(address.Bytes).BytesAsSpan, rootHash); return bytes.IsEmpty ? null : _decoder.Decode(bytes); } [DebuggerStepThrough] public bool TryGetStruct(Address address, out AccountStruct account, Hash256? rootHash = null) { - ReadOnlySpan bytes = Get(ValueKeccak.Compute(address.Bytes).BytesAsSpan, rootHash); + ReadOnlySpan bytes = Get(KeccakCache.Compute(address.Bytes).BytesAsSpan, rootHash); Rlp.ValueDecoderContext valueDecoderContext = new Rlp.ValueDecoderContext(bytes); if (bytes.IsEmpty) { @@ -69,7 +69,7 @@ public bool TryGetStruct(Address address, out AccountStruct account, Hash256? ro public void Set(Address address, Account? account) { - ValueHash256 keccak = ValueKeccak.Compute(address.Bytes); + KeccakCache.ComputeTo(address.Bytes, out ValueHash256 keccak); Set(keccak.BytesAsSpan, account is null ? null : account.IsTotallyEmpty ? EmptyAccountRlp : Rlp.Encode(account)); } diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index 472441c9e3b..748ed3aa159 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -5,6 +5,8 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -46,12 +48,16 @@ public StorageTree(IScopedTrieStore? trieStore, Hash256 rootHash, ILogManager? l TrieType = TrieType.Storage; } - private static void ComputeKey(in UInt256 index, ref Span key) + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ComputeKey(in UInt256 index, Span key) { index.ToBigEndian(key); - // in situ calculation - KeccakHash.ComputeHashBytesToSpan(key, key); + // We can't direct ComputeTo the key as its also the input, so need a separate variable + KeccakCache.ComputeTo(key, out ValueHash256 keyHash); + // Which we can then directly assign to fast update the key + Unsafe.As(ref MemoryMarshal.GetReference(key)) = keyHash; } [SkipLocalsInit] @@ -62,9 +68,15 @@ public byte[] Get(in UInt256 index, Hash256? storageRoot = null) return GetArray(Lookup[index], storageRoot); } - Span key = stackalloc byte[32]; - ComputeKey(index, ref key); - return GetArray(key, storageRoot); + return GetWithKeyGenerate(in index, storageRoot); + + [SkipLocalsInit] + byte[] GetWithKeyGenerate(in UInt256 index, Hash256 storageRoot) + { + Span key = stackalloc byte[32]; + ComputeKey(index, key); + return GetArray(key, storageRoot); + } } public byte[] GetArray(ReadOnlySpan rawKey, Hash256? rootHash = null) @@ -90,9 +102,15 @@ public void Set(in UInt256 index, byte[] value) SetInternal(Lookup[index], value); } else + { + SetWithKeyGenerate(in index, value); + } + + [SkipLocalsInit] + void SetWithKeyGenerate(in UInt256 index, byte[] value) { Span key = stackalloc byte[32]; - ComputeKey(index, ref key); + ComputeKey(index, key); SetInternal(key, value); } }