Remove unused sizeInfo and other performance tweaks

benchmark/BDN.benchmark/Operations/RawStringOperations.cs

@@ -18,10 +18,10 @@ public unsafe class RawStringOperations : OperationsBase
         static ReadOnlySpan<byte> SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\nd\r\n$1\r\n9\r\n$1\r\nd\r\n"u8;
         Request setex;
 
-        static ReadOnlySpan<byte> SETNX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nNX\r\n"u8;
+        static ReadOnlySpan<byte> SETNX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nNX\r\n"u8;   // Becomes SETEXNX rather than SETNX
         Request setnx;
 
-        static ReadOnlySpan<byte> SETXX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nXX\r\n"u8;
+        static ReadOnlySpan<byte> SETXX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nXX\r\n"u8;   // Becomes SETEXXX rather than SETXX
         Request setxx;
 
         static ReadOnlySpan<byte> GETNF => "*2\r\n$3\r\nGET\r\n$1\r\nb\r\n"u8;

libs/server/InputHeader.cs

@@ -12,7 +12,7 @@ namespace Garnet.server
 {
     /// <summary>
     /// Flags used by append-only file (AOF/WAL)
-    /// The byte representation only use the last 3 bits of the byte since the lower 5 bits of the field used to store the flag stores other data in the case of Object types.
+    /// The byte representation only use the last 3 bits of the byte since the lower 5 bits of the "union" field that is used to store the flag stores other data (see RespInputHeader.FlagMask).
     /// In the case of a Rawstring, the last 4 bits are used for flags, and the other 4 bits are unused of the byte.
     /// </summary>
     [Flags]

libs/server/Resp/RespServerSession.cs

@@ -640,9 +640,6 @@ private void ProcessMessages<TBasicApi, TTxnApi>(ref TBasicApi basicApi, ref TTx
                 {
                     var noScriptPassed = true;
 
-                    // Reset error flag unconditionally (only read when commandStats != null)
-                    commandErrorWritten = false;
-
                     if (CheckACLPermissions(cmd) && (noScriptPassed = CheckScriptPermissions(cmd)))
                     {
                         if (txnManager.state != TxnState.None)
@@ -670,7 +667,10 @@ private void ProcessMessages<TBasicApi, TTxnApi>(ref TBasicApi basicApi, ref TTx
                         {
                             commandStats.IncrementCalls(cmd);
                             if (commandErrorWritten)
+                            {
                                 commandStats.IncrementFailed(cmd);
+                                commandErrorWritten = false;
+                            }
                         }
                     }
                     else

libs/server/Storage/Functions/MainStore/RMWMethods.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using Garnet.common;
 using Microsoft.Extensions.Logging;
@@ -382,16 +383,19 @@ public readonly bool InPlaceUpdater(ref LogRecord logRecord, ref StringInput inp
                 return false;
             }
 
-            // RangeIndex type safety – reject non-RI commands on RI keys
-            if (logRecord.RecordType == RangeIndexManager.RangeIndexRecordType && !input.header.cmd.IsLegalOnRangeIndex())
+            // RangeIndex type safety – normal string records have RecordType 0; skip all checks in that common case.
+            if (logRecord.RecordType == RangeIndexManager.RangeIndexRecordType)
             {
-                rmwInfo.Action = RMWAction.WrongType;
-                return false;
+                // Reject non-RI commands on RI keys
+                if (!input.header.cmd.IsLegalOnRangeIndex())
+                {
+                    rmwInfo.Action = RMWAction.WrongType;
+                    return false;
+                }
             }
-
-            // Reject RI-specific commands on non-RI keys
-            if (logRecord.RecordType != RangeIndexManager.RangeIndexRecordType && input.header.cmd.IsRangeIndexCommand())
+            else if (input.header.cmd.IsRangeIndexCommand())
             {
+                // Reject RI-specific commands on non-RI keys
                 rmwInfo.Action = RMWAction.WrongType;
                 return false;
             }
@@ -447,6 +451,7 @@ private readonly IPUResult InPlaceUpdaterWorker(ref LogRecord logRecord, ref Str
                     return IPUResult.NotUpdated;
                 case RespCommand.SET:
                 case RespCommand.SETEXXX:
+                    // Note: SETEXXX may or may not actually have an expiration.
                     // Check if SetGet flag is set
                     if (input.header.CheckSetGetFlag())
                     {
@@ -1426,6 +1431,7 @@ public readonly bool PostCopyUpdater<TSourceLogRecord>(in TSourceLogRecord srcLo
         }
 
         /// <inheritdoc />
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void PostRMWOperation<TKey, TEpochAccessor>(TKey key, ref StringInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor)
             where TKey : IKey
 #if NET9_0_OR_GREATER

libs/server/Storage/Functions/MainStore/ReadMethods.cs

@@ -16,7 +16,19 @@ namespace Garnet.server
         public bool Reader<TSourceLogRecord>(in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, ref ReadInfo readInfo)
             where TSourceLogRecord : ISourceLogRecord
         {
-            if (srcLogRecord.Info.ValueIsObject)
+            var info = srcLogRecord.Info;
+
+            // Fast path for simple GET on a normal inline string key with no optional fields.
+            // HasOptionalOrObjectFields is false iff: KeyIsInline, ValueIsInline, !HasETag, !HasExpiration (implies !ValueIsObject).
+            // RecordType 0 means normal string (not VectorSet or RangeIndex).
+            // This avoids expiry checks (no expiration), type-safety checks, ETag handling, and custom command dispatch.
+            if (input.arg1 < 0 && !info.HasOptionalOrObjectFields && srcLogRecord.RecordType == 0)
+            {
+                CopyRespTo(srcLogRecord.ValueSpan, ref output);
+                return true;
+            }
+
+            if (info.ValueIsObject)
             {
                 readInfo.Action = ReadAction.WrongType;
                 return false;

libs/server/Storage/Functions/MainStore/UpsertMethods.cs

@@ -67,8 +67,9 @@ public void PostInitialWriter<TSourceLogRecord>(ref LogRecord logRecord, in Reco
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool InPlaceWriter(ref LogRecord logRecord, ref StringInput input, ReadOnlySpan<byte> srcValue, ref StringOutput output, ref UpsertInfo upsertInfo)
         {
-            // Prevent SET from overwriting VectorSet or RangeIndex stubs
-            if (logRecord.RecordType == VectorManager.RecordType || logRecord.RecordType == RangeIndexManager.RangeIndexRecordType)
+            // Prevent SET from overwriting VectorSet or RangeIndex stubs – normal string records have RecordType 0; skip all checks in that common case.
+            var recordType = logRecord.RecordType;
+            if (recordType != 0 && (recordType == VectorManager.RecordType || recordType == RangeIndexManager.RangeIndexRecordType))
             {
                 upsertInfo.Action = UpsertAction.WrongType;
                 return false;
@@ -103,6 +104,7 @@ public bool InPlaceWriter<TSourceLogRecord>(ref LogRecord logRecord, ref StringI
         }
 
         /// <inheritdoc />
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void PostUpsertOperation<TKey, TEpochAccessor>(TKey key, ref StringInput input, ReadOnlySpan<byte> valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor)
             where TKey : IKey
 #if NET9_0_OR_GREATER
@@ -115,6 +117,7 @@ public void PostUpsertOperation<TKey, TEpochAccessor>(TKey key, ref StringInput
         }
 
         /// <inheritdoc />
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void PostUpsertOperation<TKey, TEpochAccessor>(TKey key, ref StringInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor)
             where TKey : IKey
 #if NET9_0_OR_GREATER

libs/server/Storage/Functions/SessionFunctionsUtils.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using Garnet.common;
 using Microsoft.Extensions.Logging;
 using Tsavorite.core;
@@ -93,13 +94,14 @@ internal static bool InPlaceWriterForSpanValue<TInput, TVariableLengthInput>(ref
                 ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, TVariableLengthInput varlenInput, FunctionsState functionsState, long expiration)
             where TVariableLengthInput : IVariableLengthInput<TInput>
         {
-            RecordSizeInfo sizeInfo = new();
+            RecordSizeInfo sizeInfo;
 
             if (logRecord.Info.ValueIsInline && (expiration == 0 || logRecord.Info.HasExpiration))
             {
                 var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength;
                 if (!logRecord.TrySetPinnedValueSpan(newValue, valueAddress, ref valueLength))
                     return false;
+                sizeInfo = new();
             }
             else
             {
@@ -159,6 +161,7 @@ internal static bool InPlaceWriterForLogRecordValue<TSourceLogRecord, TInput, TV
             return true;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static void UpdateExpiration(ref LogRecord logRecord, long expiration)
         {
             if (expiration != 0)

libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs

@@ -77,7 +77,7 @@ public readonly void PostInitialWriter<TSourceLogRecord>(ref LogRecord dstLogRec
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan<byte> srcValue, ref TOutput output, ref UpsertInfo upsertInfo)
+        public bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan<byte> srcValue, ref TOutput output, ref UpsertInfo upsertInfo)
         {
             if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo))
                 return false;
@@ -86,7 +86,7 @@ public bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, r
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo)
+        public bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo)
         {
             if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo))
                 return false;
@@ -95,7 +95,7 @@ public bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, r
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool InPlaceWriter<TSourceLogRecord>(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo)
+        public bool InPlaceWriter<TSourceLogRecord>(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo)
             where TSourceLogRecord : ISourceLogRecord
         {
             if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, in inputLogRecord, ref output, ref upsertInfo))
@@ -167,7 +167,7 @@ public bool PostCopyUpdater<TSourceLogRecord>(in TSourceLogRecord srcLogRecord,
 
         #region InPlaceUpdater
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool InPlaceUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status)
+        public bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status)
         {
             // This wraps the ISessionFunctions call to provide expiration logic.
             if (_clientSession.functions.InPlaceUpdater(ref logRecord, ref input, ref output, ref rmwInfo))

libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctionsWrapper.cs

@@ -31,9 +31,9 @@ bool InitialWriter<TSourceLogRecord>(ref LogRecord logRecord, in RecordSizeInfo
         void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo);
         void PostInitialWriter<TSourceLogRecord>(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo)
             where TSourceLogRecord : ISourceLogRecord;
-        bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan<byte> srcValue, ref TOutput output, ref UpsertInfo upsertInfo);
-        bool InPlaceWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo);
-        bool InPlaceWriter<TSourceLogRecord>(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo)
+        bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan<byte> srcValue, ref TOutput output, ref UpsertInfo upsertInfo);
+        bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo);
+        bool InPlaceWriter<TSourceLogRecord>(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo)
             where TSourceLogRecord : ISourceLogRecord;
         void PostUpsertOperation<TKey, TEpochAccessor>(TKey key, ref TInput input, ReadOnlySpan<byte> srcValueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor)
              where TKey : IKey
@@ -71,7 +71,7 @@ bool PostCopyUpdater<TSourceLogRecord>(in TSourceLogRecord srcLogRecord, ref Log
         #endregion CopyUpdater
 
         #region InPlaceUpdater
-        bool InPlaceUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status);
+        bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status);
         #endregion InPlaceUpdater
 
         void PostRMWOperation<TKey, TEpochAccessor>(TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor)

libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs

@@ -140,22 +140,15 @@ internal OperationStatus InternalRMW<TKey, TInput, TOutput, TContext, TSessionFu
                         goto CreateNewRecord;
                     }
 
-                    var sizeInfo = new RecordSizeInfo(); // TODO temporary for perf work
-
-                    // Track value heap delta across in-place update. Only measure when value is
-                    // heap-allocated (overflow or object); inline values have zero heap cost.
-                    var ipuPreInline = srcLogRecord.Info.ValueIsInline;
-                    var ipuPreHeap = ipuPreInline ? 0L : srcLogRecord.GetValueHeapMemorySize();
-
-                    if (sessionFunctions.InPlaceUpdater(ref srcLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo, out status))
+                    // Track value heap delta across in-place update. HeapMemorySize is zero for inline values but IPU may change the value from inline to object or vice-versa.
+                    var sizeTracker = hlogBase.logSizeTracker;
+                    var ipuPreHeap = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() : 0L;
+                    var ipuResult = sessionFunctions.InPlaceUpdater(ref srcLogRecord, ref input, ref output, ref rmwInfo, out status);
+                    var ipuDelta = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() - ipuPreHeap : 0L;
+                    if (ipuResult)
                     {
-                        if (!ipuPreInline || !srcLogRecord.Info.ValueIsInline)
-                        {
-                            var ipuPostHeap = srcLogRecord.Info.ValueIsInline ? 0L : srcLogRecord.GetValueHeapMemorySize();
-                            var ipuDelta = ipuPostHeap - ipuPreHeap;
-                            if (ipuDelta != 0)
-                                hlogBase.logSizeTracker?.IncrementSize(ipuDelta);
-                        }
+                        if (ipuDelta != 0)
+                            sizeTracker.IncrementSize(ipuDelta);
 
                         MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx);
 
@@ -166,22 +159,15 @@ internal OperationStatus InternalRMW<TKey, TInput, TOutput, TContext, TSessionFu
 
                     if (rmwInfo.Action == RMWAction.ExpireAndStop)
                     {
-                        // ExpireAndStop: the object was mutated in-place (e.g. last element removed)
-                        // before IPU returned false. Track the delta before OnDispose subtracts the
-                        // remaining empty-collection overhead.
-                        if (!ipuPreInline || !srcLogRecord.Info.ValueIsInline)
-                        {
-                            var ipuPostHeap = srcLogRecord.Info.ValueIsInline ? 0L : srcLogRecord.GetValueHeapMemorySize();
-                            var ipuDelta = ipuPostHeap - ipuPreHeap;
-                            if (ipuDelta != 0)
-                                hlogBase.logSizeTracker?.IncrementSize(ipuDelta);
-                        }
+                        // ExpireAndStop: the object was mutated in-place (e.g. last element removed) before IPU returned false.
+                        // Track the delta before OnDispose subtracts the remaining empty-collection overhead.
+                        if (ipuDelta != 0)
+                            sizeTracker.IncrementSize(ipuDelta);
                         MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx);
 
                         pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress;
 
-                        // Dispose resources and decrement value heap BEFORE setting Tombstone,
-                        // so that GetValueHeapMemorySize returns the correct pre-tombstone value.
+                        // Dispose resources and decrement value heap BEFORE setting Tombstone so GetValueHeapMemorySize returns the correct pre-tombstone value.
                         OnDispose(ref srcLogRecord, DisposeReason.Deleted);
 
                         srcLogRecord.InfoRef.SetTombstone();
@@ -196,9 +182,8 @@ internal OperationStatus InternalRMW<TKey, TInput, TOutput, TContext, TSessionFu
                     }
                     else if (rmwInfo.Action == RMWAction.ExpireAndResume)
                     {
-                        // ExpireAndResume: for IPU, ReinitializeExpiredRecord already called
-                        // OnDispose(Deleted). If it failed and we fall through to CreateNewRecord,
-                        // the record is already disposed.
+                        // ExpireAndResume: for IPU, ReinitializeExpiredRecord already called OnDispose(Deleted).
+                        // If it failed and we fall through to CreateNewRecord, the record is already disposed.
                     }
                     else if (rmwInfo.Action == RMWAction.WrongType)
                     {