Add ETag sample code and Blog

Garnet.sln

@@ -1,4 +1,4 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31808.319
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -111,6 +111,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Garnet.resources", "libs\re
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NoOpModule", "playground\NoOpModule\NoOpModule.csproj", "{D4C9A1A0-7053-F072-21F5-4E0C5827136D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ETag", "playground\ETag\ETag.csproj", "{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -335,6 +337,14 @@ Global
 		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|x64.ActiveCfg = Release|Any CPU
 		{D4C9A1A0-7053-F072-21F5-4E0C5827136D}.Release|x64.Build.0 = Release|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Debug|x64.Build.0 = Debug|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Release|x64.ActiveCfg = Release|Any CPU
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -370,6 +380,7 @@ Global
 		{DF2DD03E-87EE-482A-9FBA-6C8FBC23BDC5} = {697766CD-2046-46D9-958A-0FD3B46C98D4}
 		{A48412B4-FD60-467E-A5D9-F155CAB4F907} = {147FCE31-EC09-4C90-8E4D-37CA87ED18C3}
 		{D4C9A1A0-7053-F072-21F5-4E0C5827136D} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
+		{4FBA1587-BAFC-49F8-803A-D1CF431A26F5} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {2C02C405-4798-41CA-AF98-61EDFEF6772E}

libs/server/Storage/Functions/MainStore/RMWMethods.cs

@@ -352,8 +352,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re
                     long newEtag = functionsState.etagState.etag + 1;
 
                     value.ShrinkSerializedLength(metadataSize + inputValue.Length + EtagConstants.EtagSize);
-                    rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize);
                     rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize);
+                    rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize);
 
                     value.SetEtagInPayload(newEtag);
 

playground/ETag/ETag.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="StackExchange.Redis" />
+  </ItemGroup>
+
+</Project>

playground/ETag/OccSimulation.cs

@@ -0,0 +1,254 @@
+using StackExchange.Redis;
+using System;
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace ETag
+{
+    /*
+        This code sample shows how to use ETags to implement lock-free synchronization for non-atomic operations
+
+        OCC is like using a CAS loop to make sure the data we are writing has not had a change in between the time
+        we have read it and written back.
+
+        Scenario 1: Lock free Json manipulation, we are using JSON as our value but this could essentially be
+        any data that you falls in the below category that is not provided by the objects API of Garnet.
+
+        Granular Data Structure: Refers to data that is divided into small, independent parts that can be manipulated individually. For example, MongoDB documents allow granular updates on individual fields.
+
+        Mutable Object: If the object allows you to modify its individual components without recreating the entire object, it’s referred to as mutable. For example, Python dictionaries and lists are mutable.
+
+        Partial Updatable Data: This term is used in contexts like databases where updates can target specific fields without affecting the entire record.
+
+        Modular Data Structure: If the object is designed to have independent, self-contained modules (like classes or subcomponents), you might describe it as modular.
+
+        Composable Data: This term applies when different parts of the data can be independently composed, used, or updated, often seen in functional programming.
+
+        Hierarchical Data Structure: Refers to objects with nested components, like JSON or XML, where parts of the hierarchy can be accessed and modified independently
+
+                Simulation Description:
+
+                We have 2 different clients that are updating the same value for a key, but different parts of it concurrently
+                We want to make sure we don't lose the updates between these 2 clients.
+
+                Client 1: Updates the number of cats a user has
+                Client 2: Changes the flag for the user for whether or not the user has too many cats.
+                    Client 2 only considers that a user has too many cats when the number of cats is divisible by 5,
+                    otherwise it marks the user as false for not having too many cats
+    */
+    class OccSimulation
+    {
+        public static async Task RunSimulation()
+        {
+            using var redis = await ConnectionMultiplexer.ConnectAsync(GarnetConnectionStr);
+            var db = redis.GetDatabase(0);
+
+            ContosoUserInfo userInfo = new ContosoUserInfo
+            {
+                FirstName = "Hamdaan",
+                LastName = "Khalid",
+                NumberOfCats = 1,
+                TooManyCats = true,
+                Basket = new List<string>()
+            };
+
+            string userKey = "hkhalid";
+            string serializedUserInfo = JsonSerializer.Serialize(userInfo);
+
+            // Seed the item in the database
+            long initialEtag = (long)await db.ExecuteAsync("SET", userKey, serializedUserInfo, "WITHETAG");
+
+            // Cancellation token is used to exit program on end of interactive repl
+            var cts = new CancellationTokenSource();
+            // Clone user info so they are task local
+            var client1Task = Task.Run(() => Client1(userKey, initialEtag, (ContosoUserInfo)userInfo.Clone(), cts.Token));
+            var client2Task = Task.Run(() => Client2(userKey, initialEtag, (ContosoUserInfo)userInfo.Clone(), cts.Token));
+
+            // Interactive REPL to change any property in the ContosoUserInfo
+            while (true)
+            {
+                Console.WriteLine("Enter the property to change (FirstName, LastName, NumberOfCats, TooManyCats, AddToBasket, RemoveFromBasket) or 'exit' to quit:");
+                Console.WriteLine($"Initial User Info: {JsonSerializer.Serialize(userInfo)}");
+                string input = Console.ReadLine()!;
+
+                if (input.ToLower() == "exit")
+                {
+                    cts.Cancel();
+                    break;
+                }
+
+                Action<ContosoUserInfo> userUpdateAction = (userInfo) => {};
+                switch (input)
+                {
+                    case "FirstName":
+                        Console.WriteLine("Enter new FirstName:");
+                        string newFirstName = Console.ReadLine()!;
+                        userUpdateAction = (info) => info.FirstName = newFirstName;
+                        break;
+                    case "LastName":
+                        Console.WriteLine("Enter new LastName:");
+                        string newLastName = Console.ReadLine()!;
+                        userUpdateAction = (info) => info.FirstName = newLastName;
+                        break;
+                    case "NumberOfCats":
+                        Console.WriteLine("Enter new NumberOfCats:");
+                        if (int.TryParse(Console.ReadLine(), out int numberOfCats))
+                        {
+                            userUpdateAction = (info) => info.NumberOfCats = numberOfCats;
+                        }
+                        else
+                        {
+                            Console.WriteLine("Invalid number.");
+                        }
+                        break;
+                    case "TooManyCats":
+                        Console.WriteLine("Enter new TooManyCats (true/false):");
+                        if (bool.TryParse(Console.ReadLine(), out bool tooManyCats))
+                        {
+                            userUpdateAction = (info) => info.TooManyCats = tooManyCats;
+                        }
+                        else
+                        {
+                            Console.WriteLine("Invalid boolean.");
+                        }
+                        break;
+                    case "AddToBasket":
+                        Console.WriteLine("Enter item to add to basket:");
+                        string addItem = Console.ReadLine()!;
+                        userUpdateAction = (info) => info.Basket.Add(addItem);
+                        break;
+                    case "RemoveFromBasket":
+                        Console.WriteLine("Enter item to remove from basket:");
+                        string removeItem = Console.ReadLine()!;
+                        userUpdateAction = (info) => info.Basket.Remove(removeItem);
+                        break;
+                    default:
+                        Console.WriteLine("Unknown property.");
+                        break;
+                }
+
+                // Update the user info in the database, and then for the REPL
+                (initialEtag, userInfo) = await LockFreeUpdateUserInfo(db, userKey, initialEtag, userInfo, userUpdateAction);
+                Console.WriteLine($"Updated User Info: {JsonSerializer.Serialize(userInfo)}");
+            }
+
+            cts.Cancel();
+
+            try
+            {
+                await Task.WhenAll(client1Task, client2Task);
+            }
+            catch (OperationCanceledException)
+            {
+                Console.WriteLine("Tasks were cancelled.");
+            }
+        }
+
+        static async Task Client1(string userKey, long initialEtag, ContosoUserInfo initialUserInfo, CancellationToken token)
+        {
+            Random random = new Random();
+            using var redis = await ConnectionMultiplexer.ConnectAsync(GarnetConnectionStr);
+            var db = redis.GetDatabase(0);
+
+            long etag = initialEtag;
+            ContosoUserInfo userInfo = initialUserInfo;
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                (etag, userInfo) = await LockFreeUpdateUserInfo(db, userKey, etag, userInfo, (ContosoUserInfo userInfo) =>
+                {
+                    userInfo.NumberOfCats++;
+                });
+                await Task.Delay(TimeSpan.FromSeconds(random.Next(0, 15)), token);
+            }
+        }
+
+        static async Task Client2(string userKey, long initialEtag, ContosoUserInfo initialUserInfo, CancellationToken token)
+        {
+            Random random = new Random();
+            using var redis = await ConnectionMultiplexer.ConnectAsync(GarnetConnectionStr);
+            var db = redis.GetDatabase(0);
+
+            long etag = initialEtag;
+            ContosoUserInfo userInfo = initialUserInfo;
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                (etag, userInfo) = await LockFreeUpdateUserInfo(db, userKey, etag, userInfo, (ContosoUserInfo userInfo) =>
+                {
+                    userInfo.TooManyCats = userInfo.NumberOfCats % 5 == 0;
+                });
+                await Task.Delay(TimeSpan.FromSeconds(random.Next(0, 15)), token);
+            }
+        }
+
+        static async Task<(long, ContosoUserInfo)> LockFreeUpdateUserInfo(IDatabase db, string userKey, long initialEtag, ContosoUserInfo initialUserInfo, Action<ContosoUserInfo> updateAction)
+        {
+            // Compare and Swap Updating
+            long etag = initialEtag;
+            ContosoUserInfo userInfo = initialUserInfo;
+            while (true)
+            {
+                // perform invoker passed update on userInfo 
+                updateAction(userInfo);
+
+                var (updatedSuccesful, newEtag, newUserInfo) = await UpdateUserIfMatch(db, etag, userKey, userInfo);
+                etag = newEtag;
+                userInfo = newUserInfo;
+
+                if (updatedSuccesful)
+                    break;
+            }
+
+            return (etag, userInfo);
+        }
+
+        static async Task<(bool updated, long etag, ContosoUserInfo)> UpdateUserIfMatch(IDatabase db, long etag, string key, ContosoUserInfo value)
+        {
+            // You may notice the "!" that is because we know that SETIFMATCH doesn't return null
+            string serializedUserInfo = JsonSerializer.Serialize(value);
+            RedisResult[] res = (RedisResult[])(await db.ExecuteAsync("SETIFMATCH", key, serializedUserInfo, etag))!;
+
+            if (res[1].IsNull)
+                return (true, (long)res[0], value);
+
+            ContosoUserInfo deserializedUserInfo = JsonSerializer.Deserialize<ContosoUserInfo>((string)res[1]!)!;
+            return (false, (long)res[0], deserializedUserInfo);
+        }
+
+        static string GarnetConnectionStr = "localhost:6379,connectTimeout=999999,syncTimeout=999999";
+    }
+
+    class ContosoUserInfo : ICloneable
+    {
+        [JsonPropertyName("first_name")]
+        public required string FirstName { get; set; }
+
+        [JsonPropertyName("last_name")]
+        public required string LastName { get; set; }
+
+        [JsonPropertyName("number_of_cats")]
+        public required int NumberOfCats { get; set; }
+
+        [JsonPropertyName("too_many_cats")]
+        public required bool TooManyCats { get; set; }
+
+        [JsonPropertyName("basket")]
+        public required List<string> Basket { get; set; }
+
+        public object Clone()
+        {
+            return new ContosoUserInfo
+            {
+                FirstName = this.FirstName,
+                LastName = this.LastName,
+                NumberOfCats = this.NumberOfCats,
+                TooManyCats = this.TooManyCats,
+                Basket = new List<string>(this.Basket)
+            };
+        }
+    }
+}

playground/ETag/Program.cs

@@ -0,0 +1,11 @@
+using System.Threading.Tasks;
+
+namespace ETag;
+
+class Program
+{
+    static async Task Main(string[] args)
+    {
+        await OccSimulation.RunSimulation();
+    }
+}

website/blog/2025-01-18-etag-when-and-how

@@ -0,0 +1,23 @@
+---
+slug: etags-when-and-how
+title: ETags, When and How
+authors: hkhalid 
+tags: [garnet, concurrency, caching, lock-free]
+---
+
+Garnet recently announced native support for ETag based commands. ETags are a primitive that give users access to sophisticated techniques such as Optimistic Concurrency Control as well as more efficient network bandwidth utilization for caching.
+
+ETags are currently provided over Raw Strings in Garnet.
+
+ETags are available to users without the need of any migration; this means your existing key-value pairs can start using ETags on the fly without affecting existing performance metrics.
+You can find the [ETag API documentation here](HK TODO).
+
+This article will help you explore when and how you can use this new shiny Garnet feature for your current and future applications.
+
+**Read this article further if you need or might want**:
+1. Reduced network bandwidth utilization for caching.
+2. Avoid the cost of transactions for working with non-atomic values in your cache-store
+
+Both the above cases are common cases for folks using Caches, lets go case by case.
+
+## 

website/blog/authors.yml

@@ -4,3 +4,8 @@ badrishc:
   url: https://badrish.net
   image_url: https://badrish.net/assets/icons/badrish4.jpg
 
+hkhalid:
+  name: Hamdaan Khalid
+  title: Software Engineer, Azure Resource Graph
+  url: https://hamdaan-rails-personal.herokuapp.com/
+  image_url: https://media.licdn.com/dms/image/v2/D5603AQEB5k6B-_kYcg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1713142460509?e=1743033600&v=beta&t=efEhRJq1SLgi09uCSUQJN3ssq-_cwljG0ysUc54GcSc