#16 external wall addition and corrections

src/MIC/Assets/.MeshCloudScripting/Presentation1/App.cs

@@ -28,16 +28,19 @@ public class App : IHostedService, IAsyncDisposable
         private readonly ICloudApplication _app;
         private readonly AppSettings _appSettings;
         private readonly PersonFlow _personFlow;
-        private readonly QLearning _qLearning1;
-        private readonly QLearning _qLearning2;
+        private readonly List<QLearning> _qLearnings;
         public App(ICloudApplication app, ILogger<App> logger)
         {
             _app = app;
             _logger = logger;
             _appSettings = LoadSettings();
             _personFlow = new PersonFlow();
-            _qLearning1 = new QLearning(100, 4, 0.5, 0.9, 0.1);
-            _qLearning2 = new QLearning(100, 4, 0.5, 0.9, 0.1);
+            _qLearnings = new List<QLearning>();
+            for (int i = 0; i < 5; i++)
+            {
+                int numStates = 1800;
+                _qLearnings.Add(new QLearning(numStates, 8, 0.7, 0.9, 0.6));
+            }
         }
         private AppSettings? LoadSettings()
         {
@@ -77,12 +80,18 @@ public async Task StartAsync(CancellationToken token)
                 await UploadImageToBlobStorage(2, _appSettings);
                 btnSphere.IsActive = false;
             };
-
+            Vector3 destination = new Vector3(-25, 0, 1);
+            var wall = (TransformNode)_app.Scene.FindChildByPath("QLearning/Wall");
             var npc1 = (TransformNode)_app.Scene.FindChildByPath("HumanMale_Character");
             var npc2 = (TransformNode)_app.Scene.FindChildByPath("HumanMale_Character1");
-            _qLearning1.MoveAction(npc1, 1000);
-            _qLearning2.MoveAction(npc2, 1000);
-            // Apply the chosen action to the NPC
+            var npc3 = (TransformNode)_app.Scene.FindChildByPath("HumanMale_Character2");
+            var npc4 = (TransformNode)_app.Scene.FindChildByPath("HumanMale_Character3");
+            var npc5 = (TransformNode)_app.Scene.FindChildByPath("HumanMale_Character4");
+            _qLearnings[0].MoveAction(npc1, destination, 0,  10000);
+            //_qLearnings[1].MoveAction(npc2, destination, 1, 100);
+            //_qLearnings[2].MoveAction(npc3, destination, 10000);
+            //_qLearnings[3].MoveAction(npc4, destination, 10000);
+            //_qLearnings[4].MoveAction(npc5, destination, 10000);
 
             //var move = 5;
             //_personFlow.Boucle(npc, move);

src/MIC/Assets/.MeshCloudScripting/Presentation1/QLearning.cs

@@ -1,14 +1,19 @@
 using System.Numerics;
 using Microsoft.Mesh.CloudScripting;
+using Newtonsoft.Json;
 
 namespace Presentation1
 {
     public class QLearning
     {
-        private double[,] qTable;
-        private int numStates, numActions;
-        private double learningRate, discountFactor, explorationRate;
-        private float lastDistance = 9999;
+        private double[,] qTable; // Table for the memory of the npc, works with a reward value
+        private double[] maxQValues; // Array that stores the maximum Q value for each state
+        private int[] maxQActions; // Array that the action with the max Q value
+        private int numStates, numActions; //numState : number of stats possible | numActions : number of actions posible
+        private double learningRate, discountFactor, explorationRate; // Parameters for the Q learning algorithm
+        private float lastDistance = 9999; // Stores the last distance from the choosen destination
+        private Random rnd = new Random();
+        const int GRID_SIZE = 60;
         public QLearning(int numStates, int numActions, double learningRate, double discountFactor, double explorationRate)
         {
             this.numStates = numStates;
@@ -18,103 +23,109 @@ public QLearning(int numStates, int numActions, double learningRate, double disc
             this.explorationRate = explorationRate;
 
             qTable = new double[numStates, numActions];
+            maxQValues = new double[numStates];
+            maxQActions = new int[numStates];
         }
 
+        // Class to choose the next action that the npc will do
         public int ChooseAction(int state)
         {
-            var rnd = new Random();
+            // Change the explorationRate to have a more explorating or a more exploiting npc
             if (rnd.NextDouble() < explorationRate)
             {
                 // Explore: select a random action
                 return rnd.Next(numActions);
             }
             else
             {
-                // Exploit: select the action with max value (greedy)
-                var maxVal = double.MinValue;
-                var maxAction = 0;
-                for (var action = 0; action < numActions; action++)
-                {
-                    if (qTable[state, action] > maxVal)
-                    {
-                        maxVal = qTable[state, action];
-                        maxAction = action;
-                    }
-                }
-                return maxAction;
+                // Exploit: select the action with max value
+                return maxQActions[state];
             }
         }
 
-        public void UpdateQValue(int prevState, int action, int reward, int nextState)
+        // Update the Q value in the Q table with the reward it gets
+        public void UpdateQValue(int prevState, int action, float reward, int nextState)
         {
-            var maxQ = double.MinValue;
-            for (var i = 0; i < numActions; i++)
+            // Bellman's Equation
+            qTable[prevState, action] += learningRate * (reward + discountFactor * maxQValues[nextState] - qTable[prevState, action]);
+            if (qTable[prevState, action] > maxQValues[prevState])
             {
-                if (qTable[nextState, i] > maxQ)
-                {
-                    maxQ = qTable[nextState, i];
-                }
+                maxQValues[prevState] = qTable[prevState, action];
+                maxQActions[prevState] = action;
             }
-
-            qTable[prevState, action] += learningRate * (reward + discountFactor * maxQ - qTable[prevState, action]);
         }
+
+        // Get the position/state of the npc
         public int GetState(TransformNode npc)
         {
-            // Assuming your grid is 10x10 and each cell is 1 unit
             int x = (int)Math.Round(npc.Position.X);
             int z = (int)Math.Round(npc.Position.Z);
 
             // Convert the 2D position to a single index
-            int state = Math.Abs(z * 10 + x);
-            while (state >= 100)
-            {
-                state = state/10;
-            }
+            int state = Math.Abs(z * 100 + x) % 100000;
 
             return state;
         }
 
-        public async void MoveAction(TransformNode npc, int numIterations)
+        // Main function that make the npc move and calls all the subfunctions
+        public async void MoveAction(TransformNode npc, Vector3 destination, int npcNum, int numIterations)
         {
+            float duration = 2f;
+            float stepSize = 0.01f / duration;
+            LoadQTable(npcNum);
+
+            bool[,] gridObstacles = GridObstacle();
+
+
             for (int i = 0; i < numIterations; i++)
             {
                 int prevState = GetState(npc);
                 int action = ChooseAction(prevState);
 
+                Vector3 direction = Vector3.Zero;
+
                 // Move the NPC based on the action
                 switch (action)
                 {
-                    case 0:  // Move up
-                        npc.Position = Vector3.Lerp(npc.Position, npc.Position + new Vector3(0, 0, 1), 0.6f);
-                        await Task.Delay(100);
-                        break;
-                    case 1:  // Move down
-                        npc.Position = Vector3.Lerp(npc.Position, npc.Position + new Vector3(0, 0, -1), 0.6f);
-                        await Task.Delay(100);
-                        break;
-                    case 2:  // Move left
-                        npc.Position = Vector3.Lerp(npc.Position, npc.Position + new Vector3(-1, 0, 0), 0.6f);
-                        await Task.Delay(100);
-                        break;
-                    case 3:  // Move right
-                        npc.Position = Vector3.Lerp(npc.Position, npc.Position + new Vector3(1, 0, 0), 0.6f);
-                        await Task.Delay(100);
-                        break;
+                    case 0: direction = new Vector3(0, 0, 2); break; // Move up
+                    case 1: direction = new Vector3(0, 0, -2); break; // Move down
+                    case 2: direction = new Vector3(-2, 0, 0); break; // Move left  
+                    case 3: direction = new Vector3(2, 0, 0); break; // Move right
+                    case 4: direction = new Vector3(1, 0f, 1); break; // Move up-right
+                    case 5: direction = new Vector3(-1, 0f, 1); break; // Move up-left
+                    case 6: direction = new Vector3(1, 0f, -1); break; // Move down-right
+                    case 7: direction = new Vector3(-1, 0f, -1); break; // Move down-left
+                }
+
+                Vector3 desiredPosition = npc.Position + direction;
+                if (desiredPosition.X >= -GRID_SIZE / 2 && desiredPosition.X < GRID_SIZE / 2 && desiredPosition.Z >= -GRID_SIZE / 2 && desiredPosition.Z < GRID_SIZE / 2 && !gridObstacles[(int)desiredPosition.X + GRID_SIZE/2, (int)desiredPosition.Z+ GRID_SIZE/2])
+                {
+                        float t = 0f;
+                    while (t < 1f)
+                    {
+                        npc.Position = Vector3.Lerp(npc.Position, desiredPosition, t);
+                        t += stepSize;
+                        if (t > 1f)
+                            t = 1f;
+                        //await Task.Delay(1);
+                    }
                 }
 
                 // Calculate the reward
-                int reward = CalculateReward(npc);
+                float reward = CalculateReward(npc, destination);
 
                 int nextState = GetState(npc);
 
                 // Update the Q-value
                 UpdateQValue(prevState, action, reward, nextState);
+                SaveQTable(npcNum);
             }
         }
-        public int CalculateReward(TransformNode npc)
+
+        // Calculate the reward for the movement with the distance of the final destination
+        public int CalculateReward(TransformNode npc, Vector3 destination)
         {
-            Vector3 goalPosition = new Vector3(6, 0, -10);
-            float distance = Vector3.Distance(npc.Position, goalPosition);
+            float distance = Vector3.Distance(npc.Position, destination);
 
             if (distance == 0)
             {
@@ -130,5 +141,74 @@ public int CalculateReward(TransformNode npc)
                 return 1;
             }
         }
+
+        // At the end of the movement, save the Q table in a json to exploit it at the next launchs 
+        public void SaveQTable(int npcNum)
+        {
+            var qTableList = new List<List<double>>();
+            for (int i = 0; i < numStates; i++)
+            {
+                var row = new List<double>();
+                for (int j = 0; j < numActions; j++)
+                {
+                    row.Add(qTable[i, j]);
+                }
+                qTableList.Add(row);
+            }
+            string finalFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "qtable" + npcNum + ".json"); ;
+            File.WriteAllText(finalFilePath, JsonConvert.SerializeObject(qTableList));
+        }
+
+
+        // Load the Q table 
+        public void LoadQTable(int npcNum)
+        {
+            string finalFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "qtable" + npcNum + ".json");
+            if (File.Exists(finalFilePath))
+            {
+                var qTableList = JsonConvert.DeserializeObject<List<List<double>>>(File.ReadAllText(finalFilePath));
+                for (int i = 0; i < numStates; i++)
+                {
+                    for (int j = 0; j < numActions; j++)
+                    {
+                        qTable[i, j] = qTableList[i][j];
+                    }
+                }
+            }
+        }
+
+        public bool[,] GridObstacle()
+        {
+            bool[,] grid = new bool[GRID_SIZE, GRID_SIZE];
+            for (int x = 0; x < GRID_SIZE; x++)
+            {
+                for (int z = 0; z < GRID_SIZE; z++)
+                {
+                    grid[x, z] = false;
+                }
+            }
+
+            // Add walls
+
+            // Add walls
+            for (int z = -10; z <= 10; z++)
+            {
+                grid[0 + GRID_SIZE / 2, z + GRID_SIZE / 2] = true; 
+                grid[1 + GRID_SIZE / 2, z + GRID_SIZE / 2] = true; 
+                grid[-29 + GRID_SIZE / 2, z + GRID_SIZE / 2] = true; 
+                grid[-30 + GRID_SIZE / 2, z + GRID_SIZE / 2] = true; 
+            }
+
+            for (int x = -30; x < 0; x++)
+            {
+                grid[x + GRID_SIZE / 2, 10 + GRID_SIZE / 2] = true; 
+                grid[x + GRID_SIZE / 2, 11 + GRID_SIZE / 2] = true; 
+                grid[x + GRID_SIZE / 2, -10 + GRID_SIZE / 2] = true; 
+                grid[x + GRID_SIZE / 2, -11 + GRID_SIZE / 2] = true; 
+            }
+
+            return grid;
+        }
     }
 }
+

src/MIC/Assets/.MeshCloudScripting/Presentation1/bin/Debug/net6.0/mesh.cloudscripting.manifest.json

@@ -20,7 +20,7 @@
   "wasLastProvisionAppSuccessful": "True",
   "wasLastDeployAppSuccessful": null,
   "wasLastUploadBlobSuccessful": "True",
-  "meshAppBuildId": "4bf10613-2c77-41e2-9924-1974d1a93121",
+  "meshAppBuildId": "64506e35-288e-44f9-87ff-5a7c2db0cb54",
   "debugTimeoutSecs": 120,
   "provisionState": "Inactive"
 }

src/MIC/Assets/.MeshCloudScripting/Presentation1/bin/Release/net6.0/mesh.cloudscripting.manifest.json

@@ -20,7 +20,7 @@
   "wasLastProvisionAppSuccessful": "True",
   "wasLastDeployAppSuccessful": null,
   "wasLastUploadBlobSuccessful": "False",
-  "meshAppBuildId": "c4262d60-4741-4e17-82fa-87867675e378",
+  "meshAppBuildId": "64506e35-288e-44f9-87ff-5a7c2db0cb54",
   "debugTimeoutSecs": 120,
   "provisionState": "Inactive"
 }