Merge pull request #689 from terascope/add-key-objects-bug

add_key nested object fix
terascope · May 26, 2023 · 7188eb9 · 7188eb9
2 parents 1bae985 + 53b3a1d
commit 7188eb9
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 13 deletions.
diff --git a/asset/asset.json b/asset/asset.json
@@ -1,5 +1,5 @@
 {
     "name": "standard",
-    "version": "0.21.1",
+    "version": "0.21.2",
     "description": "Teraslice standard processor asset bundle"
 }
diff --git a/asset/package.json b/asset/package.json
@@ -1,6 +1,6 @@
 {
     "name": "standard",
-    "version": "0.21.1",
+    "version": "0.21.2",
     "description": "Teraslice standard processor asset bundle",
     "license": "MIT",
     "private": true,

diff --git a/asset/src/add_key/processor.ts b/asset/src/add_key/processor.ts
@@ -11,7 +11,9 @@ import {
     toNumber,
     geoHash,
     setPrecision,
-    isGeoShapePoint
+    isGeoShapePoint,
+    isPlainObject,
+    flatten
 } from '@terascope/utils';
 import {
     GeoShapePoint,
@@ -251,14 +253,30 @@ export default class AddKey extends BatchProcessor {
     private createKey(keyValues: unknown[]) {
         const shasum = crypto.createHash(this.opConfig.hash_algorithm);
 
-        let key = '';
+        const key = keyValues
+            .map((value) => {
+                if (isPlainObject(value)) {
+                    return this.formatInnerObject(value as AnyObject);
+                }
 
-        keyValues.forEach((field) => {
-            key += field;
-        });
+                return value;
+            })
+            .join('');
 
         shasum.update(key);
 
         return shasum.digest('base64').replace(/=*$/g, '').replace(/\//g, '_').replace(/\+/g, '-');
     }
+
+    private formatInnerObject(value: AnyObject): string {
+        const prepped = JSON.stringify(value)
+            .split(',')
+            .map((x) => x.split('{'));
+
+        return flatten(prepped)
+            .filter((x) => x.length)
+            .sort()
+            .join('')
+            .replace(/"|}/g, '');
+    }
 }
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "standard-assets-bundle",
-    "version": "0.21.1",
+    "version": "0.21.2",
     "description": "Teraslice standard processor asset bundle",
     "private": true,
     "workspaces": [

diff --git a/test/add_key/processor-spec.ts b/test/add_key/processor-spec.ts
@@ -79,7 +79,7 @@ describe('key', () => {
                 name: 'joe',
                 age: 34,
                 foo: { bar: 'foo' },
-                _key: 'PBHKh3MmS9T0IjwvlEIXYw'
+                _key: 's_Az1xDGGOSRaVzbq1lk2Q'
             },
             {
                 name: 'frank',
@@ -265,7 +265,7 @@ describe('key', () => {
                     type: 'Point',
                     coordinates: [-43.4432343234, 55.3454349123934]
                 },
-                _key: 'ZmAUz-JlmID_QphbR9g9Rg'
+                _key: 'a7GxwT2WporvqjXqvWSYRw'
             }
         ]);
     });
@@ -457,19 +457,19 @@ describe('key', () => {
                 name: 'bob',
                 age: 122,
                 location: { lon: -43.4432343234, lat: 55.3454349123934 },
-                _key: 'ZmAUz-JlmID_QphbR9g9Rg'
+                _key: 'LvKzh7sFNReVjg405KP3eA'
             },
             {
                 name: 'joe',
                 age: 34,
                 location: { lon: -43.4432343234, lat: 55.3454349123934 },
-                _key: 'wwwfql7nsI-1P9td81Vm9A'
+                _key: 'yHNFJ-PUzlfbPzHgIrtjRg'
             },
             {
                 name: 'frank',
                 age: 99,
                 location: { lon: -43.4432343234, lat: 55.3454349123934 },
-                _key: 'LMKX1DswmPrKDq9SiG25nQ'
+                _key: 'dcTNShWJziZp7QKDNRUmjg'
             }
         ]);
     });
@@ -768,6 +768,69 @@ describe('key', () => {
         await expect(test.runSlice(data)).rejects.toThrow();
     });
 
+    it('should handle nested objects and consistently key objects in different order but same values', async () => {
+        const test = await makeTest();
+
+        const data = [
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    count: 0,
+                    foozer: { barzer: 1, a: 2 }
+                }
+            },
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    count: 1,
+                    foozer: { barzer: 2, a: 3 }
+                }
+            },
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    foozer: { a: 3, barzer: 2 },
+                    count: 1
+                }
+            }
+        ];
+
+        const results = await test.runSlice(data);
+
+        expect(results).toEqual([
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    count: 0,
+                    foozer: { barzer: 1, a: 2 }
+                },
+                _key: 'GfQnm6MUyVlVhjgKcMPdFw'
+            },
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    count: 1,
+                    foozer: { barzer: 2, a: 3 }
+                },
+                _key: '4WD27_nHjw8Si3W33j3yaA'
+            },
+            {
+                name: 'bob',
+                age: 122,
+                nested: {
+                    foozer: { a: 3, barzer: 2 },
+                    count: 1
+                },
+                _key: '4WD27_nHjw8Si3W33j3yaA'
+            }
+        ]);
+    });
+
     it('should not add empty fields or objects to key', async () => {
         const test = await makeTest();