Merge pull request #183 from chitara-01/b/310247478

Updated logic to process array data types in parquet
GoogleCloudPlatform · Nov 16, 2023 · 65fb1e4 · 65fb1e4
2 parents d02173f + 3c7ce08
commit 65fb1e4
Showing 1 changed file with 12 additions and 5 deletions.
diff --git a/src/main/java/com/google/swarm/tokenization/parquet/GenericRecordFlattener.java b/src/main/java/com/google/swarm/tokenization/parquet/GenericRecordFlattener.java
@@ -98,12 +98,19 @@ private void processType(Object value, Schema fieldSchema, String parentKey, Str
           break;
 
         case ARRAY:
-          List<?> arrayValues = (List<?>) value;
-          List<String> updatedValues = new ArrayList<>();
-          for (int index = 0; index < arrayValues.size(); index++) {
-            updatedValues.add(arrayValues.get(index).toString());
+          String listFieldKey =
+              isBlank(parentKey) ? fieldName : String.format("%s.%s", parentKey, fieldName);
+          if (value == null) {
+            putValue(listFieldKey, Value.newBuilder().getDefaultInstanceForType());
+          } else {
+            List<?> arrayValues = (List<?>) value;
+            List<String> updatedValues = new ArrayList<>();
+            for (int index = 0; index < arrayValues.size(); index++) {
+              updatedValues.add(arrayValues.get(index).toString());
+            }
+            putValue(
+                listFieldKey, Value.newBuilder().setStringValue(updatedValues.toString()).build());
           }
-          putValue(parentKey, Value.newBuilder().setStringValue(updatedValues.toString()).build());
           break;
 
         case UNION: