Add IT for gapfill and fix some bugs

apache · Oct 19, 2024 · 502f64d · 502f64d
1 parent 37fe386
commit 502f64d
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 28 deletions.
diff --git a/...n-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java b/...n-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java
@@ -27,7 +27,6 @@
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -63,6 +62,7 @@ public class IoTDBGapFillTableIT {
 
   @BeforeClass
   public static void setUp() throws Exception {
+    EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024);
     EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false);
     EnvFactory.getEnv().initClusterEnvironment();
     prepareTableData(createSqls);
@@ -73,7 +73,6 @@ public static void tearDown() throws Exception {
     EnvFactory.getEnv().cleanClusterEnvironment();
   }
 
-  @Ignore
   @Test
   public void normalGapFillTest() {
 
@@ -118,14 +117,6 @@ public void normalGapFillTest() {
     expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"};
     retArray =
         new String[] {
-          "2024-09-24T04:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T05:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T06:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,",
-          "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,",
-          "2024-09-24T09:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T10:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,",
           "2024-09-24T04:00:00.000Z,beijing,d2,25.1,",
           "2024-09-24T05:00:00.000Z,beijing,d2,null,",
           "2024-09-24T06:00:00.000Z,beijing,d2,null,",
@@ -134,6 +125,14 @@ public void normalGapFillTest() {
           "2024-09-24T09:00:00.000Z,beijing,d2,null,",
           "2024-09-24T10:00:00.000Z,beijing,d2,null,",
           "2024-09-24T11:00:00.000Z,beijing,d2,28.2,",
+          "2024-09-24T04:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T05:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T06:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,",
+          "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,",
+          "2024-09-24T09:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T10:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,",
           "2024-09-24T04:00:00.000Z,shanghai,d3,null,",
           "2024-09-24T05:00:00.000Z,shanghai,d3,null,",
           "2024-09-24T06:00:00.000Z,shanghai,d3,null,",
@@ -265,7 +264,6 @@ public void normalGapFillTest() {
         DATABASE_NAME);
   }
 
-  @Ignore
   @Test
   public void gapFillWithFillClauseTest() {
 
@@ -309,14 +307,6 @@ public void gapFillWithFillClauseTest() {
     expectedHeader = new String[] {"hour_time", "city", "device_id", "avg_s1"};
     retArray =
         new String[] {
-          "2024-09-24T04:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T05:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T06:00:00.000Z,shanghai,d1,null,",
-          "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,",
-          "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,",
-          "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,",
-          "2024-09-24T10:00:00.000Z,shanghai,d1,27.3,",
-          "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,",
           "2024-09-24T04:00:00.000Z,beijing,d2,25.1,",
           "2024-09-24T05:00:00.000Z,beijing,d2,25.1,",
           "2024-09-24T06:00:00.000Z,beijing,d2,25.1,",
@@ -325,6 +315,14 @@ public void gapFillWithFillClauseTest() {
           "2024-09-24T09:00:00.000Z,beijing,d2,25.1,",
           "2024-09-24T10:00:00.000Z,beijing,d2,25.1,",
           "2024-09-24T11:00:00.000Z,beijing,d2,28.2,",
+          "2024-09-24T04:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T05:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T06:00:00.000Z,shanghai,d1,null,",
+          "2024-09-24T07:00:00.000Z,shanghai,d1,27.2,",
+          "2024-09-24T08:00:00.000Z,shanghai,d1,27.3,",
+          "2024-09-24T09:00:00.000Z,shanghai,d1,27.3,",
+          "2024-09-24T10:00:00.000Z,shanghai,d1,27.3,",
+          "2024-09-24T11:00:00.000Z,shanghai,d1,29.3,",
           "2024-09-24T04:00:00.000Z,shanghai,d3,null,",
           "2024-09-24T05:00:00.000Z,shanghai,d3,null,",
           "2024-09-24T06:00:00.000Z,shanghai,d3,null,",
@@ -375,11 +373,11 @@ public void gapFillWithFillClauseTest() {
           "2024-09-24T08:00:00.000Z,shanghai,d3,25.8,",
           "2024-09-24T09:00:00.000Z,shanghai,d3,25.8,",
           "2024-09-24T10:00:00.000Z,shanghai,d3,null,",
-          "2024-09-24T11:00:00.000Z,shanghai,d3,null",
+          "2024-09-24T11:00:00.000Z,shanghai,d3,null,",
         };
     // with time bound
     tableResultSetEqualTest(
-        "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS TIME_BOUND 1ms FILL_GROUP 2,3 order by 2,3,1",
+        "select date_bin_gapfill(1h, time) as hour_time, city, device_id, avg(s1) as avg_s1 from table1 where (time >= 2024-09-24T04:00:00.000+00:00 AND time < 2024-09-24T12:00:00.00+00:00) GROUP BY 1,2,3 FILL METHOD PREVIOUS TIME_BOUND 1h FILL_GROUP 2,3 order by 3,2,1",
         expectedHeader,
         retArray,
         DATABASE_NAME);

diff --git a/...ache/iotdb/db/queryengine/execution/operator/process/gapfill/AbstractGapFillOperator.java b/...ache/iotdb/db/queryengine/execution/operator/process/gapfill/AbstractGapFillOperator.java
@@ -116,6 +116,8 @@ public TsBlock next() throws Exception {
         }
         resetTimeIterator();
         previousGroupKey = currentGroupKey;
+      } else if (previousGroupKey == null) {
+        previousGroupKey = currentGroupKey;
       }
 
       Column timeColumn = block.getColumn(timeColumnIndex);

diff --git a/...e/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/...e/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
@@ -63,7 +63,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -677,11 +676,8 @@ private PlanBuilder gapFill(
       @Nonnull List<Expression> gapFillGroupingKeys,
       @Nonnull Expression wherePredicate) {
     Symbol gapFillColumnSymbol = subPlan.translate(gapFillColumn);
-    List<Symbol> groupingKeys = Collections.emptyList();
-    if (!gapFillGroupingKeys.isEmpty()) {
-      groupingKeys = new ArrayList<>(gapFillGroupingKeys.size());
-      subPlan = fillGroup(subPlan, gapFillGroupingKeys, groupingKeys, gapFillColumnSymbol);
-    }
+    List<Symbol> groupingKeys = new ArrayList<>(gapFillGroupingKeys.size());
+    subPlan = fillGroup(subPlan, gapFillGroupingKeys, groupingKeys, gapFillColumnSymbol);
 
     int monthDuration = (int) ((LongLiteral) gapFillColumn.getChildren().get(0)).getParsedValue();
     long nonMonthDuration = ((LongLiteral) gapFillColumn.getChildren().get(1)).getParsedValue();

diff --git a/...c/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/GapFillNode.java b/...c/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/GapFillNode.java
@@ -149,7 +149,6 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException {
     ReadWriteIOUtils.write(monthDuration, stream);
     ReadWriteIOUtils.write(nonMonthDuration, stream);
     Symbol.serialize(gapFillColumn, stream);
-    ReadWriteIOUtils.write(true, stream);
     ReadWriteIOUtils.write(gapFillGroupingKeys.size(), stream);
     for (Symbol symbol : gapFillGroupingKeys) {
       Symbol.serialize(symbol, stream);