Fix duplicated time in result set bug

integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java

@@ -24,8 +24,6 @@
 import org.apache.iotdb.isession.SessionDataSet;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
-import org.apache.iotdb.itbase.category.ClusterIT;
-import org.apache.iotdb.itbase.category.LocalStandaloneIT;
 import org.apache.iotdb.itbase.category.ManualIT;
 import org.apache.iotdb.itbase.category.TableClusterIT;
 import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
@@ -1647,7 +1645,6 @@ private List<TimeRange> collectDataRanges(Statement statement, long timeUpperBou
   }
 
   @Test
-  @Category({LocalStandaloneIT.class, ClusterIT.class})
   public void deleteTableOfTheSameNameTest()
       throws IoTDBConnectionException, StatementExecutionException {
     int testNum = 24;

integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.query.recent;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
+import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBNullValueIT {
+  private static final String DATABASE_NAME = "test";
+
+  private static final String[] createSqls =
+      new String[] {
+        "CREATE DATABASE " + DATABASE_NAME,
+        "USE " + DATABASE_NAME,
+        "create table table1(id1 tag, s1 string)",
+        "insert into table1 values(0, 'd1', null), (1,'d1', 1)",
+        "flush",
+        "insert into table1 values(0, 'd1', 0)",
+        "flush"
+      };
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false);
+    EnvFactory.getEnv().initClusterEnvironment();
+    prepareTableData(createSqls);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    EnvFactory.getEnv().cleanClusterEnvironment();
+  }
+
+  @Test
+  public void nullTest() {
+
+    // case 1: all without time filter using previous fill without timeDuration
+    String[] expectedHeader = new String[] {"time", "id1", "s1"};
+    String[] retArray =
+        new String[] {
+          "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,",
+        };
+    tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME);
+  }
+}

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java

@@ -41,7 +41,7 @@
 import org.apache.tsfile.file.metadata.IChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
-import org.apache.tsfile.file.metadata.TableDeviceMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
 import org.apache.tsfile.file.metadata.TimeseriesMetadata;
 import org.apache.tsfile.read.controller.IChunkLoader;
 import org.apache.tsfile.read.filter.basic.Filter;
@@ -395,7 +395,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications(
     AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata =
         ignoreAllNullRows
             ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList)
-            : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList);
+            : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList);
 
     alignedTimeSeriesMetadata.setChunkMetadataLoader(
         new DiskAlignedChunkMetadataLoader(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java

@@ -37,12 +37,16 @@
 import org.apache.iotdb.db.utils.datastructure.TVList;
 
 import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
 import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
 import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
 import org.apache.tsfile.file.metadata.IChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata;
+import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
 import org.apache.tsfile.file.metadata.TimeseriesMetadata;
 import org.apache.tsfile.file.metadata.enums.CompressionType;
 import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -108,7 +112,7 @@ public AlignedResourceByPathUtils(IFullPath fullPath) {
    * have chunkMetadata, but query will use these, so we need to generate it for them.
    */
   @Override
-  public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
+  public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata(
       List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList) {
     TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata();
     timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
@@ -131,8 +135,11 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
 
     boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()];
     boolean modified = false;
+    boolean isTable = false;
     for (IChunkMetadata chunkMetadata : chunkMetadataList) {
-      AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata;
+      AbstractAlignedChunkMetadata alignedChunkMetadata =
+          (AbstractAlignedChunkMetadata) chunkMetadata;
+      isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata);
       modified = (modified || alignedChunkMetadata.isModified());
       timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
       for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
@@ -149,8 +156,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
 
     for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
       if (!memChunk.isEmpty()) {
-        AlignedChunkMetadata alignedChunkMetadata =
-            (AlignedChunkMetadata) memChunk.getChunkMetaData();
+        AbstractAlignedChunkMetadata alignedChunkMetadata =
+            (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData();
+        isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata);
         timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
         for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
           if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
@@ -174,7 +182,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
       }
     }
 
-    return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList);
+    return isTable
+        ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList)
+        : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList);
   }
 
   @Override
@@ -268,7 +278,7 @@ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
       QueryContext context,
       long timeLowerBound) {
 
-    List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
+    List<AbstractAlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
     List<ChunkMetadata> timeChunkMetadataList =
         writer.getVisibleMetadataList(
             alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR);
@@ -285,7 +295,10 @@ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
       // only need time column
       if (alignedFullPath.getMeasurementList().isEmpty()) {
         chunkMetadataList.add(
-            new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()));
+            context.isIgnoreAllNullRows()
+                ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())
+                : new TableDeviceChunkMetadata(
+                    timeChunkMetadataList.get(i), Collections.emptyList()));
       } else {
         List<IChunkMetadata> valueChunkMetadata = new ArrayList<>();
         // if all the sub sensors doesn't exist, it will be false
@@ -298,7 +311,9 @@ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
         }
         if (!context.isIgnoreAllNullRows() || exits) {
           chunkMetadataList.add(
-              new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata));
+              context.isIgnoreAllNullRows()
+                  ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)
+                  : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata));
         }
       }
     }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java

@@ -36,7 +36,7 @@
 
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.apache.tsfile.read.TsFileSequenceReader;
@@ -179,8 +179,9 @@ private void compactAlignedSeries(
       MultiTsFileDeviceIterator deviceIterator)
       throws IOException, InterruptedException, IllegalPathException, PageException {
     checkThreadInterrupted();
-    LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>> readerAndChunkMetadataList =
-        deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
+    LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
+        readerAndChunkMetadataList =
+            deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
     if (!checkAlignedSeriesExists(readerAndChunkMetadataList)) {
       return;
     }
@@ -212,9 +213,9 @@ private void checkThreadInterrupted() throws InterruptedException {
   }
 
   private boolean checkAlignedSeriesExists(
-      LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+      LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
           readerAndChunkMetadataList) {
-    for (Pair<TsFileSequenceReader, List<AlignedChunkMetadata>> readerListPair :
+    for (Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>> readerListPair :
         readerAndChunkMetadataList) {
       if (!readerListPair.right.isEmpty()) {
         return true;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java

@@ -34,7 +34,7 @@
 import org.apache.iotdb.db.utils.ModificationUtils;
 
 import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
 import org.apache.tsfile.file.metadata.IChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
@@ -384,14 +384,14 @@ public Map<String, MeasurementSchema> getAllSchemasOfCurrentDevice() throws IOEx
    * @throws IOException if io errors occurred
    */
   @SuppressWarnings({"squid:S1319", "squid:S135"})
-  public LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
+  public LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
       getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException, IllegalPathException {
     if (currentDevice == null || !currentDevice.right) {
       return new LinkedList<>();
     }
 
-    LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>> readerAndChunkMetadataList =
-        new LinkedList<>();
+    LinkedList<Pair<TsFileSequenceReader, List<AbstractAlignedChunkMetadata>>>
+        readerAndChunkMetadataList = new LinkedList<>();
     for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) {
       if (!deviceIteratorMap.containsKey(tsFileResource)) {
         continue;
@@ -403,7 +403,7 @@ public Map<String, MeasurementSchema> getAllSchemasOfCurrentDevice() throws IOEx
       MetadataIndexNode firstMeasurementNodeOfCurrentDevice =
           iterator.getFirstMeasurementNodeOfCurrentDevice();
       TsFileSequenceReader reader = readerMap.get(tsFileResource);
-      List<AlignedChunkMetadata> alignedChunkMetadataList =
+      List<AbstractAlignedChunkMetadata> alignedChunkMetadataList =
           reader.getAlignedChunkMetadataByMetadataIndexNode(
               currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows);
       applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList);
@@ -420,7 +420,7 @@ public Map<String, MeasurementSchema> getAllSchemasOfCurrentDevice() throws IOEx
    * @param alignedChunkMetadataList list of aligned chunk metadata
    */
   private void applyModificationForAlignedChunkMetadataList(
-      TsFileResource tsFileResource, List<AlignedChunkMetadata> alignedChunkMetadataList)
+      TsFileResource tsFileResource, List<AbstractAlignedChunkMetadata> alignedChunkMetadataList)
       throws IllegalPathException {
     if (alignedChunkMetadataList.isEmpty()) {
       // all the value chunks is empty chunk
@@ -437,7 +437,7 @@ private void applyModificationForAlignedChunkMetadataList(
             tsFileResource, r -> new ArrayList<>(tsFileResource.getAllModEntries()));
 
     // construct the input params List<List<Modification>> for QueryUtils.modifyAlignedChunkMetaData
-    AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0);
+    AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0);
     List<IChunkMetadata> valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList();
 
     // match time column modifications

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java

@@ -43,7 +43,7 @@
 
 import org.apache.tsfile.common.constant.TsFileConstant;
 import org.apache.tsfile.exception.write.PageException;
-import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
 import org.apache.tsfile.file.metadata.IChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
@@ -72,7 +72,7 @@ public class BatchedFastAlignedSeriesCompactionExecutor
   private final List<IMeasurementSchema> valueMeasurementSchemas;
   private final List<TsFileResource> sortedSourceFiles;
 
-  private final Map<TsFileResource, List<AlignedChunkMetadata>> alignedChunkMetadataCache;
+  private final Map<TsFileResource, List<AbstractAlignedChunkMetadata>> alignedChunkMetadataCache;
   private final BatchCompactionPlan batchCompactionPlan;
   private final int batchSize =
       IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch();
@@ -110,11 +110,11 @@ public BatchedFastAlignedSeriesCompactionExecutor(
     this.batchCompactionPlan = new BatchCompactionPlan();
   }
 
-  private List<AlignedChunkMetadata> getAlignedChunkMetadataListBySelectedValueColumn(
+  private List<AbstractAlignedChunkMetadata> getAlignedChunkMetadataListBySelectedValueColumn(
       TsFileResource tsFileResource, List<IMeasurementSchema> selectedValueMeasurementSchemas)
       throws IOException, IllegalPathException {
     // 1. get Full AlignedChunkMetadata from cache
-    List<AlignedChunkMetadata> alignedChunkMetadataList = null;
+    List<AbstractAlignedChunkMetadata> alignedChunkMetadataList = null;
     if (alignedChunkMetadataCache.containsKey(tsFileResource)) {
       alignedChunkMetadataList = alignedChunkMetadataCache.get(tsFileResource);
     } else {
@@ -124,8 +124,8 @@ private List<AlignedChunkMetadata> getAlignedChunkMetadataListBySelectedValueCol
     }
     // 2. generate AlignedChunkMetadata list by selected value columns
 
-    List<AlignedChunkMetadata> filteredAlignedChunkMetadataList = new ArrayList<>();
-    for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) {
+    List<AbstractAlignedChunkMetadata> filteredAlignedChunkMetadataList = new ArrayList<>();
+    for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) {
       filteredAlignedChunkMetadataList.add(
           AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex(
               alignedChunkMetadata, batchColumnSelection.getSelectedColumnIndexList()));
@@ -264,8 +264,8 @@ public void execute()
     }
 
     @Override
-    protected List<AlignedChunkMetadata> getAlignedChunkMetadataList(TsFileResource resource)
-        throws IOException, IllegalPathException {
+    protected List<AbstractAlignedChunkMetadata> getAlignedChunkMetadataList(
+        TsFileResource resource) throws IOException, IllegalPathException {
       return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas);
     }
 
@@ -301,10 +301,10 @@ protected ModifiedStatus isPageModified(PageElement pageElement) {
       IChunkMetadata batchedAlignedChunkMetadata =
           alignedPageElement.getChunkMetadataElement().chunkMetadata;
       TsFileResource resource = alignedPageElement.getChunkMetadataElement().fileElement.resource;
-      List<AlignedChunkMetadata> alignedChunkMetadataListOfFile =
+      List<AbstractAlignedChunkMetadata> alignedChunkMetadataListOfFile =
           alignedChunkMetadataCache.get(resource);
-      AlignedChunkMetadata originAlignedChunkMetadata = null;
-      for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) {
+      AbstractAlignedChunkMetadata originAlignedChunkMetadata = null;
+      for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) {
         if (alignedChunkMetadata.getOffsetOfChunkHeader()
             == batchedAlignedChunkMetadata.getOffsetOfChunkHeader()) {
           originAlignedChunkMetadata = alignedChunkMetadata;
@@ -383,8 +383,8 @@ protected ModifiedStatus isPageModified(PageElement pageElement) {
     }
 
     @Override
-    protected List<AlignedChunkMetadata> getAlignedChunkMetadataList(TsFileResource resource)
-        throws IOException, IllegalPathException {
+    protected List<AbstractAlignedChunkMetadata> getAlignedChunkMetadataList(
+        TsFileResource resource) throws IOException, IllegalPathException {
       return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas);
     }