diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 95eee41d007a..2449a35653cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -24,8 +24,6 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.itbase.category.ManualIT; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; @@ -1647,7 +1645,6 @@ private List collectDataRanges(Statement statement, long timeUpperBou } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void deleteTableOfTheSameNameTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 24; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java new file mode 100644 index 000000000000..531d88101ba6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullValueIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table table1(id1 tag, s1 string)", + "insert into table1 values(0, 'd1', null), (1,'d1', 1)", + "flush", + "insert into table1 values(0, 'd1', 0)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullTest() { + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = new String[] {"time", "id1", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,", + }; + tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index cd00aa5e4196..213e4990a5f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -41,7 +41,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; @@ -395,7 +395,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = ignoreAllNullRows ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList) - : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList); + : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList); alignedTimeSeriesMetadata.setChunkMetadataLoader( new DiskAlignedChunkMetadataLoader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index dbd32b47276f..12393fb2a209 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,12 +37,16 @@ import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -108,7 +112,7 @@ public AlignedResourceByPathUtils(IFullPath fullPath) { * have chunkMetadata, but query will use these, so we need to generate it for them. */ @Override - public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( + public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( List readOnlyMemChunk, List chunkMetadataList) { TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); @@ -131,8 +135,11 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; + boolean isTable = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadata; + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); modified = (modified || alignedChunkMetadata.isModified()); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -149,8 +156,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { if (!memChunk.isEmpty()) { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) memChunk.getChunkMetaData(); + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData(); + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { @@ -174,7 +182,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( } } - return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + return isTable + ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList) + : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); } @Override @@ -268,7 +278,7 @@ public List getVisibleMetadataListFromWriter( QueryContext context, long timeLowerBound) { - List chunkMetadataList = new ArrayList<>(); + List chunkMetadataList = new ArrayList<>(); List timeChunkMetadataList = writer.getVisibleMetadataList( alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); @@ -285,7 +295,10 @@ public List getVisibleMetadataListFromWriter( // only need time column if (alignedFullPath.getMeasurementList().isEmpty()) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()) + : new TableDeviceChunkMetadata( + timeChunkMetadataList.get(i), Collections.emptyList())); } else { List valueChunkMetadata = new ArrayList<>(); // if all the sub sensors doesn't exist, it will be false @@ -298,7 +311,9 @@ public List getVisibleMetadataListFromWriter( } if (!context.isIgnoreAllNullRows() || exits) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata) + : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index ca5929399767..c86c02e51b2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -36,7 +36,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; @@ -179,8 +179,9 @@ private void compactAlignedSeries( MultiTsFileDeviceIterator deviceIterator) throws IOException, InterruptedException, IllegalPathException, PageException { checkThreadInterrupted(); - LinkedList>> readerAndChunkMetadataList = - deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + LinkedList>> + readerAndChunkMetadataList = + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); if (!checkAlignedSeriesExists(readerAndChunkMetadataList)) { return; } @@ -212,9 +213,9 @@ private void checkThreadInterrupted() throws InterruptedException { } private boolean checkAlignedSeriesExists( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { if (!readerListPair.right.isEmpty()) { return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 32a1593c4a65..1c8aed4ca62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -384,14 +384,14 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @throws IOException if io errors occurred */ @SuppressWarnings({"squid:S1319", "squid:S135"}) - public LinkedList>> + public LinkedList>> getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException, IllegalPathException { if (currentDevice == null || !currentDevice.right) { return new LinkedList<>(); } - LinkedList>> readerAndChunkMetadataList = - new LinkedList<>(); + LinkedList>> + readerAndChunkMetadataList = new LinkedList<>(); for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) { if (!deviceIteratorMap.containsKey(tsFileResource)) { continue; @@ -403,7 +403,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx MetadataIndexNode firstMeasurementNodeOfCurrentDevice = iterator.getFirstMeasurementNodeOfCurrentDevice(); TsFileSequenceReader reader = readerMap.get(tsFileResource); - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode( currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows); applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList); @@ -420,7 +420,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @param alignedChunkMetadataList list of aligned chunk metadata */ private void applyModificationForAlignedChunkMetadataList( - TsFileResource tsFileResource, List alignedChunkMetadataList) + TsFileResource tsFileResource, List alignedChunkMetadataList) throws IllegalPathException { if (alignedChunkMetadataList.isEmpty()) { // all the value chunks is empty chunk @@ -437,7 +437,7 @@ private void applyModificationForAlignedChunkMetadataList( tsFileResource, r -> new ArrayList<>(tsFileResource.getAllModEntries())); // construct the input params List> for QueryUtils.modifyAlignedChunkMetaData - AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); + AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); // match time column modifications diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java index c2044da7f630..636fba8c2ca8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java @@ -43,7 +43,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -72,7 +72,7 @@ public class BatchedFastAlignedSeriesCompactionExecutor private final List valueMeasurementSchemas; private final List sortedSourceFiles; - private final Map> alignedChunkMetadataCache; + private final Map> alignedChunkMetadataCache; private final BatchCompactionPlan batchCompactionPlan; private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); @@ -110,11 +110,11 @@ public BatchedFastAlignedSeriesCompactionExecutor( this.batchCompactionPlan = new BatchCompactionPlan(); } - private List getAlignedChunkMetadataListBySelectedValueColumn( + private List getAlignedChunkMetadataListBySelectedValueColumn( TsFileResource tsFileResource, List selectedValueMeasurementSchemas) throws IOException, IllegalPathException { // 1. get Full AlignedChunkMetadata from cache - List alignedChunkMetadataList = null; + List alignedChunkMetadataList = null; if (alignedChunkMetadataCache.containsKey(tsFileResource)) { alignedChunkMetadataList = alignedChunkMetadataCache.get(tsFileResource); } else { @@ -124,8 +124,8 @@ private List getAlignedChunkMetadataListBySelectedValueCol } // 2. generate AlignedChunkMetadata list by selected value columns - List filteredAlignedChunkMetadataList = new ArrayList<>(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List filteredAlignedChunkMetadataList = new ArrayList<>(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { filteredAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, batchColumnSelection.getSelectedColumnIndexList())); @@ -264,8 +264,8 @@ public void execute() } @Override - protected List getAlignedChunkMetadataList(TsFileResource resource) - throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList( + TsFileResource resource) throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } @@ -301,10 +301,10 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { IChunkMetadata batchedAlignedChunkMetadata = alignedPageElement.getChunkMetadataElement().chunkMetadata; TsFileResource resource = alignedPageElement.getChunkMetadataElement().fileElement.resource; - List alignedChunkMetadataListOfFile = + List alignedChunkMetadataListOfFile = alignedChunkMetadataCache.get(resource); - AlignedChunkMetadata originAlignedChunkMetadata = null; - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { + AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == batchedAlignedChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -383,8 +383,8 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { } @Override - protected List getAlignedChunkMetadataList(TsFileResource resource) - throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList( + TsFileResource resource) throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java index c0d16841ab10..e343ea59b929 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java @@ -38,7 +38,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -67,13 +67,14 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); private final AlignedSeriesBatchCompactionUtils.BatchColumnSelection batchColumnSelection; - private final LinkedList>> + private final LinkedList>> originReaderAndChunkMetadataList; public BatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -113,7 +114,7 @@ private void compactFirstBatch() throws IOException, PageException { selectedColumnSchemaList = batchColumnSelection.getCurrentSelectedColumnSchemaList(); } - LinkedList>> + LinkedList>> batchedReaderAndChunkMetadataList = filterAlignedChunkMetadataList(readerAndChunkMetadataList, selectedColumnIndexList); @@ -137,7 +138,7 @@ private void compactFirstBatch() throws IOException, PageException { private void compactLeftBatches() throws PageException, IOException { while (batchColumnSelection.hasNext()) { batchColumnSelection.next(); - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = filterAlignedChunkMetadataList( readerAndChunkMetadataList, batchColumnSelection.getSelectedColumnIndexList()); @@ -155,16 +156,19 @@ private void compactLeftBatches() throws PageException, IOException { } } - private LinkedList>> + private LinkedList>> filterAlignedChunkMetadataList( - List>> readerAndChunkMetadataList, + List>> + readerAndChunkMetadataList, List selectedMeasurementIndexs) { - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = new LinkedList<>(); - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); - List selectedColumnAlignedChunkMetadataList = new LinkedList<>(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + List selectedColumnAlignedChunkMetadataList = + new LinkedList<>(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { selectedColumnAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, selectedMeasurementIndexs)); @@ -181,7 +185,7 @@ public class FirstBatchedReadChunkAlignedSeriesCompactionExecutor public FirstBatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, @@ -224,8 +228,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu String file = timePage.getFile(); ChunkMetadata timeChunkMetadata = timePage.getChunkMetadata(); - List alignedChunkMetadataList = Collections.emptyList(); - for (Pair> pair : + List alignedChunkMetadataList = Collections.emptyList(); + for (Pair> pair : originReaderAndChunkMetadataList) { TsFileSequenceReader reader = pair.getLeft(); if (reader.getFileName().equals(file)) { @@ -234,8 +238,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu } } - AlignedChunkMetadata originAlignedChunkMetadata = null; - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == timeChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -303,7 +307,7 @@ public class FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor public FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java index 25613fc2a1a8..c3c58e5bb6a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -43,17 +43,18 @@ public class AlignedSeriesBatchCompactionUtils { private AlignedSeriesBatchCompactionUtils() {} public static void markAlignedChunkHasDeletion( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); markAlignedChunkHasDeletion(alignedChunkMetadataList); } } public static void markAlignedChunkHasDeletion( - List alignedChunkMetadataList) { - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); for (IChunkMetadata iChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { if (iChunkMetadata != null && iChunkMetadata.isModified()) { @@ -68,8 +69,8 @@ public static boolean isTimeChunk(ChunkMetadata chunkMetadata) { return chunkMetadata.getMeasurementUid().isEmpty(); } - public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex( - AlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { + public static AbstractAlignedChunkMetadata filterAlignedChunkMetadataByIndex( + AbstractAlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { IChunkMetadata[] valueChunkMetadataArr = new IChunkMetadata[selectedMeasurements.size()]; List originValueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); @@ -77,12 +78,13 @@ public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex( int columnIndex = selectedMeasurements.get(i); valueChunkMetadataArr[i] = originValueChunkMetadataList.get(columnIndex); } - return new AlignedChunkMetadata( + return alignedChunkMetadata.createNewChunkMetadata( alignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(valueChunkMetadataArr)); } - public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( - AlignedChunkMetadata originAlignedChunkMetadata, List schemaList) { + public static AbstractAlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( + AbstractAlignedChunkMetadata originAlignedChunkMetadata, + List schemaList) { List originValueChunkMetadataList = originAlignedChunkMetadata.getValueChunkMetadataList(); IChunkMetadata[] newValueChunkMetadataArr = new IChunkMetadata[schemaList.size()]; @@ -109,14 +111,14 @@ public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( currentValueChunkMetadataIndex++; } } - return new AlignedChunkMetadata( + return originAlignedChunkMetadata.createNewChunkMetadata( originAlignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(newValueChunkMetadataArr)); } public static ModifiedStatus calculateAlignedPageModifiedStatus( long startTime, long endTime, - AlignedChunkMetadata originAlignedChunkMetadata, + AbstractAlignedChunkMetadata originAlignedChunkMetadata, boolean ignoreAllNullRows) { ModifiedStatus timePageModifiedStatus = checkIsModified( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 64fb585542d7..b489630894bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -44,10 +44,12 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; @@ -155,7 +157,8 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) throws IOException, IllegalPathException { for (FileElement fileElement : fileElements) { TsFileResource resource = fileElement.resource; - List alignedChunkMetadataList = getAlignedChunkMetadataList(resource); + List alignedChunkMetadataList = + getAlignedChunkMetadataList(resource); if (alignedChunkMetadataList.isEmpty()) { // all chunks have been deleted in this file or current file does not contain this aligned @@ -174,7 +177,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) } } - protected List getAlignedChunkMetadataList(TsFileResource resource) + protected List getAlignedChunkMetadataList(TsFileResource resource) throws IOException, IllegalPathException { // read time chunk metadatas and value chunk metadatas in the current file List timeChunkMetadatas = null; @@ -216,7 +219,7 @@ protected List getAlignedChunkMetadataList(TsFileResource } } - List alignedChunkMetadataList = new ArrayList<>(); + List alignedChunkMetadataList = new ArrayList<>(); // if current file contains this aligned device,then construct aligned chunk metadatas if (timeChunkMetadatas != null) { for (int i = 0; i < timeChunkMetadatas.size(); i++) { @@ -228,8 +231,10 @@ protected List getAlignedChunkMetadataList(TsFileResource valueChunkMetadataList.add(chunkMetadata.get(i)); } } - AlignedChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); + AbstractAlignedChunkMetadata alignedChunkMetadata = + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); alignedChunkMetadataList.add(alignedChunkMetadata); } @@ -345,8 +350,8 @@ protected void deserializeChunkIntoPageQueue(ChunkMetadataElement chunkMetadataE @Override void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException { updateSummary(chunkMetadataElement, ChunkStatus.READ_IN); - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; TsFileSequenceReader reader = readerCacheMap.get(chunkMetadataElement.fileElement.resource); chunkMetadataElement.chunk = readChunk(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); @@ -418,8 +423,8 @@ protected boolean flushPageToCompactionWriter(PageElement pageElement) protected ModifiedStatus isPageModified(PageElement pageElement) { long startTime = pageElement.getStartTime(); long endTime = pageElement.getEndTime(); - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; return AlignedSeriesBatchCompactionUtils.calculateAlignedPageModifiedStatus( startTime, endTime, alignedChunkMetadata, ignoreAllNullRows); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index 83dab2e76907..c2a5c6c7558f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; @@ -495,14 +495,14 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case READ_IN: summary.increaseProcessChunkNum( isAligned - ? ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ? ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList() .size() + 1 : 1); if (isAligned) { for (IChunkMetadata valueChunkMetadata : - ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList()) { if (valueChunkMetadata == null) { continue; @@ -517,7 +517,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DIRECTORY_FLUSH: if (isAligned) { summary.increaseDirectlyFlushChunkNum( - ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); @@ -528,7 +528,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DESERIALIZE_CHUNK: if (isAligned) { summary.increaseDeserializedChunkNum( - ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 12c205c472ed..17281c87051e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -39,7 +39,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -68,7 +68,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor { protected final IDeviceID device; - protected final LinkedList>> + protected final LinkedList>> readerAndChunkMetadataList; protected final TsFileResource targetResource; protected final CompactionTsFileWriter writer; @@ -85,7 +85,8 @@ public class ReadChunkAlignedSeriesCompactionExecutor { public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -108,7 +109,8 @@ public ReadChunkAlignedSeriesCompactionExecutor( public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, IMeasurementSchema timeSchema, @@ -131,11 +133,11 @@ public ReadChunkAlignedSeriesCompactionExecutor( private void collectValueColumnSchemaList() throws IOException { Map measurementSchemaMap = new HashMap<>(); for (int i = this.readerAndChunkMetadataList.size() - 1; i >= 0; i--) { - Pair> pair = + Pair> pair = this.readerAndChunkMetadataList.get(i); CompactionTsFileReader reader = (CompactionTsFileReader) pair.getLeft(); - List alignedChunkMetadataList = pair.getRight(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata == null) { continue; } @@ -176,10 +178,11 @@ private void collectValueColumnSchemaList() throws IOException { } private void fillAlignedChunkMetadataToMatchSchemaList() { - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); for (int i = 0; i < alignedChunkMetadataList.size(); i++) { - AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); + AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); alignedChunkMetadataList.set( i, AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( @@ -193,15 +196,15 @@ protected AlignedChunkWriterImpl constructAlignedChunkWriter() { } public void execute() throws IOException, PageException { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { TsFileSequenceReader reader = readerListPair.left; - List alignedChunkMetadataList = readerListPair.right; + List alignedChunkMetadataList = readerListPair.right; if (reader instanceof CompactionTsFileReader) { ((CompactionTsFileReader) reader).markStartOfAlignedSeries(); } - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { compactWithAlignedChunk(reader, alignedChunkMetadata); } if (reader instanceof CompactionTsFileReader) { @@ -215,7 +218,7 @@ public void execute() throws IOException, PageException { } private void compactWithAlignedChunk( - TsFileSequenceReader reader, AlignedChunkMetadata alignedChunkMetadata) + TsFileSequenceReader reader, AbstractAlignedChunkMetadata alignedChunkMetadata) throws IOException, PageException { ChunkLoader timeChunk = getChunkLoader(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java index ae5bc2e75911..d4b20039109e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -97,8 +97,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -132,8 +132,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java index e9709af3dbf6..39e621cfaf40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -89,8 +89,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -121,8 +121,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 569db7347ab5..a3eb66b018d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.TimeRange; @@ -75,10 +76,11 @@ public AlignedReadOnlyMemChunk( timeColumnDeletion, valueColumnsDeletionList, context.isIgnoreAllNullRows()); - initAlignedChunkMetaFromTsBlock(); + initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows()); } - private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { + private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) + throws QueryProcessException { // Time chunk Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR); IChunkMetadata timeChunkMetadata = @@ -160,7 +162,9 @@ private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { } } IChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadata, valueChunkMetadataList); alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context, this)); alignedChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = alignedChunkMetadata; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 3b2c5fe29449..b6f5629528fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -80,6 +80,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -1788,7 +1789,7 @@ RestorableTsFileIOWriter getWriter() { private void processAlignedChunkMetaDataFromFlushedMemTable( IDeviceID deviceID, - AlignedChunkMetadata alignedChunkMetadata, + AbstractAlignedChunkMetadata alignedChunkMetadata, Map> measurementToChunkMetaMap, Map> measurementToChunkHandleMap, String filePath) { @@ -1841,10 +1842,10 @@ private void buildChunkHandleForFlushedMemTable( Map> measurementToChunkMetaList, Map> measurementToChunkHandleList) { for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata instanceof AlignedChunkMetadata) { + if (chunkMetadata instanceof AbstractAlignedChunkMetadata) { processAlignedChunkMetaDataFromFlushedMemTable( deviceID, - (AlignedChunkMetadata) chunkMetadata, + (AbstractAlignedChunkMetadata) chunkMetadata, measurementToChunkMetaList, measurementToChunkHandleList, this.tsFileResource.getTsFilePath()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index adef246ec8d9..a7c6eb96d429 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -76,7 +76,8 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi throws IOException { long t1 = System.nanoTime(); try { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetaData; ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = ChunkCache.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java index 33c85408f0d3..12ac281dffc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; @@ -42,7 +42,7 @@ public MemAlignedChunkReader(AlignedReadOnlyMemChunk readableChunk, Filter globa Collections.singletonList( new MemAlignedPageReader( readableChunk.getTsBlock(), - (AlignedChunkMetadata) readableChunk.getChunkMetaData(), + (AbstractAlignedChunkMetadata) readableChunk.getChunkMetaData(), globalTimeFilter)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 473cd7366c33..6adfe09c0542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -22,7 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchDataFactory; @@ -45,7 +45,7 @@ public class MemAlignedPageReader implements IPageReader { private final TsBlock tsBlock; - private final AlignedChunkMetadata chunkMetadata; + private final AbstractAlignedChunkMetadata chunkMetadata; private Filter recordFilter; private PaginationController paginationController = UNLIMITED_PAGINATION_CONTROLLER; @@ -53,7 +53,7 @@ public class MemAlignedPageReader implements IPageReader { private TsBlockBuilder builder; public MemAlignedPageReader( - TsBlock tsBlock, AlignedChunkMetadata chunkMetadata, Filter recordFilter) { + TsBlock tsBlock, AbstractAlignedChunkMetadata chunkMetadata, Filter recordFilter) { this.tsBlock = tsBlock; this.chunkMetadata = chunkMetadata; this.recordFilter = recordFilter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java index afc57079c88b..f243b51ee1a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.read.controller.IChunkMetadataLoader; @@ -83,7 +83,7 @@ public DiskAlignedChunkMetadataLoader( public List loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) { final long t1 = System.nanoTime(); try { - List alignedChunkMetadataList = + List alignedChunkMetadataList = ((AbstractAlignedTimeSeriesMetadata) timeSeriesMetadata).getCopiedChunkMetadataList(); // when alignedChunkMetadataList.size() == 1, it means that the chunk statistics is same as diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java index 247a0a3fe828..57527c85af86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java @@ -106,7 +106,11 @@ public static TSStatus onQueryException(Exception e, String operation, TSStatusC || status.getCode() == TSStatusCode.ILLEGAL_PATH.getStatusCode() || status.getCode() == TSStatusCode.NUMERIC_VALUE_OUT_OF_RANGE.getStatusCode() || status.getCode() == TSStatusCode.DIVISION_BY_ZERO.getStatusCode() - || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode()) { + || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.COLUMN_ALREADY_EXISTS.getStatusCode()) { LOGGER.info(message); } else { LOGGER.warn(message, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 599662076480..f9a6d28e2b98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -101,7 +102,7 @@ public static void modifyAlignedChunkMetaData( } private static void modifyValueColumns( - AlignedChunkMetadata metaData, List> valueColumnsModifications) { + AbstractAlignedChunkMetadata metaData, List> valueColumnsModifications) { List valueChunkMetadataList = metaData.getValueChunkMetadataList(); // deal with each sub sensor for (int j = 0; j < valueChunkMetadataList.size(); j++) { @@ -116,7 +117,7 @@ private static void modifyValueColumns( } private static boolean areAllValueColumnsDeleted( - AlignedChunkMetadata alignedChunkMetadata, boolean modified) { + AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) { // the whole aligned path need to be removed, only set to be true if all the sub sensors // are deleted and ignoreAllNullRows is true @@ -156,11 +157,11 @@ private static boolean areAllValueColumnsDeleted( } public static void modifyAlignedChunkMetaData( - List chunkMetaData, + List chunkMetaData, List timeColumnModifications, List> valueColumnsModifications, boolean ignoreAllNullRows) { - for (AlignedChunkMetadata metaData : chunkMetaData) { + for (AbstractAlignedChunkMetadata metaData : chunkMetaData) { IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata(); for (ModEntry modification : timeColumnModifications) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java index 55cb8625da37..f62480c55291 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java @@ -32,7 +32,7 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -622,9 +622,9 @@ private void validateSingleTsFileWithAlignedSeries(TsFileSequenceReader reader) throws IOException { Map compressionTypeMap = new HashMap<>(); for (IDeviceID device : reader.getAllDevices()) { - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadata(device, true); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java index 8c9088f5cebc..f2537e9bfddf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java @@ -39,7 +39,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -2047,10 +2047,10 @@ public void testMergeAlignedSeriesTimeValuePairFromDifferentFiles() TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index a9c559a00a52..f2e39d8f8668 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -46,7 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -682,10 +682,10 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); @@ -796,7 +796,7 @@ public void testSplitChunk() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); Assert.assertEquals(3, chunkMetadataList.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java index ef804437361b..c257ee4ec203 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java @@ -34,6 +34,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -107,7 +108,7 @@ public void testBatchCompactionPointReader() throws IOException { true); try (TsFileSequenceReader reader = new TsFileSequenceReader(seqResource1.getTsFile().getAbsolutePath())) { - AlignedChunkMetadata alignedChunkMetadata = + AbstractAlignedChunkMetadata alignedChunkMetadata = reader .getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d0"), true) @@ -311,14 +312,14 @@ public void testMapAlignedChunkMetadata1() { new MeasurementSchema("s1", TSDataType.INT32), new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata = + AbstractAlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( + Arrays.asList("s0", "s1", "s2", "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(IChunkMetadata::getMeasurementUid) - .collect(Collectors.toList()), - Arrays.asList("s0", "s1", "s2", "s4")); + .collect(Collectors.toList())); } @Test @@ -333,14 +334,14 @@ public void testMapAlignedChunkMetadata2() { Arrays.asList( new MeasurementSchema("s0", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata = + AbstractAlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, "s4")); + .collect(Collectors.toList())); } @Test @@ -368,23 +369,23 @@ public void testMapAlignedChunkMetadata3() { new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata1 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( + Arrays.asList("s0", "s1", "s2", null, null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList("s0", "s1", "s2", null, null)); + .collect(Collectors.toList())); - AlignedChunkMetadata newAlignedChunkMetadata2 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, null, null, "s3", "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, null, null, "s3", "s4")); + .collect(Collectors.toList())); } @Test @@ -411,22 +412,22 @@ public void testMapAlignedChunkMetadata4() { Arrays.asList( new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata1 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( + Arrays.asList("s2", null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList("s2", null)); + .collect(Collectors.toList())); - AlignedChunkMetadata newAlignedChunkMetadata2 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, "s4")); + .collect(Collectors.toList())); } } diff --git a/pom.xml b/pom.xml index 40164fc32edf..1df4651c4d72 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ 0.14.1 1.9 1.5.6-3 - 2.0.0-250118-SNAPSHOT + 2.1.0-250206-SNAPSHOT