From a5fd903282ce80c27a9731bc9d7f55fe7e6f8c4f Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Sun, 26 Jan 2025 09:29:55 +0800 Subject: [PATCH 1/3] TryFix --- .../operator/source/FileLoaderUtils.java | 4 +-- .../utils/ResourceByPathUtils.java | 31 ++++++++++++++----- .../memtable/AlignedReadOnlyMemChunk.java | 10 ++++-- .../reader/chunk/DiskAlignedChunkLoader.java | 3 +- .../DiskAlignedChunkMetadataLoader.java | 4 +-- .../iotdb/db/utils/ModificationUtils.java | 9 +++--- pom.xml | 2 +- 7 files changed, 42 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index cd00aa5e4196..213e4990a5f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -41,7 +41,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; @@ -395,7 +395,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = ignoreAllNullRows ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList) - : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList); + : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList); alignedTimeSeriesMetadata.setChunkMetadataLoader( new DiskAlignedChunkMetadataLoader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index dbd32b47276f..12393fb2a209 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,12 +37,16 @@ import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -108,7 +112,7 @@ public AlignedResourceByPathUtils(IFullPath fullPath) { * have chunkMetadata, but query will use these, so we need to generate it for them. */ @Override - public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( + public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( List readOnlyMemChunk, List chunkMetadataList) { TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); @@ -131,8 +135,11 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; + boolean isTable = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadata; + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); modified = (modified || alignedChunkMetadata.isModified()); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -149,8 +156,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { if (!memChunk.isEmpty()) { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) memChunk.getChunkMetaData(); + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData(); + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { @@ -174,7 +182,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( } } - return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + return isTable + ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList) + : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); } @Override @@ -268,7 +278,7 @@ public List getVisibleMetadataListFromWriter( QueryContext context, long timeLowerBound) { - List chunkMetadataList = new ArrayList<>(); + List chunkMetadataList = new ArrayList<>(); List timeChunkMetadataList = writer.getVisibleMetadataList( alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); @@ -285,7 +295,10 @@ public List getVisibleMetadataListFromWriter( // only need time column if (alignedFullPath.getMeasurementList().isEmpty()) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()) + : new TableDeviceChunkMetadata( + timeChunkMetadataList.get(i), Collections.emptyList())); } else { List valueChunkMetadata = new ArrayList<>(); // if all the sub sensors doesn't exist, it will be false @@ -298,7 +311,9 @@ public List getVisibleMetadataListFromWriter( } if (!context.isIgnoreAllNullRows() || exits) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata) + : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 569db7347ab5..a3eb66b018d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.TimeRange; @@ -75,10 +76,11 @@ public AlignedReadOnlyMemChunk( timeColumnDeletion, valueColumnsDeletionList, context.isIgnoreAllNullRows()); - initAlignedChunkMetaFromTsBlock(); + initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows()); } - private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { + private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) + throws QueryProcessException { // Time chunk Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR); IChunkMetadata timeChunkMetadata = @@ -160,7 +162,9 @@ private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { } } IChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadata, valueChunkMetadataList); alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context, this)); alignedChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = alignedChunkMetadata; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index adef246ec8d9..b0cfd3c03627 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -76,7 +77,7 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi throws IOException { long t1 = System.nanoTime(); try { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; + AbstractAlignedChunkMetadata alignedChunkMetadata = (AbstractAlignedChunkMetadata) chunkMetaData; ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = ChunkCache.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java index afc57079c88b..f243b51ee1a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.read.controller.IChunkMetadataLoader; @@ -83,7 +83,7 @@ public DiskAlignedChunkMetadataLoader( public List loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) { final long t1 = System.nanoTime(); try { - List alignedChunkMetadataList = + List alignedChunkMetadataList = ((AbstractAlignedTimeSeriesMetadata) timeSeriesMetadata).getCopiedChunkMetadataList(); // when alignedChunkMetadataList.size() == 1, it means that the chunk statistics is same as diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 599662076480..f9a6d28e2b98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -101,7 +102,7 @@ public static void modifyAlignedChunkMetaData( } private static void modifyValueColumns( - AlignedChunkMetadata metaData, List> valueColumnsModifications) { + AbstractAlignedChunkMetadata metaData, List> valueColumnsModifications) { List valueChunkMetadataList = metaData.getValueChunkMetadataList(); // deal with each sub sensor for (int j = 0; j < valueChunkMetadataList.size(); j++) { @@ -116,7 +117,7 @@ private static void modifyValueColumns( } private static boolean areAllValueColumnsDeleted( - AlignedChunkMetadata alignedChunkMetadata, boolean modified) { + AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) { // the whole aligned path need to be removed, only set to be true if all the sub sensors // are deleted and ignoreAllNullRows is true @@ -156,11 +157,11 @@ private static boolean areAllValueColumnsDeleted( } public static void modifyAlignedChunkMetaData( - List chunkMetaData, + List chunkMetaData, List timeColumnModifications, List> valueColumnsModifications, boolean ignoreAllNullRows) { - for (AlignedChunkMetadata metaData : chunkMetaData) { + for (AbstractAlignedChunkMetadata metaData : chunkMetaData) { IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata(); for (ModEntry modification : timeColumnModifications) { diff --git a/pom.xml b/pom.xml index 40164fc32edf..2e6bdf4efaeb 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ 0.14.1 1.9 1.5.6-3 - 2.0.0-250118-SNAPSHOT + 2.1.0-SNAPSHOT