From 9c9b2393af307895026c501345bfd4d7b8ecd495 Mon Sep 17 00:00:00 2001 From: Haonan Date: Sun, 26 Jan 2025 09:50:16 +0800 Subject: [PATCH] Fix query error after insert an all null aligned tablet and flush (#14777) --- .../session/it/IoTDBSessionInsertNullIT.java | 70 +++++++++++++++++++ .../dataregion/flush/MemTableFlushTask.java | 2 +- .../memtable/AlignedWritableMemChunk.java | 13 +++- .../memtable/WritableMemChunkGroup.java | 2 +- 4 files changed, 84 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index f101a833fbb3..bbee36782fc4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -32,6 +32,8 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -355,4 +357,72 @@ public void insertAlignedRecordsOfOneDeviceNullTest() { fail(e.getMessage()); } } + + @Test + public void insertTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void insertAlignedTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertAlignedTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 45a398b96bb5..948f89c89fdb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -138,7 +138,7 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException for (IDeviceID deviceID : deviceIDList) { final Map value = memTableMap.get(deviceID).getMemChunkMap(); // skip the empty device/chunk group - if (memTableMap.get(deviceID).count() == 0 || value.isEmpty()) { + if (memTableMap.get(deviceID).isEmpty() || value.isEmpty()) { continue; } encodingTaskQueue.put(new StartFlushGroupIOTask(deviceID)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 3c906104c17e..9561d2531bc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -264,6 +264,9 @@ public IMeasurementSchema getSchema() { @Override public long getMaxTime() { + if (isEmpty()) { + return Long.MIN_VALUE; + } return list.getMaxTime(); } @@ -511,7 +514,15 @@ public long getLastPoint() { @Override public boolean isEmpty() { - return list.rowCount() == 0 || measurementIndexMap.isEmpty(); + if (list.rowCount() == 0) { + return true; + } + if (ignoreAllNullRows) { + return measurementIndexMap.isEmpty() + || (list.getAllValueColDeletedMap() != null + && list.getAllValueColDeletedMap().isAllMarked()); + } + return false; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index ac5d588eb4fd..ae185f890670 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -118,7 +118,7 @@ public Map getMemChunkMap() { @Override public boolean isEmpty() { - return memChunkMap.isEmpty(); + return memChunkMap.isEmpty() || count() == 0; } @Override