From f6afc4ea3ff2bcd2ec22ebc138638715ade762e6 Mon Sep 17 00:00:00 2001 From: Nero Date: Tue, 9 Dec 2025 22:18:31 +0800 Subject: [PATCH 01/17] Remove the code check port is occupied, and resolve the problem that can't rename file successfully. --- .../commons/file/SystemPropertiesHandler.java | 6 +-- scripts/sbin/windows/start-confignode.bat | 28 ----------- scripts/sbin/windows/start-datanode.bat | 48 ------------------- 3 files changed, 3 insertions(+), 79 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java index dfbb2104cfb8d..b23bf2630f403 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java @@ -21,9 +21,9 @@ import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.ratis.util.AutoCloseableLock; -import org.apache.ratis.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -185,8 +185,8 @@ private void replaceFormalFile() throws IOException { throw new IOException(msg); } try { - FileUtils.move(tmpFile.toPath(), formalFile.toPath()); - } catch (IOException e) { + FileUtils.moveFileSafe(tmpFile, formalFile); + } catch (Exception e) { String msg = String.format( "Failed to replace formal system properties file, you may manually rename it: %s -> %s", diff --git a/scripts/sbin/windows/start-confignode.bat b/scripts/sbin/windows/start-confignode.bat index 2501a0645c2a5..64efa6f5580ff 100644 --- a/scripts/sbin/windows/start-confignode.bat +++ b/scripts/sbin/windows/start-confignode.bat @@ -111,34 +111,6 @@ IF DEFINED CONFIG_FILE ( set cn_consensus_port=10720 ) -echo Check whether the ports are occupied.... -set occupied=0 -set cn_internal_port_occupied=0 -set cn_consensus_port_occupied=0 -for /f "tokens=1,3,7 delims=: " %%i in ('netstat /ano') do ( - if %%i==TCP ( - if %%j==%cn_internal_port% ( - if !cn_internal_port_occupied!==0 ( - echo The cn_internal_port %cn_internal_port% is already occupied, pid:%%k - set occupied=1 - set cn_internal_port_occupied=1 - ) - ) else if %%j==%cn_consensus_port% ( - if !cn_consensus_port_occupied!==0 ( - echo The cn_consensus_port %cn_consensus_port% is already occupied, pid:%%k - set occupied=1 - set cn_consensus_port_occupied=1 - ) - ) - ) -) - -if %occupied%==1 ( - echo There exists occupied port, please change the configuration. - TIMEOUT /T 10 /NOBREAK - exit 0 -) - set CONF_PARAMS=-s if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.confignode.service.ConfigNode if NOT DEFINED JAVA_HOME goto :err diff --git a/scripts/sbin/windows/start-datanode.bat b/scripts/sbin/windows/start-datanode.bat index 30a7aa50e8367..0cf3e6487839d 100755 --- a/scripts/sbin/windows/start-datanode.bat +++ b/scripts/sbin/windows/start-datanode.bat @@ -146,54 +146,6 @@ IF DEFINED CONFIG_FILE ( set dn_data_region_consensus_port=10760 ) -echo Check whether the ports are occupied.... -set occupied=0 -set dn_rpc_port_occupied=0 -set dn_internal_port_occupied=0 -set dn_mpp_data_exchange_port_occupied=0 -set dn_schema_region_consensus_port_occupied=0 -set dn_data_region_consensus_port_occupied=0 -for /f "tokens=1,3,7 delims=: " %%i in ('netstat /ano') do ( - if %%i==TCP ( - if %%j==%dn_rpc_port% ( - if !dn_rpc_port_occupied!==0 ( - echo The dn_rpc_port %dn_rpc_port% is already occupied, pid:%%k - set occupied=1 - set dn_rpc_port_occupied=1 - ) - ) else if %%j==%dn_internal_port% ( - if !dn_internal_port_occupied!==0 ( - echo The dn_internal_port %dn_internal_port% is already occupied, pid:%%k - set occupied=1 - set dn_internal_port_occupied=1 - ) - ) else if %%j==%dn_mpp_data_exchange_port% ( - if !dn_mpp_data_exchange_port_occupied!==0 ( - echo The dn_mpp_data_exchange_port %dn_mpp_data_exchange_port% is already occupied, pid:%%k - set occupied=1 - set dn_mpp_data_exchange_port_occupied=1 - ) - ) else if %%j==%dn_schema_region_consensus_port% ( - if !dn_schema_region_consensus_port_occupied!==0 ( - echo The dn_schema_region_consensus_port %dn_schema_region_consensus_port% is already occupied, pid:%%k - set occupied=1 - set dn_schema_region_consensus_port_occupied=1 - ) - ) else if %%j==%dn_data_region_consensus_port% ( - if !dn_data_region_consensus_port_occupied!==0 ( - echo The dn_data_region_consensus_port %dn_data_region_consensus_port% is already occupied, pid:%%k - set occupied=1 - ) - ) - ) -) - -if %occupied%==1 ( - echo There exists occupied port, please change the configuration. - TIMEOUT /T 10 /NOBREAK - exit 0 -) - @setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS set CONF_PARAMS=-s if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.service.DataNode From f9b7f9dd75b662bc5e8995bc539ea0e203a3a202 Mon Sep 17 00:00:00 2001 From: Nero Date: Fri, 19 Dec 2025 17:54:23 +0800 Subject: [PATCH 02/17] Only delete data, but don't delete tsfile when performing a drop column statement on the table model or a drop tag statement on the tree model. --- .../apache/iotdb/db/it/IoTDBDeletionIT.java | 52 +++++++++ .../it/db/it/IoTDBDeletionTableIT.java | 109 ++++++++++++++++++ .../it/session/IoTDBSessionRelationalIT.java | 2 +- .../storageengine/dataregion/DataRegion.java | 15 ++- 4 files changed, 176 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index 17ceb9b696255..e00dad90df7f4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -488,6 +488,58 @@ public void testDeleteByRangeComparison() throws SQLException { } } + @Test + public void testDropAndAlter() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.test"); + statement.execute( + "CREATE TIMESERIES root.test.g_0.d3.s_10 with datatype=INT32 tags(tag1=v1, tag2=v2)"); + + // time=1 and time=2 are INT32 and deleted by drop column + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(1, 1)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(2, 2)"); + + statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 DROP tag1"); + + // time=3 and time=4 are STRING + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(3, 3)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(4, 4)"); + + statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 ADD TAGS tag1=v1"); + + // time=5 and time=6 are TEXT + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(5, 5)"); + + statement.execute("FLUSH"); + + statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(6, 6)"); + + try (ResultSet dataSet = + statement.executeQuery("select * from root.test.g_0.d3 order by time")) { + // s1 is dropped but the time should remain + int i = 1; + while (dataSet.next()) { + assertEquals(i, dataSet.getLong(1)); + i++; + } + Assert.assertEquals(6, i - 1); + assertFalse(dataSet.next()); + } + } finally { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE root.test"); + } + } + } + private static void prepareSeries() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 3d61b64e71a64..18d5333da5a3a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -33,8 +33,11 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.write.record.Tablet; import org.awaitility.Awaitility; import org.junit.After; import org.junit.AfterClass; @@ -60,6 +63,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Random; @@ -74,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -2268,6 +2273,110 @@ public void testMultiDeviceCompletelyDeleteTable() throws SQLException { cleanData(testNum); } + @Test + public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); + + // time=1 and time=2 are INT32 and deleted by drop column + Tablet tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + + // time=3 and time=4 are STRING + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 4)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + session.executeNonQueryStatement("ALTER TABLE drop_and_alter ADD COLUMN s1 TEXT"); + + // time=5 and time=6 are TEXT + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 5)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 6)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from drop_and_alter order by time"); + // s1 is dropped but the time should remain + RowRecord rec; + int cnt = 0; + for (int i = 1; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + cnt++; + } + Assert.assertEquals(6, cnt); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS drop_and_alter"); + } + } + } + private static void prepareDatabase() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 9c02ac94208ae..6ae696ba454d4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -1596,7 +1596,7 @@ private void testOneCastWithRow( } @SuppressWarnings("SameParameterValue") - private Object genValue(TSDataType dataType, int i) { + public static Object genValue(TSDataType dataType, int i) { switch (dataType) { case INT32: return i; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 09c9ee8430415..73805045c37ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3030,6 +3030,19 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M Set involvedModificationFiles = new HashSet<>(); List deletedByMods = new ArrayList<>(); List deletedByFiles = new ArrayList<>(); + boolean isDropMeasurementExist = false; + boolean isDropTagExist = false; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry entry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !entry.getPredicate().getMeasurementNames().isEmpty(); + } else { + TreeDeletionEntry entry = (TreeDeletionEntry) deletion; + if (entry.getPathPattern() instanceof MeasurementPath) { + isDropTagExist = !((MeasurementPath) entry.getPathPattern()).getTagMap().isEmpty(); + } + } + for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; @@ -3126,7 +3139,7 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty()) { + if (!deletedByFiles.isEmpty() && !isDropMeasurementExist && !isDropTagExist) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( From 8c92be627d3351c9a4f1e0bdc255e8bc20a14aad Mon Sep 17 00:00:00 2001 From: Nero Date: Fri, 19 Dec 2025 19:21:18 +0800 Subject: [PATCH 03/17] Fix NPE --- .../apache/iotdb/db/storageengine/dataregion/DataRegion.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 73805045c37ad..b3e3dce4e3342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3039,7 +3039,8 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } else { TreeDeletionEntry entry = (TreeDeletionEntry) deletion; if (entry.getPathPattern() instanceof MeasurementPath) { - isDropTagExist = !((MeasurementPath) entry.getPathPattern()).getTagMap().isEmpty(); + Map tagMap = ((MeasurementPath) entry.getPathPattern()).getTagMap(); + isDropTagExist = (tagMap != null) && !tagMap.isEmpty(); } } From 8b91c88571b53533f096b62ce5fb45d2f4c90a32 Mon Sep 17 00:00:00 2001 From: Nero Date: Sun, 21 Dec 2025 23:53:35 +0800 Subject: [PATCH 04/17] Don't delete tsfile when use tag as a where clause in the delete statement on table model. --- .../it/db/it/IoTDBDeletionTableIT.java | 60 +++++++++++++++++++ .../storageengine/dataregion/DataRegion.java | 18 +++--- .../modification/DeletionPredicate.java | 4 ++ 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 18d5333da5a3a..7abb5d2ca05d1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -2273,6 +2273,59 @@ public void testMultiDeviceCompletelyDeleteTable() throws SQLException { cleanData(testNum); } + @Test + public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS delete_by_tag (deviceId STRING TAG, s1 INT32 FIELD)"); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from delete_by_tag order by time"); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + session.executeNonQueryStatement("FLUSH"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + dataSet = session.executeQueryStatement("select * from delete_by_tag order by time"); + + RowRecord rec; + int cnt = 0; + for (int i = 1; i < 5; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + Assert.assertEquals(i, rec.getFields().get(2).getIntV()); + Assert.assertEquals(TSDataType.INT32, rec.getFields().get(2).getDataType()); + cnt++; + } + Assert.assertEquals(4, cnt); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag"); + } + } + } + @Test public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { @@ -2366,6 +2419,13 @@ public void testDropAndAlter() throws IoTDBConnectionException, StatementExecuti for (int i = 1; i < 7; i++) { rec = dataSet.next(); assertEquals(i, rec.getFields().get(0).getLongV()); + LOGGER.error( + "time is {}, value is {}, value type is {}", + rec.getFields().get(0).getLongV(), + rec.getFields().get(1), + rec.getFields().get(1).getDataType()); + // assertNull(rec.getFields().get(1).getDataType()); + // Assert.assertEquals(TSDataType.TEXT, rec.getFields().get(1).getDataType()); cnt++; } Assert.assertEquals(6, cnt); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index b3e3dce4e3342..a4aa20c3de3b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -114,6 +114,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo; +import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; @@ -3031,17 +3032,12 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M List deletedByMods = new ArrayList<>(); List deletedByFiles = new ArrayList<>(); boolean isDropMeasurementExist = false; - boolean isDropTagExist = false; + IDPredicate.IDPredicateType idPredicateType = null; if (deletion instanceof TableDeletionEntry) { - TableDeletionEntry entry = (TableDeletionEntry) deletion; - isDropMeasurementExist = !entry.getPredicate().getMeasurementNames().isEmpty(); - } else { - TreeDeletionEntry entry = (TreeDeletionEntry) deletion; - if (entry.getPathPattern() instanceof MeasurementPath) { - Map tagMap = ((MeasurementPath) entry.getPathPattern()).getTagMap(); - isDropTagExist = (tagMap != null) && !tagMap.isEmpty(); - } + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !tableDeletionEntry.getPredicate().getMeasurementNames().isEmpty(); + idPredicateType = tableDeletionEntry.getPredicate().getIdPredicateType(); } for (TsFileResource sealedTsFile : sealedTsFiles) { @@ -3140,7 +3136,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty() && !isDropMeasurementExist && !isDropTagExist) { + if (!deletedByFiles.isEmpty() + && !isDropMeasurementExist + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 7e79e8f580dc5..294999788fa70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -72,6 +72,10 @@ public void setIdPredicate(IDPredicate idPredicate) { this.idPredicate = idPredicate; } + public IDPredicate.IDPredicateType getIdPredicateType() { + return this.idPredicate.type; + } + public String getTableName() { return tableName; } From bdb9396db68607f1fa2462033bc24fdb243addbb Mon Sep 17 00:00:00 2001 From: Nero Date: Wed, 31 Dec 2025 16:51:01 +0800 Subject: [PATCH 05/17] stash --- .../it/db/it/IoTDBDeletionTableIT.java | 23 ++++++++++--------- .../storageengine/dataregion/DataRegion.java | 14 +++++------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 7abb5d2ca05d1..235d09b3954a5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -2307,18 +2307,19 @@ public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExec session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); dataSet = session.executeQueryStatement("select * from delete_by_tag order by time"); - - RowRecord rec; - int cnt = 0; - for (int i = 1; i < 5; i++) { - rec = dataSet.next(); - assertEquals(i, rec.getFields().get(0).getLongV()); - Assert.assertEquals(i, rec.getFields().get(2).getIntV()); - Assert.assertEquals(TSDataType.INT32, rec.getFields().get(2).getDataType()); - cnt++; - } - Assert.assertEquals(4, cnt); assertFalse(dataSet.hasNext()); + + // RowRecord rec; + // int cnt = 0; + // for (int i = 1; i < 5; i++) { + // rec = dataSet.next(); + // assertEquals(i, rec.getFields().get(0).getLongV()); + // Assert.assertEquals(i, rec.getFields().get(2).getIntV()); + // Assert.assertEquals(TSDataType.INT32, rec.getFields().get(2).getDataType()); + // cnt++; + // } + // Assert.assertEquals(4, cnt); + // assertFalse(dataSet.hasNext()); } finally { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index a4aa20c3de3b6..664b7a71db04f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3041,9 +3041,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } for (TsFileResource sealedTsFile : sealedTsFiles) { - if (canSkipDelete(sealedTsFile, deletion)) { - continue; - } + // if (canSkipDelete(sealedTsFile, deletion)) { + // continue; + // } ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); @@ -3110,7 +3110,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M break; } } - if (matchSize == devicesInFile.size()) { + if (matchSize == devicesInFile.size() + && !isDropMeasurementExist + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { deletedByFiles.add(sealedTsFile); } @@ -3136,9 +3138,7 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty() - && !isDropMeasurementExist - && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { + if (!deletedByFiles.isEmpty()) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( From 7eef316c7cc3805a0f0ccea67640285ee122a89d Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 5 Jan 2026 18:55:32 +0800 Subject: [PATCH 06/17] Won't delete according tsfile files directly if IDPredicate type is not NOP when delete table data by tag. --- .../apache/iotdb/db/it/IoTDBDeletionIT.java | 52 ------------------- .../it/db/it/IoTDBDeletionTableIT.java | 12 ----- .../storageengine/dataregion/DataRegion.java | 13 +++-- 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index e00dad90df7f4..17ceb9b696255 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -488,58 +488,6 @@ public void testDeleteByRangeComparison() throws SQLException { } } - @Test - public void testDropAndAlter() throws SQLException { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute("CREATE DATABASE root.test"); - statement.execute( - "CREATE TIMESERIES root.test.g_0.d3.s_10 with datatype=INT32 tags(tag1=v1, tag2=v2)"); - - // time=1 and time=2 are INT32 and deleted by drop column - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(1, 1)"); - - statement.execute("FLUSH"); - - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(2, 2)"); - - statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 DROP tag1"); - - // time=3 and time=4 are STRING - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(3, 3)"); - - statement.execute("FLUSH"); - - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(4, 4)"); - - statement.execute("ALTER TIMESERIES root.test.g_0.d3.s_10 ADD TAGS tag1=v1"); - - // time=5 and time=6 are TEXT - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(5, 5)"); - - statement.execute("FLUSH"); - - statement.execute("INSERT INTO root.test.g_0.d3(timestamp, s_10) VALUES(6, 6)"); - - try (ResultSet dataSet = - statement.executeQuery("select * from root.test.g_0.d3 order by time")) { - // s1 is dropped but the time should remain - int i = 1; - while (dataSet.next()) { - assertEquals(i, dataSet.getLong(1)); - i++; - } - Assert.assertEquals(6, i - 1); - assertFalse(dataSet.next()); - } - } finally { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute("DROP DATABASE root.test"); - } - } - } - private static void prepareSeries() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 235d09b3954a5..aabee738d72cb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -2308,18 +2308,6 @@ public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExec dataSet = session.executeQueryStatement("select * from delete_by_tag order by time"); assertFalse(dataSet.hasNext()); - - // RowRecord rec; - // int cnt = 0; - // for (int i = 1; i < 5; i++) { - // rec = dataSet.next(); - // assertEquals(i, rec.getFields().get(0).getLongV()); - // Assert.assertEquals(i, rec.getFields().get(2).getIntV()); - // Assert.assertEquals(TSDataType.INT32, rec.getFields().get(2).getDataType()); - // cnt++; - // } - // Assert.assertEquals(4, cnt); - // assertFalse(dataSet.hasNext()); } finally { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index a426b3bbc9787..bdb3ae30d9b48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3141,9 +3141,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } for (TsFileResource sealedTsFile : sealedTsFiles) { - // if (canSkipDelete(sealedTsFile, deletion)) { - // continue; - // } + if (canSkipDelete(sealedTsFile, deletion)) { + continue; + } ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); @@ -3203,16 +3203,15 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M fileStartTime, fileEndTime); } - if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) { + if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime) + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { ++matchSize; } else { deletedByMods.add(sealedTsFile); break; } } - if (matchSize == devicesInFile.size() - && !isDropMeasurementExist - && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { + if (matchSize == devicesInFile.size() && !isDropMeasurementExist) { deletedByFiles.add(sealedTsFile); } From 524cc4ccc8ca7eabb34c2a869aff8be094c1001a Mon Sep 17 00:00:00 2001 From: Nero Date: Thu, 8 Jan 2026 16:31:24 +0800 Subject: [PATCH 07/17] Occur incompatible exception when merging statistics, need to rewrite statistics in current chunk metadata so that resolve "Statistics classes mismatched: class org.apache.tsfile.file.metadata.statistics.BinaryStatistics vs. class org.apache.tsfile.file.metadata.statistics.IntegerStatistics" exception. --- .../utils/ResourceByPathUtils.java | 8 + .../apache/iotdb/db/utils/SchemaUtils.java | 213 +++++++++--------- 2 files changed, 119 insertions(+), 102 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index 188c4ec652984..bff3231f700bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -542,6 +542,14 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata( boolean isModified = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { isModified = (isModified || chunkMetadata.isModified()); + TSDataType targetDataType = fullPath.getMeasurementSchema().getType(); + if (targetDataType.equals(TSDataType.STRING) + && (chunkMetadata.getDataType() != targetDataType)) { + // create new statistics object via new data type, and merge statistics information + SchemaUtils.rewriteNonAlignedChunkMetadataStatistics( + (ChunkMetadata) chunkMetadata, targetDataType); + chunkMetadata.setModified(true); + } if (!useFakeStatistics) { seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index c16f5339ee882..e0997f852a7e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -404,108 +404,7 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics List newValueChunkMetadataList = new ArrayList<>(); for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { Statistics statistics = Statistics.getStatsByType(targetDataType); - switch (valueChunkMetadata.getDataType()) { - case INT32: - case DATE: - case INT64: - case TIMESTAMP: - case FLOAT: - case DOUBLE: - case BOOLEAN: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[4]; - binaryValues[0] = - new Binary( - valueChunkMetadata.getStatistics().getFirstValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - valueChunkMetadata.getStatistics().getLastValue().toString(), - StandardCharsets.UTF_8); - if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[2] = - new Binary( - valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[3] = - new Binary( - valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - } - long[] longValues = new long[4]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - longValues[2] = longValues[1]; - longValues[3] = longValues[1]; - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[0] = - new Binary( - valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - } - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - case STRING: - if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = - new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(valueChunkMetadata.getDataType()) - ? "" - : valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(valueChunkMetadata.getDataType()) - ? "" - : valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - case TEXT: - case BLOB: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = new Binary("", StandardCharsets.UTF_8); - binaryValues[1] = new Binary("", StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - default: - break; - } + statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; newChunkMetadata.setTsDataType(targetDataType); @@ -516,10 +415,120 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList); } + public static void rewriteNonAlignedChunkMetadataStatistics( + ChunkMetadata chunkMetadata, TSDataType targetDataType) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(chunkMetadata, targetDataType, statistics); + + chunkMetadata.setTsDataType(targetDataType); + chunkMetadata.setStatistics(statistics); + } + public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEncoding encoding) { if (!encoding.isSupported(dataType)) { return EncodingInferenceUtils.getDefaultEncoding(dataType); } return encoding; } + + public static Statistics getNewStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + switch (chunkMetadata.getDataType()) { + case INT32: + case DATE: + case INT64: + case TIMESTAMP: + case FLOAT: + case DOUBLE: + case BOOLEAN: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[4]; + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getFirstValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getLastValue().toString(), StandardCharsets.UTF_8); + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[2] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[3] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + long[] longValues = new long[4]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + longValues[2] = longValues[1]; + longValues[3] = longValues[1]; + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case STRING: + if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = + new Binary( + Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) + .contains(chunkMetadata.getDataType()) + ? "" + : chunkMetadata.getStatistics().getMinValue().toString(), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) + .contains(chunkMetadata.getDataType()) + ? "" + : chunkMetadata.getStatistics().getMaxValue().toString(), + StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case TEXT: + case BLOB: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = new Binary("", StandardCharsets.UTF_8); + binaryValues[1] = new Binary("", StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + default: + break; + } + return statistics; + } } From a9f3fb5ee5b1d280137b70be9cd5366f83f04851 Mon Sep 17 00:00:00 2001 From: Nero Date: Sat, 10 Jan 2026 00:08:44 +0800 Subject: [PATCH 08/17] Don't merge statistics when two types are not compatible; Fix regenerate statistics logics. --- .../utils/ResourceByPathUtils.java | 7 +- .../apache/iotdb/db/utils/SchemaUtils.java | 66 ++++--- .../iotdb/db/utils/SchemaUtilsTest.java | 163 ++++++++++++++++++ 3 files changed, 212 insertions(+), 24 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index bff3231f700bb..ab6fbd4547a84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -280,7 +280,8 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( if (!useFakeStatistics) { timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (!alignedChunkMetadata.getValueChunkMetadataList().isEmpty() + && alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { exist[i] = true; valueTimeSeriesMetadataList .get(i) @@ -551,7 +552,9 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata( chunkMetadata.setModified(true); } if (!useFakeStatistics) { - seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + } continue; } startTime = Math.min(startTime, chunkMetadata.getStartTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index e0997f852a7e4..9920850fc5262 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -403,13 +403,15 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType targetDataType) { List newValueChunkMetadataList = new ArrayList<>(); for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { - Statistics statistics = Statistics.getStatsByType(targetDataType); - statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); - - ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; - newChunkMetadata.setTsDataType(targetDataType); - newChunkMetadata.setStatistics(statistics); - newValueChunkMetadataList.add(newChunkMetadata); + if (targetDataType.isCompatible(valueChunkMetadata.getDataType())) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); + + ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; + newChunkMetadata.setTsDataType(targetDataType); + newChunkMetadata.setStatistics(statistics); + newValueChunkMetadataList.add(newChunkMetadata); + } } return new AlignedChunkMetadata( alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList); @@ -417,11 +419,13 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics public static void rewriteNonAlignedChunkMetadataStatistics( ChunkMetadata chunkMetadata, TSDataType targetDataType) { - Statistics statistics = Statistics.getStatsByType(targetDataType); - statistics = getNewStatistics(chunkMetadata, targetDataType, statistics); + if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(chunkMetadata, targetDataType, statistics); - chunkMetadata.setTsDataType(targetDataType); - chunkMetadata.setStatistics(statistics); + chunkMetadata.setTsDataType(targetDataType); + chunkMetadata.setStatistics(statistics); + } } public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEncoding encoding) { @@ -492,28 +496,28 @@ public static Statistics getNewStatistics( Binary[] binaryValues = new Binary[2]; binaryValues[0] = new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(chunkMetadata.getDataType()) - ? "" - : chunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); binaryValues[1] = new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(chunkMetadata.getDataType()) - ? "" - : chunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); long[] longValues = new long[2]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.BLOB) { + statistics.update( + chunkMetadata.getStatistics().getStartTime(), + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8)); + statistics.update( + chunkMetadata.getStatistics().getEndTime(), + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8)); } else { statistics = chunkMetadata.getStatistics(); } break; case TEXT: - case BLOB: if (targetDataType == TSDataType.STRING) { Binary[] binaryValues = new Binary[2]; binaryValues[0] = new Binary("", StandardCharsets.UTF_8); @@ -522,6 +526,24 @@ public static Statistics getNewStatistics( longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.BLOB) { + statistics.update( + chunkMetadata.getStatistics().getStartTime(), new Binary("", StandardCharsets.UTF_8)); + statistics.update( + chunkMetadata.getStatistics().getEndTime(), new Binary("", StandardCharsets.UTF_8)); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case BLOB: + if (targetDataType == TSDataType.STRING || targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = new Binary("", StandardCharsets.UTF_8); + binaryValues[1] = new Binary("", StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); } else { statistics = chunkMetadata.getStatistics(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 17dffd6978509..373bcd8f2d55d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -29,13 +29,20 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.Binary; import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; + +import static org.apache.tsfile.file.metadata.statistics.Statistics.canMerge; public class SchemaUtilsTest { @Test @@ -112,4 +119,160 @@ public void rewriteAlignedChunkMetadataStatistics() { } } } + + public static Object genValue(TSDataType dataType, int i) { + switch (dataType) { + case INT32: + return i; + case DATE: + return LocalDate.ofEpochDay(i); + case TIMESTAMP: + case INT64: + return (long) i; + case BOOLEAN: + return i % 2 == 0; + case FLOAT: + return i * 1.0f; + case DOUBLE: + return i * 1.0; + case STRING: + case TEXT: + case BLOB: + return new Binary(Integer.toString(i), StandardCharsets.UTF_8); + case UNKNOWN: + case VECTOR: + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } + + @Test + public void mergeMetadataStatistics() throws Exception { + Set unsupportTsDataType = new HashSet<>(); + unsupportTsDataType.add(TSDataType.UNKNOWN); + unsupportTsDataType.add(TSDataType.VECTOR); + for (TSDataType sourceDataType : Arrays.asList(TSDataType.DOUBLE)) { + for (TSDataType targetDataType : Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)) { + + if (sourceDataType.equals(targetDataType)) { + continue; + } + if (unsupportTsDataType.contains(sourceDataType) + || unsupportTsDataType.contains(targetDataType)) { + continue; + } + + System.out.println("from " + sourceDataType + " to " + targetDataType); + + // Aligned series + Statistics s1 = Statistics.getStatsByType(sourceDataType); + s1.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2); + Statistics s2 = Statistics.getStatsByType(TSDataType.DOUBLE); + s2.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2); + List valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata( + "s0", + sourceDataType, + SchemaUtils.getDataTypeCompatibleEncoding(sourceDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s1), + new ChunkMetadata( + "s1", + TSDataType.DOUBLE, + SchemaUtils.getDataTypeCompatibleEncoding(TSDataType.DOUBLE, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s2)); + IChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + + Statistics s3 = Statistics.getStatsByType(targetDataType); + if (targetDataType == TSDataType.BLOB) { + s3.update(3, new Binary("3", StandardCharsets.UTF_8)); + s3.update(4, new Binary("4", StandardCharsets.UTF_8)); + } else { + s3.update( + new long[] {1, 2}, + new Binary[] { + new Binary("3", StandardCharsets.UTF_8), + new Binary("4", StandardCharsets.UTF_8), + new Binary("3", StandardCharsets.UTF_8), + new Binary("4", StandardCharsets.UTF_8) + }, + 2); + } + Statistics s4 = Statistics.getStatsByType(targetDataType); + if (targetDataType == TSDataType.BLOB) { + s3.update(4, new Binary("4", StandardCharsets.UTF_8)); + } else { + s4.update( + new long[] {1, 2}, + new Binary[] { + new Binary("5", StandardCharsets.UTF_8), + new Binary("6", StandardCharsets.UTF_8), + new Binary("5", StandardCharsets.UTF_8), + new Binary("6", StandardCharsets.UTF_8) + }, + 2); + } + List targetChunkMetadatas = + Arrays.asList( + new ChunkMetadata( + "s0", + targetDataType, + SchemaUtils.getDataTypeCompatibleEncoding(targetDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s3), + new ChunkMetadata( + "s1", + targetDataType, + SchemaUtils.getDataTypeCompatibleEncoding(targetDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s4)); + AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = + (AbstractAlignedChunkMetadata) alignedChunkMetadata; + try { + abstractAlignedChunkMetadata = + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + abstractAlignedChunkMetadata, targetDataType); + } catch (ClassCastException e) { + Assert.fail(e.getMessage()); + } + + for (int i = 0; i < targetChunkMetadatas.size(); i++) { + if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty() + && abstractAlignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (targetChunkMetadatas.get(i).getStatistics().getClass() + == abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics() + .getClass() + || canMerge( + abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics() + .getType(), + targetChunkMetadatas.get(i).getStatistics().getType())) { + targetChunkMetadatas + .get(i) + .getStatistics() + .mergeStatistics( + abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics()); + } else { + throw new Exception("unsupported"); + } + } + } + } + } + } } From a864880bf698ead795ea0031368fd2f4248b0aef Mon Sep 17 00:00:00 2001 From: Nero Date: Sat, 10 Jan 2026 07:35:05 +0800 Subject: [PATCH 09/17] Fix out of bounds problem from unit test. --- .../iotdb/db/utils/SchemaUtilsTest.java | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 373bcd8f2d55d..0d9875a5b2dba 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -34,7 +34,6 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -110,9 +109,11 @@ public void rewriteAlignedChunkMetadataStatistics() { AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = SchemaUtils.rewriteAlignedChunkMetadataStatistics( alignedChunkMetadata, targetDataType); - Assert.assertEquals( - targetDataType, - abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { + Assert.assertEquals( + targetDataType, + abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + } } catch (ClassCastException e) { Assert.fail(e.getMessage()); } @@ -120,32 +121,6 @@ public void rewriteAlignedChunkMetadataStatistics() { } } - public static Object genValue(TSDataType dataType, int i) { - switch (dataType) { - case INT32: - return i; - case DATE: - return LocalDate.ofEpochDay(i); - case TIMESTAMP: - case INT64: - return (long) i; - case BOOLEAN: - return i % 2 == 0; - case FLOAT: - return i * 1.0f; - case DOUBLE: - return i * 1.0; - case STRING: - case TEXT: - case BLOB: - return new Binary(Integer.toString(i), StandardCharsets.UTF_8); - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); - } - } - @Test public void mergeMetadataStatistics() throws Exception { Set unsupportTsDataType = new HashSet<>(); From 2c250e11079bce64f6222ab1e842c725d4d9c442 Mon Sep 17 00:00:00 2001 From: Nero Date: Sat, 10 Jan 2026 07:43:31 +0800 Subject: [PATCH 10/17] Use a constant variable instead of frequently creating a empty Binary object. --- .../org/apache/iotdb/db/utils/SchemaUtils.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 9920850fc5262..7345241bf784e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -55,6 +55,7 @@ public class SchemaUtils { private static final Map dataTypeColumnClassMap; public static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); + private static final Binary EMPTY_BINARY = new Binary("", StandardCharsets.UTF_8); static { dataTypeColumnClassMap = new HashMap<>(); @@ -520,17 +521,15 @@ public static Statistics getNewStatistics( case TEXT: if (targetDataType == TSDataType.STRING) { Binary[] binaryValues = new Binary[2]; - binaryValues[0] = new Binary("", StandardCharsets.UTF_8); - binaryValues[1] = new Binary("", StandardCharsets.UTF_8); + binaryValues[0] = EMPTY_BINARY; + binaryValues[1] = EMPTY_BINARY; long[] longValues = new long[2]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); statistics.update(longValues, binaryValues, binaryValues.length); } else if (targetDataType == TSDataType.BLOB) { - statistics.update( - chunkMetadata.getStatistics().getStartTime(), new Binary("", StandardCharsets.UTF_8)); - statistics.update( - chunkMetadata.getStatistics().getEndTime(), new Binary("", StandardCharsets.UTF_8)); + statistics.update(chunkMetadata.getStatistics().getStartTime(), EMPTY_BINARY); + statistics.update(chunkMetadata.getStatistics().getEndTime(), EMPTY_BINARY); } else { statistics = chunkMetadata.getStatistics(); } @@ -538,8 +537,8 @@ public static Statistics getNewStatistics( case BLOB: if (targetDataType == TSDataType.STRING || targetDataType == TSDataType.TEXT) { Binary[] binaryValues = new Binary[2]; - binaryValues[0] = new Binary("", StandardCharsets.UTF_8); - binaryValues[1] = new Binary("", StandardCharsets.UTF_8); + binaryValues[0] = EMPTY_BINARY; + binaryValues[1] = EMPTY_BINARY; long[] longValues = new long[2]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); From b2927fc8718567a3bf4ebae186b6934d5036661c Mon Sep 17 00:00:00 2001 From: Nero Date: Sun, 11 Jan 2026 21:07:07 +0800 Subject: [PATCH 11/17] When the statement to execute is to drop a column, put it in the mods file instead of deleting the tsfile. --- .../apache/iotdb/db/storageengine/dataregion/DataRegion.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index df5a06b385a7e..746e042ab14bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3308,14 +3308,15 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M fileEndTime); } if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime) - && idPredicateType.equals(IDPredicate.IDPredicateType.NOP)) { + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP) + && !isDropMeasurementExist) { ++matchSize; } else { deletedByMods.add(sealedTsFile); break; } } - if (matchSize == devicesInFile.size() && !isDropMeasurementExist) { + if (matchSize == devicesInFile.size()) { deletedByFiles.add(sealedTsFile); } From 080b69d3eb54b634a0af7562e33fe8da8ab30b10 Mon Sep 17 00:00:00 2001 From: Nero Date: Sun, 11 Jan 2026 23:48:35 +0800 Subject: [PATCH 12/17] Repair and add the logic involve DATE data type. --- .../it/schema/IoTDBAlterColumnTypeIT.java | 1 + .../apache/iotdb/db/utils/SchemaUtils.java | 54 ++++++++++++++++--- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java index d2f01f928b8ea..53f15a192e889 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java @@ -140,6 +140,7 @@ private void doWriteAndAlter(TSDataType from, TSDataType to) throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + session.executeNonQueryStatement("SET CONFIGURATION enable_seq_space_compaction='false'"); if (from == TSDataType.DATE && !to.isCompatible(from)) { throw new NotSupportedException("Not supported DATE type."); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 7345241bf784e..143b76de34d24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -440,7 +440,6 @@ public static Statistics getNewStatistics( IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { switch (chunkMetadata.getDataType()) { case INT32: - case DATE: case INT64: case TIMESTAMP: case FLOAT: @@ -465,11 +464,9 @@ public static Statistics getNewStatistics( new Binary( chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); } - long[] longValues = new long[4]; + long[] longValues = new long[2]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); - longValues[2] = longValues[1]; - longValues[3] = longValues[1]; statistics.update(longValues, binaryValues, binaryValues.length); } else if (targetDataType == TSDataType.TEXT) { Binary[] binaryValues = new Binary[2]; @@ -492,6 +489,51 @@ public static Statistics getNewStatistics( statistics = chunkMetadata.getStatistics(); } break; + case DATE: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[4]; + binaryValues[0] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getFirstValue()), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getLastValue()), + StandardCharsets.UTF_8); + binaryValues[2] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getMinValue()), + StandardCharsets.UTF_8); + binaryValues[3] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getMaxValue()), + StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getFirstValue()), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + TSDataType.getDateStringValue( + (Integer) chunkMetadata.getStatistics().getLastValue()), + StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } + break; case STRING: if (targetDataType == TSDataType.TEXT) { Binary[] binaryValues = new Binary[2]; @@ -521,8 +563,8 @@ public static Statistics getNewStatistics( case TEXT: if (targetDataType == TSDataType.STRING) { Binary[] binaryValues = new Binary[2]; - binaryValues[0] = EMPTY_BINARY; - binaryValues[1] = EMPTY_BINARY; + binaryValues[0] = (Binary) chunkMetadata.getStatistics().getFirstValue(); + binaryValues[1] = (Binary) chunkMetadata.getStatistics().getLastValue(); long[] longValues = new long[2]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); From 9ba0fda74aedf4ab0556abb92271c94d3ff4cb57 Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 12 Jan 2026 09:52:28 +0800 Subject: [PATCH 13/17] Fix problem that out of bounds in the SchemaUtilsTest. --- .../main/java/org/apache/iotdb/db/utils/SchemaUtils.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 143b76de34d24..df95d1cf965e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -464,9 +464,11 @@ public static Statistics getNewStatistics( new Binary( chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); } - long[] longValues = new long[2]; + long[] longValues = new long[4]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); + longValues[2] = longValues[1]; + longValues[3] = longValues[1]; statistics.update(longValues, binaryValues, binaryValues.length); } else if (targetDataType == TSDataType.TEXT) { Binary[] binaryValues = new Binary[2]; @@ -512,9 +514,11 @@ public static Statistics getNewStatistics( TSDataType.getDateStringValue( (Integer) chunkMetadata.getStatistics().getMaxValue()), StandardCharsets.UTF_8); - long[] longValues = new long[2]; + long[] longValues = new long[4]; longValues[0] = chunkMetadata.getStatistics().getStartTime(); longValues[1] = chunkMetadata.getStatistics().getEndTime(); + longValues[2] = longValues[1]; + longValues[3] = longValues[1]; statistics.update(longValues, binaryValues, binaryValues.length); } else if (targetDataType == TSDataType.TEXT) { Binary[] binaryValues = new Binary[2]; From 87a4da5ac4d8110978301ba4874e30d4bbbcb4f6 Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 12 Jan 2026 11:24:17 +0800 Subject: [PATCH 14/17] In order to avoid get chunkMetadata of other index in the array, wish to get chunkMetadata current index itself, when execute query statement. Place null value that when chunkMetaData is not compatible with chunkMetadData of target data type --- .../schemaregion/utils/ResourceByPathUtils.java | 2 +- .../java/org/apache/iotdb/db/utils/SchemaUtils.java | 7 +++++-- .../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 10 ++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index ab6fbd4547a84..09597b90bf92c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -552,7 +552,7 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata( chunkMetadata.setModified(true); } if (!useFakeStatistics) { - if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + if (chunkMetadata != null && targetDataType.isCompatible(chunkMetadata.getDataType())) { seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); } continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index df95d1cf965e3..f59d0980d956a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -404,7 +404,8 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType targetDataType) { List newValueChunkMetadataList = new ArrayList<>(); for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { - if (targetDataType.isCompatible(valueChunkMetadata.getDataType())) { + if (valueChunkMetadata != null + && targetDataType.isCompatible(valueChunkMetadata.getDataType())) { Statistics statistics = Statistics.getStatsByType(targetDataType); statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); @@ -412,6 +413,8 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics newChunkMetadata.setTsDataType(targetDataType); newChunkMetadata.setStatistics(statistics); newValueChunkMetadataList.add(newChunkMetadata); + } else { + newValueChunkMetadataList.add(null); } } return new AlignedChunkMetadata( @@ -420,7 +423,7 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics public static void rewriteNonAlignedChunkMetadataStatistics( ChunkMetadata chunkMetadata, TSDataType targetDataType) { - if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + if (chunkMetadata != null && targetDataType.isCompatible(chunkMetadata.getDataType())) { Statistics statistics = Statistics.getStatsByType(targetDataType); statistics = getNewStatistics(chunkMetadata, targetDataType, statistics); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 0d9875a5b2dba..10dcf3e284415 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -171,10 +171,7 @@ public void mergeMetadataStatistics() throws Exception { s3.update( new long[] {1, 2}, new Binary[] { - new Binary("3", StandardCharsets.UTF_8), - new Binary("4", StandardCharsets.UTF_8), - new Binary("3", StandardCharsets.UTF_8), - new Binary("4", StandardCharsets.UTF_8) + new Binary("3", StandardCharsets.UTF_8), new Binary("4", StandardCharsets.UTF_8), }, 2); } @@ -185,10 +182,7 @@ public void mergeMetadataStatistics() throws Exception { s4.update( new long[] {1, 2}, new Binary[] { - new Binary("5", StandardCharsets.UTF_8), - new Binary("6", StandardCharsets.UTF_8), - new Binary("5", StandardCharsets.UTF_8), - new Binary("6", StandardCharsets.UTF_8) + new Binary("5", StandardCharsets.UTF_8), new Binary("6", StandardCharsets.UTF_8), }, 2); } From bdf854350b6cde7d7fd68bc8c21307d2ee5f6031 Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 12 Jan 2026 11:50:59 +0800 Subject: [PATCH 15/17] Fix the issue "Only the target component should be written, not all of them". --- .../utils/ResourceByPathUtils.java | 11 +++---- .../apache/iotdb/db/utils/SchemaUtils.java | 33 ++++++++----------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index 09597b90bf92c..2bfbdec4e3b3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -268,13 +268,12 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( modified = (modified || alignedChunkMetadata.isModified()); TSDataType targetDataType = alignedFullPath.getSchemaList().get(index).getType(); if (targetDataType.equals(TSDataType.STRING) - && (alignedChunkMetadata.getValueChunkMetadataList().stream() - .filter(iChunkMetadata -> iChunkMetadata.getDataType() != targetDataType) - .count() - > 0)) { + && ((alignedChunkMetadata.getValueChunkMetadataList().get(index) != null) + && (alignedChunkMetadata.getValueChunkMetadataList().get(index).getDataType() + != targetDataType))) { // create new statistics object via new data type, and merge statistics information - alignedChunkMetadata = - SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, targetDataType); + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + alignedChunkMetadata, index, targetDataType); alignedChunkMetadata.setModified(true); } if (!useFakeStatistics) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index f59d0980d956a..320f00483f5ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -27,7 +27,6 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -400,25 +398,20 @@ public static void changeAlignedMetadataModified( } } - public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics( - AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType targetDataType) { - List newValueChunkMetadataList = new ArrayList<>(); - for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { - if (valueChunkMetadata != null - && targetDataType.isCompatible(valueChunkMetadata.getDataType())) { - Statistics statistics = Statistics.getStatsByType(targetDataType); - statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); - - ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; - newChunkMetadata.setTsDataType(targetDataType); - newChunkMetadata.setStatistics(statistics); - newValueChunkMetadataList.add(newChunkMetadata); - } else { - newValueChunkMetadataList.add(null); - } + public static void rewriteAlignedChunkMetadataStatistics( + AbstractAlignedChunkMetadata alignedChunkMetadata, int index, TSDataType targetDataType) { + IChunkMetadata valueChunkMetadata = alignedChunkMetadata.getValueChunkMetadataList().get(index); + if (valueChunkMetadata != null + && targetDataType.isCompatible(valueChunkMetadata.getDataType())) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); + + ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; + newChunkMetadata.setTsDataType(targetDataType); + newChunkMetadata.setStatistics(statistics); + } else { + alignedChunkMetadata.getValueChunkMetadataList().set(index, null); } - return new AlignedChunkMetadata( - alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList); } public static void rewriteNonAlignedChunkMetadataStatistics( From 14f88e3ef2362a6c4f61cc890fbacf88169aa90e Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 12 Jan 2026 14:33:41 +0800 Subject: [PATCH 16/17] Fix unit test. --- .../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 10dcf3e284415..baf60a724fe09 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -108,7 +108,7 @@ public void rewriteAlignedChunkMetadataStatistics() { try { AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = SchemaUtils.rewriteAlignedChunkMetadataStatistics( - alignedChunkMetadata, targetDataType); + alignedChunkMetadata, 0, targetDataType); if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { Assert.assertEquals( targetDataType, @@ -205,9 +205,11 @@ public void mergeMetadataStatistics() throws Exception { AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = (AbstractAlignedChunkMetadata) alignedChunkMetadata; try { - abstractAlignedChunkMetadata = - SchemaUtils.rewriteAlignedChunkMetadataStatistics( - abstractAlignedChunkMetadata, targetDataType); + for (int i = 0; i < 2; i++) { + abstractAlignedChunkMetadata = + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + abstractAlignedChunkMetadata, i, targetDataType); + } } catch (ClassCastException e) { Assert.fail(e.getMessage()); } From 517ffad7cd7a9ed485a47b98560847b195a9a7c1 Mon Sep 17 00:00:00 2001 From: Nero Date: Mon, 12 Jan 2026 15:23:45 +0800 Subject: [PATCH 17/17] Fix unit test. --- .../iotdb/db/utils/SchemaUtilsTest.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index baf60a724fe09..cffc16f22ac0d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -91,7 +91,9 @@ public void checkDataTypeWithEncoding() { public void rewriteAlignedChunkMetadataStatistics() { for (TSDataType targetDataType : Arrays.asList(TSDataType.STRING, TSDataType.TEXT)) { for (TSDataType tsDataType : TSDataType.values()) { - if (tsDataType == TSDataType.UNKNOWN) { + if (tsDataType == TSDataType.UNKNOWN + || tsDataType == TSDataType.VECTOR + || tsDataType == TSDataType.OBJECT) { continue; } List valueChunkMetadatas = @@ -106,13 +108,13 @@ public void rewriteAlignedChunkMetadataStatistics() { AlignedChunkMetadata alignedChunkMetadata = new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); try { - AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = - SchemaUtils.rewriteAlignedChunkMetadataStatistics( - alignedChunkMetadata, 0, targetDataType); - if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + alignedChunkMetadata, 0, targetDataType); + if (alignedChunkMetadata != null + && !alignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { Assert.assertEquals( targetDataType, - abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + alignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); } } catch (ClassCastException e) { Assert.fail(e.getMessage()); @@ -206,9 +208,8 @@ public void mergeMetadataStatistics() throws Exception { (AbstractAlignedChunkMetadata) alignedChunkMetadata; try { for (int i = 0; i < 2; i++) { - abstractAlignedChunkMetadata = - SchemaUtils.rewriteAlignedChunkMetadataStatistics( - abstractAlignedChunkMetadata, i, targetDataType); + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + abstractAlignedChunkMetadata, i, targetDataType); } } catch (ClassCastException e) { Assert.fail(e.getMessage());