From 80ef5f41aee4bddc75678f29d8e34c9dc6638d3c Mon Sep 17 00:00:00 2001 From: Karman Singh Talwar Date: Sat, 14 Nov 2026 05:46:43 +0530 Subject: [PATCH] Fix multi-value IN on INT32/INT64 fields in the table model A multi-value IN list on an INT32 or INT64 field converted its literals without the range and decimal handling that "=" already has. Out-of-range integers were narrowed with an unchecked int cast and matched unrelated rows, and decimal literals were cast to LongLiteral, which failed with a ClassCastException. Both the pushed-down filter and the projection evaluator were affected, and "x = a OR x = b" hit the same code after it is normalized into IN. Convert each literal through a shared helper that keeps integers inside the column range and integral decimals, and drops literals that can never match. An impossible list becomes an empty IN set, so NOT IN still excludes NULL rows. Closes #19276 --- .../IoTDBPredicateConversionTableIT.java | 129 ++++++++++++++++++ .../apache/iotdb/calc/utils/TypeServices.java | 44 ++++-- .../iotdb/calc/utils/TypeServicesTest.java | 45 ++++++ .../ConvertPredicateToFilterVisitor.java | 18 ++- .../TablePredicateConversionTest.java | 76 +++++++++++ 5 files changed, 301 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java index 438757a334c52..b40fb4c477970 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java @@ -376,4 +376,133 @@ public void testDoubleLiteralOverflow() { allInt64Rows, DATABASE_NAME); } + + @Test + public void testMultiValueInPredicateAndInt32Column() { + String[] header = {"time", "int32_col"}; + String[] onlyRow1 = {"1970-01-01T00:00:00.001Z,20,"}; + + // 4294967326 and -4294967266 wrap to 30 when narrowed to INT32, so they must not match row 3 + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 4294967326) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, -4294967266) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + // OR of equalities is normalized into the same IN predicate + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col = 20 OR int32_col = 4294967326 ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + // duplicate and integral decimal values + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 20.0, 20) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + + // non-integral decimals never match + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (29, 29.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.002Z,29,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (30.0, 31.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.003Z,30,"}, + DATABASE_NAME); + + // NOT IN keeps row 3 and still excludes NULL rows 5 and 7 + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (20, 4294967326) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.006Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + // no literal can match: NOT IN returns every non-NULL row + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (29.5, 4294967326) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.001Z,20,", + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.006Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + + // projection is evaluated by the column transformer instead of the pushed-down filter + tableResultSetEqualTest( + "SELECT time, int32_col IN (29, 29.5, 4294967326) AS m FROM PredicateTestTable ORDER BY time", + new String[] {"time", "m"}, + new String[] { + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,false,", + "1970-01-01T00:00:00.004Z,false,", + "1970-01-01T00:00:00.005Z,null,", + "1970-01-01T00:00:00.006Z,false,", + "1970-01-01T00:00:00.007Z,null,", + "1970-01-01T00:00:00.008Z,false,", + "1970-01-01T00:00:00.009Z,false,", + "1970-01-01T00:00:00.010Z,false," + }, + DATABASE_NAME); + } + + @Test + public void testMultiValueInPredicateAndInt64Column() { + String[] header = {"time", "int64_col"}; + tableResultSetEqualTest( + "SELECT time, int64_col FROM PredicateTestTable WHERE int64_col IN (29, 29.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.002Z,29,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int64_col FROM PredicateTestTable WHERE int64_col NOT IN (20, 29.5) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.005Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int64_col IN (29, 29.5) AS m FROM PredicateTestTable ORDER BY time", + new String[] {"time", "m"}, + new String[] { + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,false,", + "1970-01-01T00:00:00.004Z,false,", + "1970-01-01T00:00:00.005Z,false,", + "1970-01-01T00:00:00.006Z,null,", + "1970-01-01T00:00:00.007Z,null,", + "1970-01-01T00:00:00.008Z,false,", + "1970-01-01T00:00:00.009Z,false,", + "1970-01-01T00:00:00.010Z,false," + }, + DATABASE_NAME); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java index 1dd102277766b..f87c7ab0eb600 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java @@ -145,6 +145,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -185,6 +186,32 @@ private static TypeService cached(final TypeService service) { return type -> cache.computeIfAbsent(type.getTypeEnum(), ignored -> service.call(type)); } + /** + * Converts a numeric IN-list literal for an INT32 or INT64 column. Returns {@code null} when the + * literal can never equal a value of the column (out of range or not an integer), which matches + * how {@code =} treats such literals. + */ + public static Long getIntegralInListValue(final Literal value, final Type type) { + final boolean isInt32 = type.getTypeEnum() == TypeEnum.INT32; + final long min = isInt32 ? Integer.MIN_VALUE : Long.MIN_VALUE; + final long max = isInt32 ? Integer.MAX_VALUE : Long.MAX_VALUE; + if (value instanceof LongLiteral) { + final long v = ((LongLiteral) value).getParsedValue(); + return v >= min && v <= max ? v : null; + } + final double v; + if (value instanceof DoubleLiteral) { + v = ((DoubleLiteral) value).getValue(); + } else if (value instanceof FloatLiteral) { + v = ((FloatLiteral) value).getValue(); + } else { + throw new SemanticException( + String.format(CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + // (double) max + 1 is exact for INT32 and rounds to 2^63 for INT64, so the bound stays exact. + return v == Math.floor(v) && v >= min && v < (double) max + 1 ? (long) v : null; + } + private static final IdentityLinearFill IDENTITY_LINEAR_FILL = new IdentityLinearFill(); private static final IdentityFill IDENTITY_FILL = new IdentityFill(); @@ -1137,15 +1164,9 @@ public Object castBinary(Binary value, ZoneId zoneId) { (valueColumnTransformers, values) -> { Set intSet = new HashSet<>(); for (Literal value : values) { - try { - long v = ((LongLiteral) value).getParsedValue(); - if (v <= Integer.MAX_VALUE && v >= Integer.MIN_VALUE) { - intSet.add((int) v); - } - } catch (IllegalArgumentException e) { - throw new SemanticException( - String.format( - CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + Long v = getIntegralInListValue(value, type); + if (v != null) { + intSet.add(v.intValue()); } } return new InInt32MultiColumnTransformer(intSet, valueColumnTransformers); @@ -1162,7 +1183,10 @@ public Object castBinary(Binary value, ZoneId zoneId) { (valueColumnTransformers, values) -> { Set longSet = new HashSet<>(); for (Literal value : values) { - longSet.add(((LongLiteral) value).getParsedValue()); + Long v = getIntegralInListValue(value, type); + if (v != null) { + longSet.add(v); + } } return new InInt64MultiColumnTransformer(longSet, valueColumnTransformers); }; diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java index ffa1f6965236a..8b0f174e6e2fd 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java @@ -24,6 +24,10 @@ import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; import org.apache.iotdb.calc.execution.operator.process.window.utils.ColumnList; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -36,6 +40,8 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.IntType; +import org.apache.tsfile.read.common.type.LongType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -51,11 +57,50 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class TypeServicesTest { + @Test + public void testIntegralInListValue() { + // INT32: in-range integers and integral decimals are kept, others never match + assertEquals(Long.valueOf(1), integralInListValue("1", IntType.INT32)); + assertEquals(Long.valueOf(2), integralInListValue("2.0", IntType.INT32)); + assertEquals(Long.valueOf(-3), integralInListValue("-3.0", IntType.INT32)); + assertEquals( + Long.valueOf(Integer.MIN_VALUE), integralInListValue("-2147483648", IntType.INT32)); + assertEquals(Long.valueOf(Integer.MAX_VALUE), integralInListValue("2147483647", IntType.INT32)); + assertNull(integralInListValue("2147483648", IntType.INT32)); + assertNull(integralInListValue("-2147483649", IntType.INT32)); + assertNull(integralInListValue("4294967298", IntType.INT32)); + assertNull(integralInListValue("-4294967293", IntType.INT32)); + assertNull(integralInListValue("2.5", IntType.INT32)); + assertNull(integralInListValue("2147483648.0", IntType.INT32)); + + // INT64 + assertEquals(Long.valueOf(4294967298L), integralInListValue("4294967298", LongType.INT64)); + assertEquals(Long.valueOf(3), integralInListValue("3.0", LongType.INT64)); + assertEquals( + Long.valueOf(Long.MIN_VALUE), + integralInListValue("-9223372036854775808.0", LongType.INT64)); + assertNull(integralInListValue("2.5", LongType.INT64)); + assertNull(integralInListValue("9223372036854775808.0", LongType.INT64)); + assertNull(integralInListValue("-9.3E18", LongType.INT64)); + + assertThrows( + SemanticException.class, + () -> TypeServices.getIntegralInListValue(new StringLiteral("2"), IntType.INT32)); + } + + private static Long integralInListValue(String literal, Type type) { + boolean isDecimal = literal.contains(".") || literal.contains("E"); + return TypeServices.getIntegralInListValue( + isDecimal ? new DoubleLiteral(literal) : new LongLiteral(literal), type); + } + @Test public void testStatisticsValueSetterCreatesExceptionOnlyForUnsupportedType() { AtomicInteger exceptionCount = new AtomicInteger(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index ecd45a638b7b6..2f940cac0c63a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -75,6 +75,7 @@ import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn; import static org.apache.iotdb.calc.transformation.dag.util.CommonTransformUtils.getEscapeCharacter; +import static org.apache.iotdb.calc.utils.TypeServices.getIntegralInListValue; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor.getLongValue; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; @@ -137,10 +138,25 @@ private > Filter constructInFilter( return ValueFilterApi.in(measurementIndex, inSet, dataType); } + @SuppressWarnings("unchecked") private > Set constructInSet(List literals, Type dataType) { + TSDataType tsDataType = InternalTypeManager.getTSDataType(dataType); Set values = new HashSet<>(); for (Literal literal : literals) { - values.add(getValue(literal, dataType)); + if (tsDataType != INT32 && tsDataType != INT64) { + values.add(getValue(literal, dataType)); + continue; + } + // Like "=", drop literals that can never equal an INT32/INT64 value instead of narrowing them + Long value = getIntegralInListValue(literal, dataType); + if (value == null) { + continue; + } + if (tsDataType == INT32) { + values.add((T) Integer.valueOf(value.intValue())); + } else { + values.add((T) value); + } } return values; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java index 3c38de79b2c85..6e7f136993ba0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java @@ -24,20 +24,29 @@ import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.type.IntType; import org.apache.tsfile.read.common.type.LongType; import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.read.filter.operator.FalseLiteralFilter; import org.apache.tsfile.read.filter.operator.LongFilterOperators; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD; @@ -503,4 +512,71 @@ public void testInt64LiteralAndInt32Column() { Assert.assertEquals(new LongFilterOperators.ValueGtEq(0, 123), actualFilter); } + + @Test + public void testMultiValueInPredicateAndInt32Column() { + // Out-of-range integers must not be narrowed onto other INT32 values, and decimal literals + // must behave like "=": integral ones match, non-integral ones never match. + Filter filter = + convertInt32( + inPredicate( + int32Column, "1", "4294967298", "-4294967293", "2147483648", "2.5", "3.0", "1")); + Assert.assertEquals( + ValueFilterApi.in(0, new HashSet<>(Arrays.asList(1, 3)), TSDataType.INT32), filter); + Assert.assertFalse(filter.satisfyInteger(0, 2)); + Assert.assertFalse(filter.satisfyInteger(0, Integer.MIN_VALUE)); + + // NOT IN keeps rows whose value would have been hit by a narrowed literal + Filter notInFilter = + convertInt32(new NotExpression(inPredicate(int32Column, "1", "4294967298"))); + Assert.assertFalse(notInFilter.satisfyInteger(0, 1)); + Assert.assertTrue(notInFilter.satisfyInteger(0, 2)); + + // No literal can match: IN matches nothing and NOT IN matches every non-null value + Filter impossible = convertInt32(inPredicate(int32Column, "4294967298", "2.5")); + Assert.assertEquals( + ValueFilterApi.in(0, Collections.emptySet(), TSDataType.INT32), impossible); + Assert.assertFalse(impossible.satisfyInteger(0, 2)); + Filter notImpossible = + convertInt32(new NotExpression(inPredicate(int32Column, "4294967298", "2.5"))); + Assert.assertTrue(notImpossible.satisfyInteger(0, 2)); + Assert.assertTrue(notImpossible.satisfyInteger(0, Integer.MIN_VALUE)); + } + + @Test + public void testMultiValueInPredicateAndInt64Column() { + Filter filter = + PredicateUtils.convertPredicateToFilter( + inPredicate( + int64Column, + "2", + "2.5", + "3.0", + "4294967298", + "-9223372036854775808.0", + "9223372036854775808.0"), + int64MeasurementColumnsIndexMap, + int64SchemaMap, + null, + null, + null); + Assert.assertEquals( + ValueFilterApi.in( + 0, new HashSet<>(Arrays.asList(2L, 3L, 4294967298L, Long.MIN_VALUE)), TSDataType.INT64), + filter); + Assert.assertFalse(filter.satisfyLong(0, Long.MAX_VALUE)); + } + + private Filter convertInt32(Expression predicate) { + return PredicateUtils.convertPredicateToFilter( + predicate, int32MeasurementColumnsIndexMap, int32SchemaMap, null, null, null); + } + + private static InPredicate inPredicate(SymbolReference column, String... literals) { + List values = new ArrayList<>(); + for (String literal : literals) { + values.add(literal.contains(".") ? new DoubleLiteral(literal) : new LongLiteral(literal)); + } + return new InPredicate(column, new InListExpression(values)); + } }