diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java index 438757a334c52..b40fb4c477970 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java @@ -376,4 +376,133 @@ public void testDoubleLiteralOverflow() { allInt64Rows, DATABASE_NAME); } + + @Test + public void testMultiValueInPredicateAndInt32Column() { + String[] header = {"time", "int32_col"}; + String[] onlyRow1 = {"1970-01-01T00:00:00.001Z,20,"}; + + // 4294967326 and -4294967266 wrap to 30 when narrowed to INT32, so they must not match row 3 + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 4294967326) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, -4294967266) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + // OR of equalities is normalized into the same IN predicate + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col = 20 OR int32_col = 4294967326 ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + // duplicate and integral decimal values + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 20.0, 20) ORDER BY time", + header, + onlyRow1, + DATABASE_NAME); + + // non-integral decimals never match + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (29, 29.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.002Z,29,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (30.0, 31.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.003Z,30,"}, + DATABASE_NAME); + + // NOT IN keeps row 3 and still excludes NULL rows 5 and 7 + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (20, 4294967326) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.006Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + // no literal can match: NOT IN returns every non-NULL row + tableResultSetEqualTest( + "SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (29.5, 4294967326) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.001Z,20,", + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.006Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + + // projection is evaluated by the column transformer instead of the pushed-down filter + tableResultSetEqualTest( + "SELECT time, int32_col IN (29, 29.5, 4294967326) AS m FROM PredicateTestTable ORDER BY time", + new String[] {"time", "m"}, + new String[] { + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,false,", + "1970-01-01T00:00:00.004Z,false,", + "1970-01-01T00:00:00.005Z,null,", + "1970-01-01T00:00:00.006Z,false,", + "1970-01-01T00:00:00.007Z,null,", + "1970-01-01T00:00:00.008Z,false,", + "1970-01-01T00:00:00.009Z,false,", + "1970-01-01T00:00:00.010Z,false," + }, + DATABASE_NAME); + } + + @Test + public void testMultiValueInPredicateAndInt64Column() { + String[] header = {"time", "int64_col"}; + tableResultSetEqualTest( + "SELECT time, int64_col FROM PredicateTestTable WHERE int64_col IN (29, 29.5) ORDER BY time", + header, + new String[] {"1970-01-01T00:00:00.002Z,29,"}, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int64_col FROM PredicateTestTable WHERE int64_col NOT IN (20, 29.5) ORDER BY time", + header, + new String[] { + "1970-01-01T00:00:00.002Z,29,", + "1970-01-01T00:00:00.003Z,30,", + "1970-01-01T00:00:00.004Z,31,", + "1970-01-01T00:00:00.005Z,35,", + "1970-01-01T00:00:00.008Z,1000,", + "1970-01-01T00:00:00.009Z,-29,", + "1970-01-01T00:00:00.010Z,-30," + }, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, int64_col IN (29, 29.5) AS m FROM PredicateTestTable ORDER BY time", + new String[] {"time", "m"}, + new String[] { + "1970-01-01T00:00:00.001Z,false,", + "1970-01-01T00:00:00.002Z,true,", + "1970-01-01T00:00:00.003Z,false,", + "1970-01-01T00:00:00.004Z,false,", + "1970-01-01T00:00:00.005Z,false,", + "1970-01-01T00:00:00.006Z,null,", + "1970-01-01T00:00:00.007Z,null,", + "1970-01-01T00:00:00.008Z,false,", + "1970-01-01T00:00:00.009Z,false,", + "1970-01-01T00:00:00.010Z,false," + }, + DATABASE_NAME); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java index 1dd102277766b..f87c7ab0eb600 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java @@ -145,6 +145,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -185,6 +186,32 @@ private static TypeService cached(final TypeService service) { return type -> cache.computeIfAbsent(type.getTypeEnum(), ignored -> service.call(type)); } + /** + * Converts a numeric IN-list literal for an INT32 or INT64 column. Returns {@code null} when the + * literal can never equal a value of the column (out of range or not an integer), which matches + * how {@code =} treats such literals. + */ + public static Long getIntegralInListValue(final Literal value, final Type type) { + final boolean isInt32 = type.getTypeEnum() == TypeEnum.INT32; + final long min = isInt32 ? Integer.MIN_VALUE : Long.MIN_VALUE; + final long max = isInt32 ? Integer.MAX_VALUE : Long.MAX_VALUE; + if (value instanceof LongLiteral) { + final long v = ((LongLiteral) value).getParsedValue(); + return v >= min && v <= max ? v : null; + } + final double v; + if (value instanceof DoubleLiteral) { + v = ((DoubleLiteral) value).getValue(); + } else if (value instanceof FloatLiteral) { + v = ((FloatLiteral) value).getValue(); + } else { + throw new SemanticException( + String.format(CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + } + // (double) max + 1 is exact for INT32 and rounds to 2^63 for INT64, so the bound stays exact. + return v == Math.floor(v) && v >= min && v < (double) max + 1 ? (long) v : null; + } + private static final IdentityLinearFill IDENTITY_LINEAR_FILL = new IdentityLinearFill(); private static final IdentityFill IDENTITY_FILL = new IdentityFill(); @@ -1137,15 +1164,9 @@ public Object castBinary(Binary value, ZoneId zoneId) { (valueColumnTransformers, values) -> { Set intSet = new HashSet<>(); for (Literal value : values) { - try { - long v = ((LongLiteral) value).getParsedValue(); - if (v <= Integer.MAX_VALUE && v >= Integer.MIN_VALUE) { - intSet.add((int) v); - } - } catch (IllegalArgumentException e) { - throw new SemanticException( - String.format( - CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum())); + Long v = getIntegralInListValue(value, type); + if (v != null) { + intSet.add(v.intValue()); } } return new InInt32MultiColumnTransformer(intSet, valueColumnTransformers); @@ -1162,7 +1183,10 @@ public Object castBinary(Binary value, ZoneId zoneId) { (valueColumnTransformers, values) -> { Set longSet = new HashSet<>(); for (Literal value : values) { - longSet.add(((LongLiteral) value).getParsedValue()); + Long v = getIntegralInListValue(value, type); + if (v != null) { + longSet.add(v); + } } return new InInt64MultiColumnTransformer(longSet, valueColumnTransformers); }; diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java index ffa1f6965236a..8b0f174e6e2fd 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java @@ -24,6 +24,10 @@ import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition; import org.apache.iotdb.calc.execution.operator.process.window.utils.ColumnList; import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -36,6 +40,8 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.IntType; +import org.apache.tsfile.read.common.type.LongType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -51,11 +57,50 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class TypeServicesTest { + @Test + public void testIntegralInListValue() { + // INT32: in-range integers and integral decimals are kept, others never match + assertEquals(Long.valueOf(1), integralInListValue("1", IntType.INT32)); + assertEquals(Long.valueOf(2), integralInListValue("2.0", IntType.INT32)); + assertEquals(Long.valueOf(-3), integralInListValue("-3.0", IntType.INT32)); + assertEquals( + Long.valueOf(Integer.MIN_VALUE), integralInListValue("-2147483648", IntType.INT32)); + assertEquals(Long.valueOf(Integer.MAX_VALUE), integralInListValue("2147483647", IntType.INT32)); + assertNull(integralInListValue("2147483648", IntType.INT32)); + assertNull(integralInListValue("-2147483649", IntType.INT32)); + assertNull(integralInListValue("4294967298", IntType.INT32)); + assertNull(integralInListValue("-4294967293", IntType.INT32)); + assertNull(integralInListValue("2.5", IntType.INT32)); + assertNull(integralInListValue("2147483648.0", IntType.INT32)); + + // INT64 + assertEquals(Long.valueOf(4294967298L), integralInListValue("4294967298", LongType.INT64)); + assertEquals(Long.valueOf(3), integralInListValue("3.0", LongType.INT64)); + assertEquals( + Long.valueOf(Long.MIN_VALUE), + integralInListValue("-9223372036854775808.0", LongType.INT64)); + assertNull(integralInListValue("2.5", LongType.INT64)); + assertNull(integralInListValue("9223372036854775808.0", LongType.INT64)); + assertNull(integralInListValue("-9.3E18", LongType.INT64)); + + assertThrows( + SemanticException.class, + () -> TypeServices.getIntegralInListValue(new StringLiteral("2"), IntType.INT32)); + } + + private static Long integralInListValue(String literal, Type type) { + boolean isDecimal = literal.contains(".") || literal.contains("E"); + return TypeServices.getIntegralInListValue( + isDecimal ? new DoubleLiteral(literal) : new LongLiteral(literal), type); + } + @Test public void testStatisticsValueSetterCreatesExceptionOnlyForUnsupportedType() { AtomicInteger exceptionCount = new AtomicInteger(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index ecd45a638b7b6..2f940cac0c63a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -75,6 +75,7 @@ import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn; import static org.apache.iotdb.calc.transformation.dag.util.CommonTransformUtils.getEscapeCharacter; +import static org.apache.iotdb.calc.utils.TypeServices.getIntegralInListValue; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor.getLongValue; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; @@ -137,10 +138,25 @@ private > Filter constructInFilter( return ValueFilterApi.in(measurementIndex, inSet, dataType); } + @SuppressWarnings("unchecked") private > Set constructInSet(List literals, Type dataType) { + TSDataType tsDataType = InternalTypeManager.getTSDataType(dataType); Set values = new HashSet<>(); for (Literal literal : literals) { - values.add(getValue(literal, dataType)); + if (tsDataType != INT32 && tsDataType != INT64) { + values.add(getValue(literal, dataType)); + continue; + } + // Like "=", drop literals that can never equal an INT32/INT64 value instead of narrowing them + Long value = getIntegralInListValue(literal, dataType); + if (value == null) { + continue; + } + if (tsDataType == INT32) { + values.add((T) Integer.valueOf(value.intValue())); + } else { + values.add((T) value); + } } return values; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java index 3c38de79b2c85..6e7f136993ba0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/predicate/TablePredicateConversionTest.java @@ -24,20 +24,29 @@ import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.type.IntType; import org.apache.tsfile.read.common.type.LongType; import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.read.filter.operator.FalseLiteralFilter; import org.apache.tsfile.read.filter.operator.LongFilterOperators; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD; @@ -503,4 +512,71 @@ public void testInt64LiteralAndInt32Column() { Assert.assertEquals(new LongFilterOperators.ValueGtEq(0, 123), actualFilter); } + + @Test + public void testMultiValueInPredicateAndInt32Column() { + // Out-of-range integers must not be narrowed onto other INT32 values, and decimal literals + // must behave like "=": integral ones match, non-integral ones never match. + Filter filter = + convertInt32( + inPredicate( + int32Column, "1", "4294967298", "-4294967293", "2147483648", "2.5", "3.0", "1")); + Assert.assertEquals( + ValueFilterApi.in(0, new HashSet<>(Arrays.asList(1, 3)), TSDataType.INT32), filter); + Assert.assertFalse(filter.satisfyInteger(0, 2)); + Assert.assertFalse(filter.satisfyInteger(0, Integer.MIN_VALUE)); + + // NOT IN keeps rows whose value would have been hit by a narrowed literal + Filter notInFilter = + convertInt32(new NotExpression(inPredicate(int32Column, "1", "4294967298"))); + Assert.assertFalse(notInFilter.satisfyInteger(0, 1)); + Assert.assertTrue(notInFilter.satisfyInteger(0, 2)); + + // No literal can match: IN matches nothing and NOT IN matches every non-null value + Filter impossible = convertInt32(inPredicate(int32Column, "4294967298", "2.5")); + Assert.assertEquals( + ValueFilterApi.in(0, Collections.emptySet(), TSDataType.INT32), impossible); + Assert.assertFalse(impossible.satisfyInteger(0, 2)); + Filter notImpossible = + convertInt32(new NotExpression(inPredicate(int32Column, "4294967298", "2.5"))); + Assert.assertTrue(notImpossible.satisfyInteger(0, 2)); + Assert.assertTrue(notImpossible.satisfyInteger(0, Integer.MIN_VALUE)); + } + + @Test + public void testMultiValueInPredicateAndInt64Column() { + Filter filter = + PredicateUtils.convertPredicateToFilter( + inPredicate( + int64Column, + "2", + "2.5", + "3.0", + "4294967298", + "-9223372036854775808.0", + "9223372036854775808.0"), + int64MeasurementColumnsIndexMap, + int64SchemaMap, + null, + null, + null); + Assert.assertEquals( + ValueFilterApi.in( + 0, new HashSet<>(Arrays.asList(2L, 3L, 4294967298L, Long.MIN_VALUE)), TSDataType.INT64), + filter); + Assert.assertFalse(filter.satisfyLong(0, Long.MAX_VALUE)); + } + + private Filter convertInt32(Expression predicate) { + return PredicateUtils.convertPredicateToFilter( + predicate, int32MeasurementColumnsIndexMap, int32SchemaMap, null, null, null); + } + + private static InPredicate inPredicate(SymbolReference column, String... literals) { + List values = new ArrayList<>(); + for (String literal : literals) { + values.add(literal.contains(".") ? new DoubleLiteral(literal) : new LongLiteral(literal)); + } + return new InPredicate(column, new InListExpression(values)); + } }