Skip to content

Fix multi-value IN on INT32/INT64 fields in the table model #19277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Karman580 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix multi-value IN on INT32/INT64 fields in the table model #19277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 129 additions & 0 deletions ...est/java/org/apache/iotdb/relational/it/query/recent/IoTDBPredicateConversionTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,4 +376,133 @@ public void testDoubleLiteralOverflow() {
allInt64Rows,
DATABASE_NAME);
}

@Test
public void testMultiValueInPredicateAndInt32Column() {
String[] header = {"time", "int32_col"};
String[] onlyRow1 = {"1970-01-01T00:00:00.001Z,20,"};

// 4294967326 and -4294967266 wrap to 30 when narrowed to INT32, so they must not match row 3
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 4294967326) ORDER BY time",
header,
onlyRow1,
DATABASE_NAME);
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, -4294967266) ORDER BY time",
header,
onlyRow1,
DATABASE_NAME);
// OR of equalities is normalized into the same IN predicate
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col = 20 OR int32_col = 4294967326 ORDER BY time",
header,
onlyRow1,
DATABASE_NAME);
// duplicate and integral decimal values
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (20, 20.0, 20) ORDER BY time",
header,
onlyRow1,
DATABASE_NAME);

// non-integral decimals never match
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (29, 29.5) ORDER BY time",
header,
new String[] {"1970-01-01T00:00:00.002Z,29,"},
DATABASE_NAME);
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col IN (30.0, 31.5) ORDER BY time",
header,
new String[] {"1970-01-01T00:00:00.003Z,30,"},
DATABASE_NAME);

// NOT IN keeps row 3 and still excludes NULL rows 5 and 7
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (20, 4294967326) ORDER BY time",
header,
new String[] {
"1970-01-01T00:00:00.002Z,29,",
"1970-01-01T00:00:00.003Z,30,",
"1970-01-01T00:00:00.004Z,31,",
"1970-01-01T00:00:00.006Z,35,",
"1970-01-01T00:00:00.008Z,1000,",
"1970-01-01T00:00:00.009Z,-29,",
"1970-01-01T00:00:00.010Z,-30,"
},
DATABASE_NAME);
// no literal can match: NOT IN returns every non-NULL row
tableResultSetEqualTest(
"SELECT time, int32_col FROM PredicateTestTable WHERE int32_col NOT IN (29.5, 4294967326) ORDER BY time",
header,
new String[] {
"1970-01-01T00:00:00.001Z,20,",
"1970-01-01T00:00:00.002Z,29,",
"1970-01-01T00:00:00.003Z,30,",
"1970-01-01T00:00:00.004Z,31,",
"1970-01-01T00:00:00.006Z,35,",
"1970-01-01T00:00:00.008Z,1000,",
"1970-01-01T00:00:00.009Z,-29,",
"1970-01-01T00:00:00.010Z,-30,"
},
DATABASE_NAME);

// projection is evaluated by the column transformer instead of the pushed-down filter
tableResultSetEqualTest(
"SELECT time, int32_col IN (29, 29.5, 4294967326) AS m FROM PredicateTestTable ORDER BY time",
new String[] {"time", "m"},
new String[] {
"1970-01-01T00:00:00.001Z,false,",
"1970-01-01T00:00:00.002Z,true,",
"1970-01-01T00:00:00.003Z,false,",
"1970-01-01T00:00:00.004Z,false,",
"1970-01-01T00:00:00.005Z,null,",
"1970-01-01T00:00:00.006Z,false,",
"1970-01-01T00:00:00.007Z,null,",
"1970-01-01T00:00:00.008Z,false,",
"1970-01-01T00:00:00.009Z,false,",
"1970-01-01T00:00:00.010Z,false,"
},
DATABASE_NAME);
}

@Test
public void testMultiValueInPredicateAndInt64Column() {
String[] header = {"time", "int64_col"};
tableResultSetEqualTest(
"SELECT time, int64_col FROM PredicateTestTable WHERE int64_col IN (29, 29.5) ORDER BY time",
header,
new String[] {"1970-01-01T00:00:00.002Z,29,"},
DATABASE_NAME);
tableResultSetEqualTest(
"SELECT time, int64_col FROM PredicateTestTable WHERE int64_col NOT IN (20, 29.5) ORDER BY time",
header,
new String[] {
"1970-01-01T00:00:00.002Z,29,",
"1970-01-01T00:00:00.003Z,30,",
"1970-01-01T00:00:00.004Z,31,",
"1970-01-01T00:00:00.005Z,35,",
"1970-01-01T00:00:00.008Z,1000,",
"1970-01-01T00:00:00.009Z,-29,",
"1970-01-01T00:00:00.010Z,-30,"
},
DATABASE_NAME);
tableResultSetEqualTest(
"SELECT time, int64_col IN (29, 29.5) AS m FROM PredicateTestTable ORDER BY time",
new String[] {"time", "m"},
new String[] {
"1970-01-01T00:00:00.001Z,false,",
"1970-01-01T00:00:00.002Z,true,",
"1970-01-01T00:00:00.003Z,false,",
"1970-01-01T00:00:00.004Z,false,",
"1970-01-01T00:00:00.005Z,false,",
"1970-01-01T00:00:00.006Z,null,",
"1970-01-01T00:00:00.007Z,null,",
"1970-01-01T00:00:00.008Z,false,",
"1970-01-01T00:00:00.009Z,false,",
"1970-01-01T00:00:00.010Z,false,"
},
DATABASE_NAME);
}
}
44 changes: 34 additions & 10 deletions iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/TypeServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeEnum;
import org.apache.tsfile.read.common.type.service.TypeService;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
Expand Down Expand Up @@ -185,6 +186,32 @@ private static <T> TypeService<T> cached(final TypeService<T> service) {
return type -> cache.computeIfAbsent(type.getTypeEnum(), ignored -> service.call(type));
}

/**
* Converts a numeric IN-list literal for an INT32 or INT64 column. Returns {@code null} when the
* literal can never equal a value of the column (out of range or not an integer), which matches
* how {@code =} treats such literals.
*/
public static Long getIntegralInListValue(final Literal value, final Type type) {
final boolean isInt32 = type.getTypeEnum() == TypeEnum.INT32;
final long min = isInt32 ? Integer.MIN_VALUE : Long.MIN_VALUE;
final long max = isInt32 ? Integer.MAX_VALUE : Long.MAX_VALUE;
if (value instanceof LongLiteral) {
final long v = ((LongLiteral) value).getParsedValue();
return v >= min && v <= max ? v : null;
}
final double v;
if (value instanceof DoubleLiteral) {
v = ((DoubleLiteral) value).getValue();
} else if (value instanceof FloatLiteral) {
v = ((FloatLiteral) value).getValue();
} else {
throw new SemanticException(
String.format(CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum()));
}
// (double) max + 1 is exact for INT32 and rounds to 2^63 for INT64, so the bound stays exact.
return v == Math.floor(v) && v >= min && v < (double) max + 1 ? (long) v : null;
}

private static final IdentityLinearFill IDENTITY_LINEAR_FILL = new IdentityLinearFill();
private static final IdentityFill IDENTITY_FILL = new IdentityFill();

Expand Down Expand Up @@ -1137,15 +1164,9 @@ public Object castBinary(Binary value, ZoneId zoneId) {
(valueColumnTransformers, values) -> {
Set<Integer> intSet = new HashSet<>();
for (Literal value : values) {
try {
long v = ((LongLiteral) value).getParsedValue();
if (v <= Integer.MAX_VALUE && v >= Integer.MIN_VALUE) {
intSet.add((int) v);
}
} catch (IllegalArgumentException e) {
throw new SemanticException(
String.format(
CalcMessages.CANNOT_CAST_TO_TYPE, value, type.getTypeEnum()));
Long v = getIntegralInListValue(value, type);
if (v != null) {
intSet.add(v.intValue());
}
}
return new InInt32MultiColumnTransformer(intSet, valueColumnTransformers);
Expand All @@ -1162,7 +1183,10 @@ public Object castBinary(Binary value, ZoneId zoneId) {
(valueColumnTransformers, values) -> {
Set<Long> longSet = new HashSet<>();
for (Literal value : values) {
longSet.add(((LongLiteral) value).getParsedValue());
Long v = getIntegralInListValue(value, type);
if (v != null) {
longSet.add(v);
}
}
return new InInt64MultiColumnTransformer(longSet, valueColumnTransformers);
};
Expand Down
45 changes: 45 additions & 0 deletions iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/utils/TypeServicesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
import org.apache.iotdb.calc.execution.operator.process.window.partition.Partition;
import org.apache.iotdb.calc.execution.operator.process.window.utils.ColumnList;
import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral;

import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.block.column.ColumnBuilder;
Expand All @@ -36,6 +40,8 @@
import org.apache.tsfile.read.common.block.column.IntColumn;
import org.apache.tsfile.read.common.block.column.LongColumn;
import org.apache.tsfile.read.common.block.column.TimeColumn;
import org.apache.tsfile.read.common.type.IntType;
import org.apache.tsfile.read.common.type.LongType;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.TsPrimitiveType;
Expand All @@ -51,11 +57,50 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;

public class TypeServicesTest {

@Test
public void testIntegralInListValue() {
// INT32: in-range integers and integral decimals are kept, others never match
assertEquals(Long.valueOf(1), integralInListValue("1", IntType.INT32));
assertEquals(Long.valueOf(2), integralInListValue("2.0", IntType.INT32));
assertEquals(Long.valueOf(-3), integralInListValue("-3.0", IntType.INT32));
assertEquals(
Long.valueOf(Integer.MIN_VALUE), integralInListValue("-2147483648", IntType.INT32));
assertEquals(Long.valueOf(Integer.MAX_VALUE), integralInListValue("2147483647", IntType.INT32));
assertNull(integralInListValue("2147483648", IntType.INT32));
assertNull(integralInListValue("-2147483649", IntType.INT32));
assertNull(integralInListValue("4294967298", IntType.INT32));
assertNull(integralInListValue("-4294967293", IntType.INT32));
assertNull(integralInListValue("2.5", IntType.INT32));
assertNull(integralInListValue("2147483648.0", IntType.INT32));

// INT64
assertEquals(Long.valueOf(4294967298L), integralInListValue("4294967298", LongType.INT64));
assertEquals(Long.valueOf(3), integralInListValue("3.0", LongType.INT64));
assertEquals(
Long.valueOf(Long.MIN_VALUE),
integralInListValue("-9223372036854775808.0", LongType.INT64));
assertNull(integralInListValue("2.5", LongType.INT64));
assertNull(integralInListValue("9223372036854775808.0", LongType.INT64));
assertNull(integralInListValue("-9.3E18", LongType.INT64));

assertThrows(
SemanticException.class,
() -> TypeServices.getIntegralInListValue(new StringLiteral("2"), IntType.INT32));
}

private static Long integralInListValue(String literal, Type type) {
boolean isDecimal = literal.contains(".") || literal.contains("E");
return TypeServices.getIntegralInListValue(
isDecimal ? new DoubleLiteral(literal) : new LongLiteral(literal), type);
}

@Test
public void testStatisticsValueSetterCreatesExceptionOnlyForUnsupportedType() {
AtomicInteger exceptionCount = new AtomicInteger();
Expand Down
18 changes: 17 additions & 1 deletion ...db/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn;
import static org.apache.iotdb.calc.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn;
import static org.apache.iotdb.calc.transformation.dag.util.CommonTransformUtils.getEscapeCharacter;
import static org.apache.iotdb.calc.utils.TypeServices.getIntegralInListValue;
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor.getLongValue;
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral;
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference;
Expand Down Expand Up @@ -137,10 +138,25 @@ private <T extends Comparable<T>> Filter constructInFilter(
return ValueFilterApi.in(measurementIndex, inSet, dataType);
}

@SuppressWarnings("unchecked")
private <T extends Comparable<T>> Set<T> constructInSet(List<Literal> literals, Type dataType) {
TSDataType tsDataType = InternalTypeManager.getTSDataType(dataType);
Set<T> values = new HashSet<>();
for (Literal literal : literals) {
values.add(getValue(literal, dataType));
if (tsDataType != INT32 && tsDataType != INT64) {
values.add(getValue(literal, dataType));
continue;
}
// Like "=", drop literals that can never equal an INT32/INT64 value instead of narrowing them
Long value = getIntegralInListValue(literal, dataType);
if (value == null) {
continue;
}
if (tsDataType == INT32) {
values.add((T) Integer.valueOf(value.intValue()));
} else {
values.add((T) value);
}
}
return values;
}
Expand Down
Loading