Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.tree.TerminalNode;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.Objects;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -208,11 +210,47 @@ public Expression visitLogicalNot(
@Override
public Expression visitLogicalBinary(
final SubscriptionColumnFilterParser.LogicalBinaryContext context) {
final Expression left = visit(context.booleanExpression(0));
final Expression right = visit(context.booleanExpression(1));
final LogicalExpression.Operator operator = logicalOperator(context);
final List<Expression> terms = new ArrayList<>();
final Deque<SubscriptionColumnFilterParser.BooleanExpressionContext> stack =
new ArrayDeque<>();
stack.push(context.booleanExpression(1));
stack.push(context.booleanExpression(0));
while (!stack.isEmpty()) {
final SubscriptionColumnFilterParser.BooleanExpressionContext current =
unwrapParenthesized(stack.pop());
if (current instanceof SubscriptionColumnFilterParser.LogicalBinaryContext) {
final SubscriptionColumnFilterParser.LogicalBinaryContext binary =
(SubscriptionColumnFilterParser.LogicalBinaryContext) current;
if (logicalOperator(binary) == operator) {
stack.push(binary.booleanExpression(1));
stack.push(binary.booleanExpression(0));
continue;
}
}
terms.add(visit(current));
}
return new LogicalExpression(operator, terms);
}

private static LogicalExpression.Operator logicalOperator(
final SubscriptionColumnFilterParser.LogicalBinaryContext context) {
return Objects.nonNull(context.AND())
? LogicalExpression.and(left, right)
: LogicalExpression.or(left, right);
? LogicalExpression.Operator.AND
: LogicalExpression.Operator.OR;
}

private static SubscriptionColumnFilterParser.BooleanExpressionContext unwrapParenthesized(
SubscriptionColumnFilterParser.BooleanExpressionContext context) {
while (context instanceof SubscriptionColumnFilterParser.PredicateExpressionContext) {
final SubscriptionColumnFilterParser.PredicateContext predicate =
((SubscriptionColumnFilterParser.PredicateExpressionContext) context).predicate();
if (Objects.isNull(predicate.booleanExpression())) {
break;
}
context = predicate.booleanExpression();
}
return context;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,11 @@ public static Tablet filter(
// Upstream event conversion supplies a structurally valid Tablet. Only check the metadata
// needed to bind the predicate here; do not rescan every value array and null bitmap.
final List<ColumnCategory> categories = getColumnCategories(tablet, schemas.size());
validateBinding(effectiveMatcher, databaseName, tablet.getTableName());
final boolean schemaBound =
validateBinding(effectiveMatcher, databaseName, tablet.getTableName());
final Map<String, Integer> resolvedFields =
resolveReferencedTagColumns(effectiveMatcher.getReferencedFields(), schemas, categories);
resolveReferencedTagColumns(
effectiveMatcher.getReferencedFields(), schemas, categories, schemaBound);

final int[] selectedRows = new int[tablet.getRowSize()];
int selectedRowCount = 0;
Expand Down Expand Up @@ -114,20 +116,33 @@ public static Tablet filter(
return copySelectedRows(tablet, schemas, categories, selectedRows, selectedRowCount);
}

private static void validateBinding(
private static boolean validateBinding(
final TagFilterMatcher matcher, final String databaseName, final String tableName) {
if (!matcher.isBindingEnforced()) {
return;
}
final TagFilterMatcher.TableBinding binding =
matcher.getTableBindings().get(TagFilterMatcher.TableKey.of(databaseName, tableName));
if (!matcher.isBindingEnforced()) {
if (binding == null) {
return false;
}
if (binding.isFailed()) {
throw TagFilterEvaluationException.schemaBinding(binding.getFailureReason());
}
validateBindingTagColumns(matcher, binding);
return true;
}
if (binding == null) {
throw TagFilterEvaluationException.schemaBinding(
DataNodeMiscMessages.EXCEPTION_TABLE_BINDING_IS_NOT_AVAILABLE_882C5F2F);
}
if (binding.isFailed()) {
throw TagFilterEvaluationException.schemaBinding(binding.getFailureReason());
}
validateBindingTagColumns(matcher, binding);
return true;
}

private static void validateBindingTagColumns(
final TagFilterMatcher matcher, final TagFilterMatcher.TableBinding binding) {
for (final Identifier identifier : matcher.getReferencedFields()) {
final String normalizedName = identifier.getValue().toLowerCase(Locale.ROOT);
if (!binding.getTagNames().contains(normalizedName)) {
Expand All @@ -140,7 +155,8 @@ private static void validateBinding(
private static Map<String, Integer> resolveReferencedTagColumns(
final List<Identifier> referencedFields,
final List<IMeasurementSchema> schemas,
final List<ColumnCategory> categories) {
final List<ColumnCategory> categories,
final boolean schemaBound) {
final Map<String, Integer> exactColumns = new HashMap<>();
final Map<String, Integer> normalizedColumns = new HashMap<>();
for (int i = 0; i < schemas.size(); i++) {
Expand All @@ -156,6 +172,11 @@ private static Map<String, Integer> resolveReferencedTagColumns(
? exactColumns.get(identifier.getValue())
: normalizedColumns.get(identifier.getValue().toLowerCase(Locale.ROOT));
if (Objects.isNull(columnIndex)) {
if (schemaBound) {
// Table schema binding already confirmed this identifier is a valid TAG column; older
// tablets written before ALTER TABLE ADD COLUMN treat the absent TAG column as NULL.
continue;
}
throw TagFilterEvaluationException.schemaBinding(
DataNodeMiscMessages.EXCEPTION_REFERENCED_TAG_COLUMN_IS_MISSING_55E07377);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,19 @@

package org.apache.iotdb.db.subscription.columnfilter;

import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral;
import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;

import org.junit.Assert;
import org.junit.Test;

import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class ColumnFilterParserTest {

private static final ColumnFilterParser PARSER = new ColumnFilterParser();
Expand Down Expand Up @@ -79,6 +85,56 @@ public void testLogicalPrecedence() throws Exception {
"(category = \"TAG\" OR category = \"FIELD\") AND datatype = \"STRING\"", TEMPERATURE));
}

@Test
public void testFlattenedOrChainParsesIntoSingleLogicalExpression() throws Exception {
final int termCount = 500;
final String orChain =
IntStream.range(0, termCount)
.mapToObj(i -> "column_name = \"s" + i + "\"")
.collect(Collectors.joining(" OR "));
final Expression parsed = PARSER.parseAndValidate(orChain);
Assert.assertTrue(parsed instanceof LogicalExpression);
final LogicalExpression logical = (LogicalExpression) parsed;
Assert.assertEquals(LogicalExpression.Operator.OR, logical.getOperator());
Assert.assertEquals(termCount, logical.getTerms().size());
Assert.assertEquals(
"s0",
((StringLiteral) ((ComparisonExpression) logical.getTerms().get(0)).getRight()).getValue());
Assert.assertEquals(
"s" + (termCount - 1),
((StringLiteral) ((ComparisonExpression) logical.getTerms().get(termCount - 1)).getRight())
.getValue());
Assert.assertTrue(
ColumnFilterEvaluator.evaluate(
parsed, new ColumnMetadata("db1", "weather", "s499", "DOUBLE", "FIELD")));
Assert.assertFalse(ColumnFilterEvaluator.evaluate(parsed, TEMPERATURE));
}

@Test
public void testParenthesizedSameOperatorFlattensWhilePreservingMixedPrecedence()
throws Exception {
final Expression parsed =
PARSER.parseAndValidate(
"(column_name = \"s0\" OR (column_name = \"s1\" OR column_name = \"s2\"))"
+ " OR (column_name = \"s3\" AND datatype = \"DOUBLE\")");
Assert.assertTrue(parsed instanceof LogicalExpression);
final LogicalExpression root = (LogicalExpression) parsed;
Assert.assertEquals(LogicalExpression.Operator.OR, root.getOperator());
Assert.assertEquals(4, root.getTerms().size());
Assert.assertTrue(root.getTerms().get(3) instanceof LogicalExpression);
Assert.assertEquals(
LogicalExpression.Operator.AND, ((LogicalExpression) root.getTerms().get(3)).getOperator());
Assert.assertTrue(
ColumnFilterEvaluator.evaluate(
parsed, new ColumnMetadata("db1", "weather", "s2", "INT32", "FIELD")));
Assert.assertTrue(
ColumnFilterEvaluator.evaluate(
parsed, new ColumnMetadata("db1", "weather", "s3", "DOUBLE", "FIELD")));
Assert.assertFalse(
ColumnFilterEvaluator.evaluate(
parsed, new ColumnMetadata("db1", "weather", "s3", "INT32", "FIELD")));
}

@Test
public void testIsNullAndIsNotNull() throws Exception {
Assert.assertFalse(evaluate("column_name IS NULL", TAG));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@

package org.apache.iotdb.db.subscription.tagfilter;

import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TagColumnSchema;
import org.apache.iotdb.db.pipe.sink.util.sorter.PipeTableModelTabletEventSorter;
import org.apache.iotdb.rpc.subscription.config.TopicConfig;
import org.apache.iotdb.rpc.subscription.config.TopicConstant;
Expand All @@ -31,6 +34,7 @@
import org.junit.Test;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -134,13 +138,71 @@ public void testTrivialFilterReturnsOriginalTablet() throws Exception {
Assert.assertNull(TabletTagFilter.filter(tablet, TagFilterMatcher.matchNone()));
}

@Test
public void testBoundSchemaTreatsAbsentNewlyAddedTagColumnInOlderTabletAsNull() throws Exception {
final TsTable evolvedTable = new TsTable("weather");
evolvedTable.addColumnSchema(new TagColumnSchema("region", TSDataType.STRING));
evolvedTable.addColumnSchema(new TagColumnSchema("device", TSDataType.STRING));
evolvedTable.addColumnSchema(new TagColumnSchema("new_tag", TSDataType.STRING));
evolvedTable.addColumnSchema(new FieldColumnSchema("temperature", TSDataType.DOUBLE));
final Map<String, Map<String, TsTable>> tables =
Collections.singletonMap("db1", Collections.singletonMap("weather", evolvedTable));

// Older tablet written before ALTER TABLE ADD COLUMN new_tag STRING TAG lacks new_tag.
final Tablet olderTablet = createTablet();

final Tablet isNullFiltered =
TabletTagFilter.filter(
olderTablet, boundMatcher("db1", "weather", "new_tag IS NULL", tables), "db1");
Assert.assertSame(olderTablet, isNullFiltered);

Assert.assertNull(
TabletTagFilter.filter(
olderTablet, boundMatcher("db1", "weather", "new_tag = \"v1\"", tables), "db1"));

// SQL 3VL: NOT (NULL = "v1") evaluates to UNKNOWN (false), not TRUE.
Assert.assertNull(
TabletTagFilter.filter(
olderTablet, boundMatcher("db1", "weather", "NOT (new_tag = \"v1\")", tables), "db1"));

final Tablet orFiltered =
TabletTagFilter.filter(
olderTablet,
boundMatcher("db1", "weather", "new_tag = \"v1\" OR region = \"north\"", tables),
"db1");
Assert.assertNotNull(orFiltered);
Assert.assertEquals(1, orFiltered.getRowSize());
Assert.assertEquals(1L, orFiltered.getTimestamp(0));

// Wildcard topic that already has a valid TableBinding for (db1, weather) also treats absent
// newly-added TAG column in historical tablets as NULL.
final Tablet wildcardFiltered =
TabletTagFilter.filter(
olderTablet, boundMatcher("db1", ".*", "new_tag IS NULL", tables), "db1");
Assert.assertSame(olderTablet, wildcardFiltered);
}

private static TagFilterMatcher matcher(final String filter) throws Exception {
final Map<String, String> attributes = new HashMap<>();
attributes.put("__system.sql-dialect", "table");
attributes.put(TopicConstant.TAG_FILTER_KEY, filter);
return TagFilterMatcher.fromTopicConfig(new TopicConfig(attributes));
}

private static TagFilterMatcher boundMatcher(
final String database,
final String table,
final String filter,
final Map<String, Map<String, TsTable>> tables)
throws Exception {
final Map<String, String> attributes = new HashMap<>();
attributes.put("__system.sql-dialect", "table");
attributes.put(TopicConstant.DATABASE_KEY, database);
attributes.put(TopicConstant.TABLE_KEY, table);
attributes.put(TopicConstant.TAG_FILTER_KEY, filter);
return TagFilterMatcher.fromTopicConfig(new TopicConfig(attributes), tables);
}

private static void assertFailsClosed(final String filter) throws Exception {
try {
TabletTagFilter.filter(createTablet(), matcher(filter));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@
package org.apache.iotdb.db.subscription.tagfilter;

import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;

import org.junit.Assert;
import org.junit.Test;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class TagFilterParserTest {

Expand Down Expand Up @@ -96,6 +99,50 @@ public void testNullUsesThreeValuedLogic() throws Exception {
Assert.assertFalse(evaluate("NOT (true OR region = \"north\")", values));
}

@Test
public void testFlatLogicalChainsDoNotExceedMaxAstDepth() throws Exception {
final String orChain =
IntStream.range(0, 40)
.mapToObj(i -> "region = \"r" + i + "\"")
.collect(Collectors.joining(" OR "));
final Expression orExpression = PARSER.parseAndValidate(orChain);
Assert.assertTrue(orExpression instanceof LogicalExpression);
Assert.assertEquals(
LogicalExpression.Operator.OR, ((LogicalExpression) orExpression).getOperator());
Assert.assertEquals(40, ((LogicalExpression) orExpression).getTerms().size());
Assert.assertTrue(TagFilterEvaluator.evaluate(orExpression, identifier -> "r39"));
Assert.assertFalse(TagFilterEvaluator.evaluate(orExpression, identifier -> "r40"));

final String andChain =
IntStream.range(0, 40)
.mapToObj(i -> "region <> \"bad" + i + "\"")
.collect(Collectors.joining(" AND "));
final Expression andExpression = PARSER.parseAndValidate(andChain);
Assert.assertTrue(andExpression instanceof LogicalExpression);
Assert.assertEquals(
LogicalExpression.Operator.AND, ((LogicalExpression) andExpression).getOperator());
Assert.assertEquals(40, ((LogicalExpression) andExpression).getTerms().size());
Assert.assertTrue(TagFilterEvaluator.evaluate(andExpression, identifier -> "good"));
Assert.assertFalse(TagFilterEvaluator.evaluate(andExpression, identifier -> "bad39"));

// Precedence between OR and AND must still be preserved when flattening.
Assert.assertFalse(
evaluate(
"(region = \"north\" OR region = \"south\") AND device = \"d2\"",
Map.of("region", "north", "device", "d1")));

// Genuine nesting deeper than MAX_AST_DEPTH (32) must still be rejected.
final StringBuilder deeplyNested = new StringBuilder();
for (int i = 0; i < 33; i++) {
deeplyNested.append("NOT (");
}
deeplyNested.append("region = \"north\"");
for (int i = 0; i < 33; i++) {
deeplyNested.append(")");
}
assertRejected(deeplyNested.toString(), "AST depth exceeds maximum");
}

@Test
public void testRejectInvalidExpressions() {
assertRejected("", "tag-filter should not be empty");
Expand Down
Loading