Skip to content

Commit e191094

Browse files
committed
fundemental poc implementation for rex command (basic func only)
Signed-off-by: Jialiang Liang <jiallian@amazon.com>
1 parent a8da040 commit e191094

13 files changed

Lines changed: 463 additions & 1 deletion

File tree

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
import org.opensearch.sql.ast.tree.Paginate;
7676
import org.opensearch.sql.ast.tree.Parse;
7777
import org.opensearch.sql.ast.tree.Patterns;
78+
import org.opensearch.sql.ast.tree.Rex;
7879
import org.opensearch.sql.ast.tree.Project;
7980
import org.opensearch.sql.ast.tree.RareTopN;
8081
import org.opensearch.sql.ast.tree.Regex;
@@ -107,6 +108,7 @@
107108
import org.opensearch.sql.expression.function.FunctionName;
108109
import org.opensearch.sql.expression.function.TableFunctionImplementation;
109110
import org.opensearch.sql.expression.parse.ParseExpression;
111+
import org.opensearch.sql.expression.parse.RexExpression;
110112
import org.opensearch.sql.planner.logical.LogicalAD;
111113
import org.opensearch.sql.planner.logical.LogicalAggregation;
112114
import org.opensearch.sql.planner.logical.LogicalCloseCursor;
@@ -461,6 +463,14 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) {
461463
return child;
462464
}
463465

466+
/** Build {@link RexExpression} to context and skip to child nodes. */
467+
@Override
468+
public LogicalPlan visitRex(Rex node, AnalysisContext context) {
469+
LogicalPlan child = node.getChild().get(0).accept(this, context);
470+
analyzeRexNode(node, context);
471+
return child;
472+
}
473+
464474
// TODO: We may need to align output structure with Calcite's output structure
465475
@Override
466476
public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
@@ -788,6 +798,27 @@ private void analyzeParseNode(Parse node, AnalysisContext context) {
788798
});
789799
}
790800

801+
private void analyzeRexNode(Rex node, AnalysisContext context) {
802+
Expression sourceField = expressionAnalyzer.analyze(node.getSourceField(), context);
803+
String pattern = (String) node.getPattern().getValue();
804+
Expression patternExpression = DSL.literal(pattern);
805+
806+
// Get max_match parameter if specified, default to 1
807+
final int maxMatch = node.getArguments().containsKey("max_match") ?
808+
Integer.parseInt(node.getArguments().get("max_match").getValue().toString()) : 1;
809+
final String offsetField = node.getArguments().containsKey("offset_field") ?
810+
node.getArguments().get("offset_field").getValue().toString() : null;
811+
812+
TypeEnvironment curEnv = context.peek();
813+
RexExpression.getNamedGroupCandidates(pattern)
814+
.forEach(
815+
group -> {
816+
RexExpression expr = new RexExpression(sourceField, patternExpression, maxMatch, offsetField);
817+
curEnv.define(new Symbol(Namespace.FIELD_NAME, group), expr.type());
818+
context.getNamedParseExpressions().add(new NamedExpression(group, expr));
819+
});
820+
}
821+
791822
private LogicalAggregation analyzeAggregation(
792823
Aggregation node, LogicalPlan child, AnalysisContext context) {
793824
ImmutableList.Builder<NamedAggregator> aggregatorBuilder = new ImmutableList.Builder<>();

‎core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.opensearch.sql.ast.tree.Project;
6868
import org.opensearch.sql.ast.tree.RareTopN;
6969
import org.opensearch.sql.ast.tree.Regex;
70+
import org.opensearch.sql.ast.tree.Rex;
7071
import org.opensearch.sql.ast.tree.Relation;
7172
import org.opensearch.sql.ast.tree.RelationSubquery;
7273
import org.opensearch.sql.ast.tree.Rename;
@@ -254,6 +255,10 @@ public T visitRegex(Regex node, C context) {
254255
return visitChildren(node, context);
255256
}
256257

258+
public T visitRex(Rex node, C context) {
259+
return visitChildren(node, context);
260+
}
261+
257262
public T visitLambdaFunction(LambdaFunction node, C context) {
258263
return visitChildren(node, context);
259264
}
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ast.tree;
7+
8+
import com.google.common.collect.ImmutableList;
9+
import java.util.List;
10+
import java.util.Map;
11+
import lombok.AllArgsConstructor;
12+
import lombok.EqualsAndHashCode;
13+
import lombok.Getter;
14+
import lombok.RequiredArgsConstructor;
15+
import lombok.Setter;
16+
import lombok.ToString;
17+
import org.opensearch.sql.ast.AbstractNodeVisitor;
18+
import org.opensearch.sql.ast.expression.Literal;
19+
import org.opensearch.sql.ast.expression.UnresolvedExpression;
20+
21+
/** AST node represent Rex field extraction operation. */
22+
@Getter
23+
@Setter
24+
@ToString
25+
@EqualsAndHashCode(callSuper = false)
26+
@RequiredArgsConstructor
27+
@AllArgsConstructor
28+
public class Rex extends UnresolvedPlan {
29+
/** Source field to extract from. */
30+
private final UnresolvedExpression sourceField;
31+
32+
/** Regular expression pattern with named capture groups. */
33+
private final Literal pattern;
34+
35+
/** Optional arguments. */
36+
private final Map<String, Literal> arguments;
37+
38+
/** Child Plan. */
39+
private UnresolvedPlan child;
40+
41+
@Override
42+
public Rex attach(UnresolvedPlan child) {
43+
this.child = child;
44+
return this;
45+
}
46+
47+
@Override
48+
public List<UnresolvedPlan> getChild() {
49+
return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
50+
}
51+
52+
@Override
53+
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
54+
return nodeVisitor.visitRex(this, context);
55+
}
56+
}

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
import org.opensearch.sql.ast.tree.Project;
100100
import org.opensearch.sql.ast.tree.RareTopN;
101101
import org.opensearch.sql.ast.tree.Regex;
102+
import org.opensearch.sql.ast.tree.Rex;
102103
import org.opensearch.sql.ast.tree.Relation;
103104
import org.opensearch.sql.ast.tree.Rename;
104105
import org.opensearch.sql.ast.tree.Sort;
@@ -192,6 +193,13 @@ public RelNode visitRegex(Regex node, CalcitePlanContext context) {
192193
return context.relBuilder.peek();
193194
}
194195

196+
@Override
197+
public RelNode visitRex(Rex node, CalcitePlanContext context) {
198+
visitChildren(node, context);
199+
buildRexRelNode(node, context);
200+
return context.relBuilder.peek();
201+
}
202+
195203
private boolean containsSubqueryExpression(Node expr) {
196204
if (expr == null) {
197205
return false;
@@ -1437,6 +1445,46 @@ private void buildParseRelNode(Parse node, CalcitePlanContext context) {
14371445
projectPlusOverriding(newFields, groupCandidates, context);
14381446
}
14391447

1448+
private void buildRexRelNode(Rex node, CalcitePlanContext context) {
1449+
RexNode sourceField = rexVisitor.analyze(node.getSourceField(), context);
1450+
String patternValue = (String) node.getPattern().getValue();
1451+
1452+
// Get named capture groups from the pattern
1453+
List<String> groupCandidates =
1454+
org.opensearch.sql.expression.parse.RexExpression.getNamedGroupCandidates(patternValue);
1455+
1456+
if (groupCandidates.isEmpty()) {
1457+
throw new SemanticCheckException(
1458+
"Rex pattern must contain at least one named capture group: " + patternValue);
1459+
}
1460+
1461+
// Create arguments for rex UDF
1462+
RexNode[] rexNodeList = new RexNode[] {
1463+
sourceField,
1464+
context.rexBuilder.makeLiteral(
1465+
patternValue, context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), true)
1466+
};
1467+
1468+
// Add max_match parameter if specified
1469+
if (node.getArguments().containsKey("max_match")) {
1470+
int maxMatch = Integer.parseInt(node.getArguments().get("max_match").getValue().toString());
1471+
rexNodeList = ArrayUtils.add(rexNodeList,
1472+
context.rexBuilder.makeLiteral(maxMatch,
1473+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER), true));
1474+
}
1475+
1476+
List<RexNode> newFields = new ArrayList<>();
1477+
for (String groupCandidate : groupCandidates) {
1478+
RexNode innerRex = PPLFuncImpTable.INSTANCE.resolve(
1479+
context.rexBuilder,
1480+
BuiltinFunctionName.REX_EXTRACT,
1481+
ArrayUtils.add(rexNodeList, context.relBuilder.literal(groupCandidate)));
1482+
newFields.add(innerRex);
1483+
}
1484+
1485+
projectPlusOverriding(newFields, groupCandidates, context);
1486+
}
1487+
14401488
private void flattenParsedPattern(
14411489
String originalPatternResultAlias,
14421490
RexNode parsedNode,

‎core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.opensearch.sql.expression.function.FunctionImplementation;
1313
import org.opensearch.sql.expression.operator.predicate.RegexMatch;
1414
import org.opensearch.sql.expression.parse.ParseExpression;
15+
import org.opensearch.sql.expression.parse.RexExpression;
1516

1617
/**
1718
* Abstract visitor for expression tree nodes.
@@ -70,6 +71,10 @@ public T visitParse(ParseExpression node, C context) {
7071
return visitNode(node, context);
7172
}
7273

74+
public T visitRex(RexExpression node, C context) {
75+
return visitNode(node, context);
76+
}
77+
7378
public T visitFunction(FunctionExpression node, C context) {
7479
return visitChildren(node, context);
7580
}

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ public enum BuiltinFunctionName {
312312
INTERNAL_UNCOLLECT_PATTERNS(FunctionName.of("uncollect_patterns")),
313313
INTERNAL_REGEXP_EXTRACT(FunctionName.of("regexp_extract"), true),
314314
INTERNAL_GROK(FunctionName.of("grok"), true),
315-
INTERNAL_REGEXP_REPLACE_3(FunctionName.of("regexp_replace_3"), true);
315+
INTERNAL_REGEXP_REPLACE_3(FunctionName.of("regexp_replace_3"), true),
316+
REX_EXTRACT(FunctionName.of("rex_extract"), true);
316317

317318
private final FunctionName name;
318319
private boolean isInternal;

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
383383
new NumberToStringFunction().toUDF("NUMBER_TO_STRING");
384384
public static final SqlOperator REGEX_MATCH = SqlLibraryOperators.REGEXP_CONTAINS;
385385

386+
// Rex field extraction operator for Calcite engine
387+
public static final SqlOperator REX_EXTRACT =
388+
new org.opensearch.sql.expression.function.udf.RexExtractFunctionImpl().toUDF("REX_EXTRACT");
389+
386390
/**
387391
* Returns the PPL specific operator table, creating it if necessary.
388392
*

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@
161161
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REDUCE;
162162
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP;
163163
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEX_MATCH;
164+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_EXTRACT;
164165
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REPLACE;
165166
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REVERSE;
166167
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT;
@@ -785,6 +786,7 @@ void populate() {
785786
registerOperator(QUERY_STRING, PPLBuiltinOperators.QUERY_STRING);
786787
registerOperator(MULTI_MATCH, PPLBuiltinOperators.MULTI_MATCH);
787788
registerOperator(REGEX_MATCH, PPLBuiltinOperators.REGEX_MATCH);
789+
registerOperator(REX_EXTRACT, PPLBuiltinOperators.REX_EXTRACT);
788790

789791
// Register PPL Datetime UDF operator
790792
registerOperator(TIMESTAMP, PPLBuiltinOperators.TIMESTAMP);
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.util.List;
9+
import java.util.regex.Matcher;
10+
import java.util.regex.Pattern;
11+
import java.util.regex.PatternSyntaxException;
12+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
13+
import org.apache.calcite.adapter.enumerable.NullPolicy;
14+
import org.apache.calcite.adapter.enumerable.RexImpTable;
15+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
16+
import org.apache.calcite.linq4j.tree.Expression;
17+
import org.apache.calcite.linq4j.tree.Types;
18+
import org.apache.calcite.rex.RexCall;
19+
import org.apache.calcite.schema.impl.ScalarFunctionImpl;
20+
import org.apache.calcite.sql.type.ReturnTypes;
21+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
22+
import org.opensearch.sql.expression.function.ImplementorUDF;
23+
import org.opensearch.sql.expression.function.UDFOperandMetadata;
24+
25+
/**
26+
* REX_EXTRACT UDF implementation for Calcite engine. This function extracts specific named
27+
* capture groups from text using Java regex patterns.
28+
*/
29+
public class RexExtractFunctionImpl extends ImplementorUDF {
30+
31+
public RexExtractFunctionImpl() {
32+
super(new RexExtractImplementor(), NullPolicy.ANY);
33+
}
34+
35+
@Override
36+
public SqlReturnTypeInference getReturnTypeInference() {
37+
return ReturnTypes.VARCHAR_2000_NULLABLE;
38+
}
39+
40+
@Override
41+
public UDFOperandMetadata getOperandMetadata() {
42+
return null;
43+
}
44+
45+
public static class RexExtractImplementor implements NotNullImplementor {
46+
@Override
47+
public Expression implement(
48+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
49+
ScalarFunctionImpl function =
50+
(ScalarFunctionImpl)
51+
ScalarFunctionImpl.create(
52+
Types.lookupMethod(
53+
RexExtractFunctionImpl.class, "eval", String.class, String.class, String.class));
54+
return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL);
55+
}
56+
}
57+
58+
/**
59+
* Evaluation method for REX_EXTRACT function. This method is called by Calcite's generated code
60+
* during execution.
61+
*
62+
* @param field The field value to extract from
63+
* @param pattern The Java regex pattern with named capture groups
64+
* @param groupName The name of the capture group to extract
65+
* @return String value of the extracted group, or null if no match or group not found
66+
*/
67+
public static String eval(String field, String pattern, String groupName) {
68+
if (field == null || pattern == null || groupName == null) {
69+
return null;
70+
}
71+
72+
try {
73+
Pattern compiledPattern = Pattern.compile(pattern);
74+
Matcher matcher = compiledPattern.matcher(field);
75+
76+
if (matcher.find()) {
77+
try {
78+
return matcher.group(groupName);
79+
} catch (IllegalArgumentException e) {
80+
// Named group doesn't exist
81+
return null;
82+
}
83+
}
84+
return null; // No match found
85+
} catch (PatternSyntaxException e) {
86+
throw new IllegalArgumentException("Invalid regex pattern: " + e.getMessage());
87+
}
88+
}
89+
90+
/**
91+
* Overloaded eval method with max_match parameter.
92+
*
93+
* @param field The field value to extract from
94+
* @param pattern The Java regex pattern with named capture groups
95+
* @param maxMatch Maximum number of matches to process (currently ignored, defaults to first match)
96+
* @param groupName The name of the capture group to extract
97+
* @return String value of the extracted group, or null if no match or group not found
98+
*/
99+
public static String eval(String field, String pattern, Integer maxMatch, String groupName) {
100+
// For now, we only support the first match, but maxMatch parameter is ready for future extension
101+
return eval(field, pattern, groupName);
102+
}
103+
}

0 commit comments

Comments
 (0)