Skip to content

Commit d045ea5

Browse files
RyanL1997RyanL1997
committed
IT - initial setup
Signed-off-by: Jialiang Liang <jiallian@amazon.com>
1 parent 9dedfc9 commit d045ea5

3 files changed

Lines changed: 198 additions & 0 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
CalciteQueryAnalysisIT.class,
8484
CalciteRareCommandIT.class,
8585
CalciteRenameCommandIT.class,
86+
CalciteRenameCommandIT.class,
8687
CalciteResourceMonitorIT.class,
8788
CalciteSearchCommandIT.class,
8889
CalciteSettingsIT.class,

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.remote;
7+
8+
import org.opensearch.sql.ppl.RexCommandIT;
9+
10+
public class CalciteRexCommandIT extends RexCommandIT {
11+
@Override
12+
public void init() throws Exception {
13+
super.init();
14+
enableCalcite();
15+
}
16+
}

‎integ-test/src/test/java/org/opensearch/sql/ppl/RexCommandIT.java‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ppl;
7+
8+
import static org.opensearch.sql.legacy.TestsConstants.*;
9+
10+
import java.io.IOException;
11+
import org.json.JSONObject;
12+
import org.junit.jupiter.api.Test;
13+
14+
public class RexCommandIT extends PPLIntegTestCase {
15+
16+
@Override
17+
public void init() throws Exception {
18+
super.init();
19+
enableCalcite();
20+
loadIndex(Index.ACCOUNT);
21+
}
22+
23+
@Test
24+
public void testRexBasicFieldExtraction() throws IOException {
25+
JSONObject result =
26+
executeQuery(
27+
String.format(
28+
"source=%s | rex field=email \\\"(?<user>[^@]+)@(?<domain>.+)\\\" | fields email,"
29+
+ " user, domain",
30+
TEST_INDEX_ACCOUNT));
31+
32+
assertEquals(1000, result.getJSONArray("datarows").length());
33+
assertEquals("amberduke@pyrami.com", result.getJSONArray("datarows").getJSONArray(0).get(0));
34+
assertEquals("amberduke", result.getJSONArray("datarows").getJSONArray(0).get(1));
35+
assertEquals("pyrami.com", result.getJSONArray("datarows").getJSONArray(0).get(2));
36+
}
37+
38+
@Test
39+
public void testRexErrorNoNamedGroups() throws IOException {
40+
try {
41+
executeQuery(
42+
String.format(
43+
"source=%s | rex field=email \\\"([^@]+)@(.+)\\\" | fields email",
44+
TEST_INDEX_ACCOUNT));
45+
fail("Should have thrown an exception for pattern without named capture groups");
46+
} catch (Exception e) {
47+
assertTrue(
48+
e.getMessage().contains("Rex pattern must contain at least one named capture group"));
49+
}
50+
}
51+
52+
@Test
53+
public void testRexWithFiltering() throws IOException {
54+
JSONObject result =
55+
executeQuery(
56+
String.format(
57+
"source=%s | rex field=address"
58+
+ " \\\"(?<streetnum>\\\\\\\\d+)\\\\\\\\s+(?<streetname>.+)\\\" | fields"
59+
+ " address, streetnum, streetname",
60+
TEST_INDEX_ACCOUNT));
61+
62+
assertEquals(1000, result.getJSONArray("datarows").length());
63+
assertEquals("880 Holmes Lane", result.getJSONArray("datarows").getJSONArray(0).get(0));
64+
assertEquals("880", result.getJSONArray("datarows").getJSONArray(0).get(1));
65+
assertEquals("Holmes Lane", result.getJSONArray("datarows").getJSONArray(0).get(2));
66+
}
67+
68+
@Test
69+
public void testRexMultipleMatches() throws IOException {
70+
JSONObject result =
71+
executeQuery(
72+
String.format(
73+
"source=%s | rex field=address \\\"(?<words>[A-Za-z]+)\\\" max_match=3 | fields"
74+
+ " address, words",
75+
TEST_INDEX_ACCOUNT));
76+
77+
assertEquals(1000, result.getJSONArray("datarows").length());
78+
String wordsArray = result.getJSONArray("datarows").getJSONArray(0).get(1).toString();
79+
assertTrue(wordsArray.contains("Holmes") && wordsArray.contains("Lane"));
80+
assertTrue(wordsArray.startsWith("[") && wordsArray.endsWith("]"));
81+
}
82+
83+
@Test
84+
public void testRexSedMode() throws IOException {
85+
JSONObject result =
86+
executeQuery(
87+
String.format(
88+
"source=%s | rex field=email mode=sed \\\"s/@.*/@company.com/\\\" | fields email",
89+
TEST_INDEX_ACCOUNT));
90+
91+
assertEquals(1000, result.getJSONArray("datarows").length());
92+
assertEquals("amberduke@company.com", result.getJSONArray("datarows").getJSONArray(0).get(0));
93+
assertEquals("hattiebond@company.com", result.getJSONArray("datarows").getJSONArray(1).get(0));
94+
}
95+
96+
@Test
97+
public void testRexChainedCommands() throws IOException {
98+
JSONObject result =
99+
executeQuery(
100+
String.format(
101+
"source=%s | rex field=firstname \\\"(?<firstinitial>^.)\\\" | rex field=lastname"
102+
+ " \\\"(?<lastinitial>^.)\\\" | fields firstname, lastname, firstinitial,"
103+
+ " lastinitial",
104+
TEST_INDEX_ACCOUNT));
105+
106+
assertEquals(1000, result.getJSONArray("datarows").length());
107+
assertEquals("Amber", result.getJSONArray("datarows").getJSONArray(0).get(0));
108+
assertEquals("Duke", result.getJSONArray("datarows").getJSONArray(0).get(1));
109+
assertEquals("A", result.getJSONArray("datarows").getJSONArray(0).get(2));
110+
assertEquals("D", result.getJSONArray("datarows").getJSONArray(0).get(3));
111+
}
112+
113+
@Test
114+
public void testRexWithOffsetField() throws IOException {
115+
JSONObject result =
116+
executeQuery(
117+
String.format(
118+
"source=%s | rex field=email \\\"(?<user>[^@]+)@(?<domain>[^.]+)\\\""
119+
+ " offset_field=matchpos | fields email, user, domain, matchpos",
120+
TEST_INDEX_ACCOUNT));
121+
122+
assertEquals(1000, result.getJSONArray("datarows").length());
123+
String email = result.getJSONArray("datarows").getJSONArray(0).get(0).toString();
124+
String user = result.getJSONArray("datarows").getJSONArray(0).get(1).toString();
125+
String matchPos = result.getJSONArray("datarows").getJSONArray(0).get(3).toString();
126+
assertEquals(email.substring(0, email.indexOf("@")), user);
127+
assertTrue(matchPos.contains("user=") && matchPos.contains("domain="));
128+
assertTrue(matchPos.contains("-"));
129+
}
130+
131+
@Test
132+
public void testRexComplexPattern() throws IOException {
133+
JSONObject result =
134+
executeQuery(
135+
String.format(
136+
"source=%s | rex field=email"
137+
+ " \\\"(?<user>[a-zA-Z0-9._%%+-]+)@(?<domain>[a-zA-Z0-9.-]+)\\\\\\\\.(?<tld>[a-zA-Z]{2,})\\\""
138+
+ " | fields email, user, domain, tld",
139+
TEST_INDEX_ACCOUNT));
140+
141+
assertEquals(1000, result.getJSONArray("datarows").length());
142+
assertEquals("amberduke@pyrami.com", result.getJSONArray("datarows").getJSONArray(0).get(0));
143+
assertEquals("amberduke", result.getJSONArray("datarows").getJSONArray(0).get(1));
144+
assertEquals("pyrami", result.getJSONArray("datarows").getJSONArray(0).get(2));
145+
assertEquals("com", result.getJSONArray("datarows").getJSONArray(0).get(3));
146+
}
147+
148+
@Test
149+
public void testRexWithWhere() throws IOException {
150+
JSONObject result =
151+
executeQuery(
152+
String.format(
153+
"source=%s | where state=\\\"CA\\\" | rex field=email"
154+
+ " \\\"(?<user>[^@]+)@(?<domain>.+)\\\" | fields email, user, domain",
155+
TEST_INDEX_ACCOUNT));
156+
157+
assertTrue(result.getJSONArray("datarows").length() > 0);
158+
String email = result.getJSONArray("datarows").getJSONArray(0).get(0).toString();
159+
String user = result.getJSONArray("datarows").getJSONArray(0).get(1).toString();
160+
String domain = result.getJSONArray("datarows").getJSONArray(0).get(2).toString();
161+
assertTrue(email.startsWith(user));
162+
assertTrue(email.endsWith(domain));
163+
}
164+
165+
@Test
166+
public void testRexWithStatsCommand() throws IOException {
167+
JSONObject result =
168+
executeQuery(
169+
String.format(
170+
"source=%s | rex field=email \\\"[^@]+@(?<domain>[^.]+)\\\" | stats count() by"
171+
+ " domain",
172+
TEST_INDEX_ACCOUNT));
173+
174+
assertTrue(result.getJSONArray("datarows").length() > 0);
175+
int count = Integer.parseInt(result.getJSONArray("datarows").getJSONArray(0).get(0).toString());
176+
String domain = result.getJSONArray("datarows").getJSONArray(0).get(1).toString();
177+
assertTrue(count > 0);
178+
assertFalse(domain.contains("@"));
179+
assertTrue(domain.matches("[a-z]+"));
180+
}
181+
}

0 commit comments

Comments
 (0)