Skip to content

Commit e64d5e8

Browse files
samdgupisamdgupi
authored
Merge pull request #1392 from cloudsufi/bqmt_e2e-ITN
E2E test coverage for BQMT
2 parents 66aafa1 + bbb81d8 commit e64d5e8

11 files changed

Lines changed: 597 additions & 1 deletion

File tree

‎src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
# Copyright © 2024 Cask Data, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4+
# use this file except in compliance with the License. You may obtain a copy of
5+
# the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12+
# License for the specific language governing permissions and limitations under
13+
# the License.
14+
15+
@BigQueryMultiTable_Sink
16+
Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
17+
18+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_DELETE_TABLES_TEST
19+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
20+
Given Open Datafusion Project to configure pipeline
21+
When Expand Plugin group in the LHS plugins list: "Source"
22+
When Select plugin: "BigQuery" from the plugins list as: "Source"
23+
Then Navigate to the properties page of plugin: "BigQuery"
24+
Then Replace input plugin property: "project" with value: "projectId"
25+
Then Replace input plugin property: "dataset" with value: "dataset"
26+
Then Replace input plugin property: "table" with value: "bqSourceTable"
27+
Then Click on the Get Schema button
28+
Then Validate "BigQuery" plugin properties
29+
Then Close the Plugin Properties page
30+
When Expand Plugin group in the LHS plugins list: "Source"
31+
When Select plugin: "BigQuery" from the plugins list as: "Source"
32+
Then Navigate to the properties page of plugin: "BigQuery2"
33+
Then Replace input plugin property: "project" with value: "projectId"
34+
Then Replace input plugin property: "dataset" with value: "dataset"
35+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
36+
Then Click on the Get Schema button
37+
Then Validate "BigQuery2" plugin properties
38+
Then Close the Plugin Properties page
39+
When Expand Plugin group in the LHS plugins list: "Sink"
40+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
41+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
42+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
43+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
44+
And Enter input plugin property: "referenceName" with value: "Reference"
45+
And Replace input plugin property: "project" with value: "projectId"
46+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
47+
And Enter input plugin property: "dataset" with value: "dataset"
48+
Then Override Service account details if set in environment variables
49+
Then Click plugin property: "flexibleSchema"
50+
Then Validate "BigQuery Multi Table" plugin properties
51+
And Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Deploy the pipeline
54+
Then Run the Pipeline in Runtime
55+
Then Wait till pipeline is in running state
56+
Then Open and capture logs
57+
Then Verify the pipeline status is "Succeeded"
58+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
59+
60+
@BQ_SINGLE_SOURCE_BQMT_TEST @BQ_SINK_BQMT_TEST
61+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in one table
62+
Given Open Datafusion Project to configure pipeline
63+
When Expand Plugin group in the LHS plugins list: "Source"
64+
When Select plugin: "BigQuery" from the plugins list as: "Source"
65+
Then Navigate to the properties page of plugin: "BigQuery"
66+
Then Replace input plugin property: "project" with value: "projectId"
67+
Then Replace input plugin property: "dataset" with value: "dataset"
68+
Then Replace input plugin property: "table" with value: "bqSourceTable"
69+
Then Click on the Get Schema button
70+
Then Validate "BigQuery" plugin properties
71+
Then Close the Plugin Properties page
72+
When Expand Plugin group in the LHS plugins list: "Sink"
73+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
74+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
75+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
76+
And Enter input plugin property: "referenceName" with value: "Reference"
77+
And Replace input plugin property: "project" with value: "projectId"
78+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
79+
And Enter input plugin property: "dataset" with value: "dataset"
80+
Then Override Service account details if set in environment variables
81+
Then Click plugin property: "flexibleSchema"
82+
Then Validate "BigQuery Multi Table" plugin properties
83+
And Close the Plugin Properties page
84+
Then Save the pipeline
85+
Then Deploy the pipeline
86+
Then Run the Pipeline in Runtime
87+
Then Wait till pipeline is in running state
88+
Then Open and capture logs
89+
Then Verify the pipeline status is "Succeeded"
90+
Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
91+
92+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
93+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
94+
Given Open Datafusion Project to configure pipeline
95+
When Expand Plugin group in the LHS plugins list: "Source"
96+
When Select plugin: "BigQuery" from the plugins list as: "Source"
97+
Then Navigate to the properties page of plugin: "BigQuery"
98+
Then Replace input plugin property: "project" with value: "projectId"
99+
Then Replace input plugin property: "dataset" with value: "dataset"
100+
Then Replace input plugin property: "table" with value: "bqSourceTable"
101+
Then Click on the Get Schema button
102+
Then Validate "BigQuery" plugin properties
103+
Then Close the Plugin Properties page
104+
When Expand Plugin group in the LHS plugins list: "Source"
105+
When Select plugin: "BigQuery" from the plugins list as: "Source"
106+
Then Navigate to the properties page of plugin: "BigQuery2"
107+
Then Replace input plugin property: "project" with value: "projectId"
108+
Then Replace input plugin property: "dataset" with value: "dataset"
109+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
110+
Then Click on the Get Schema button
111+
Then Validate "BigQuery2" plugin properties
112+
Then Close the Plugin Properties page
113+
When Expand Plugin group in the LHS plugins list: "Sink"
114+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
115+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
116+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
117+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
118+
And Enter input plugin property: "referenceName" with value: "Reference"
119+
And Replace input plugin property: "project" with value: "projectId"
120+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
121+
And Enter input plugin property: "dataset" with value: "dataset"
122+
Then Override Service account details if set in environment variables
123+
Then Click plugin property: "flexibleSchema"
124+
Then Validate "BigQuery Multi Table" plugin properties
125+
And Close the Plugin Properties page
126+
Then Save the pipeline
127+
Then Deploy the pipeline
128+
Then Run the Pipeline in Runtime
129+
Then Wait till pipeline is in running state
130+
Then Open and capture logs
131+
Then Verify the pipeline status is "Succeeded"
132+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
133+
134+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
135+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
136+
Given Open Datafusion Project to configure pipeline
137+
When Expand Plugin group in the LHS plugins list: "Source"
138+
When Select plugin: "BigQuery" from the plugins list as: "Source"
139+
Then Navigate to the properties page of plugin: "BigQuery"
140+
Then Replace input plugin property: "project" with value: "projectId"
141+
Then Replace input plugin property: "dataset" with value: "dataset"
142+
Then Replace input plugin property: "table" with value: "bqSourceTable"
143+
Then Click on the Get Schema button
144+
Then Validate "BigQuery" plugin properties
145+
Then Close the Plugin Properties page
146+
When Expand Plugin group in the LHS plugins list: "Source"
147+
When Select plugin: "BigQuery" from the plugins list as: "Source"
148+
Then Navigate to the properties page of plugin: "BigQuery2"
149+
Then Replace input plugin property: "project" with value: "projectId"
150+
Then Replace input plugin property: "dataset" with value: "dataset"
151+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
152+
Then Click on the Get Schema button
153+
Then Validate "BigQuery2" plugin properties
154+
Then Close the Plugin Properties page
155+
When Expand Plugin group in the LHS plugins list: "Sink"
156+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
157+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
158+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
159+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
160+
And Enter input plugin property: "referenceName" with value: "Reference"
161+
And Replace input plugin property: "project" with value: "projectId"
162+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
163+
And Enter input plugin property: "dataset" with value: "dataset"
164+
Then Override Service account details if set in environment variables
165+
Then Click plugin property: "flexibleSchema"
166+
Then Toggle BigQuery sink property truncateTable to true
167+
Then Validate "BigQuery Multi Table" plugin properties
168+
And Close the Plugin Properties page
169+
Then Save the pipeline
170+
Then Deploy the pipeline
171+
Then Run the Pipeline in Runtime
172+
Then Wait till pipeline is in running state
173+
Then Open and capture logs
174+
Then Verify the pipeline status is "Succeeded"
175+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
176+
177+
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
178+
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
179+
Given Open Datafusion Project to configure pipeline
180+
When Expand Plugin group in the LHS plugins list: "Source"
181+
When Select plugin: "BigQuery" from the plugins list as: "Source"
182+
Then Navigate to the properties page of plugin: "BigQuery"
183+
Then Replace input plugin property: "project" with value: "projectId"
184+
Then Replace input plugin property: "dataset" with value: "dataset"
185+
Then Replace input plugin property: "table" with value: "bqSourceTable"
186+
Then Click on the Get Schema button
187+
Then Validate "BigQuery" plugin properties
188+
Then Close the Plugin Properties page
189+
When Expand Plugin group in the LHS plugins list: "Source"
190+
When Select plugin: "BigQuery" from the plugins list as: "Source"
191+
Then Navigate to the properties page of plugin: "BigQuery2"
192+
Then Replace input plugin property: "project" with value: "projectId"
193+
Then Replace input plugin property: "dataset" with value: "dataset"
194+
Then Replace input plugin property: "table" with value: "bqSourceTable2"
195+
Then Click on the Get Schema button
196+
Then Validate "BigQuery2" plugin properties
197+
Then Close the Plugin Properties page
198+
When Expand Plugin group in the LHS plugins list: "Sink"
199+
When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
200+
Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
201+
Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
202+
Then Navigate to the properties page of plugin: "BigQuery Multi Table"
203+
And Enter input plugin property: "referenceName" with value: "Reference"
204+
And Replace input plugin property: "project" with value: "projectId"
205+
And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
206+
And Enter input plugin property: "dataset" with value: "dataset"
207+
Then Override Service account details if set in environment variables
208+
Then Click plugin property: "flexibleSchema"
209+
Then Select radio button plugin property: "updateSchema" with value: "true"
210+
Then Validate "BigQuery Multi Table" plugin properties
211+
And Close the Plugin Properties page
212+
Then Save the pipeline
213+
Then Deploy the pipeline
214+
Then Run the Pipeline in Runtime
215+
Then Wait till pipeline is in running state
216+
Then Open and capture logs
217+
Then Verify the pipeline status is "Succeeded"
218+
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.bigquery.runners.multitablerunner;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute BigQuery Multitable Sink plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
29+
"stepsdesign", "io.cdap.plugin.common.stepsdesign"},
30+
tags = {"@BigQueryMultiTable_Sink"},
31+
monochrome = true,
32+
plugin = {"pretty", "html:target/cucumber-html-report/bigquerymultitable-sink",
33+
"json:target/cucumber-reports/cucumber-bigquerymultitable-sink.json",
34+
"junit:target/cucumber-reports/cucumber-bigquerymultitable-sink.xml"}
35+
)
36+
public class TestRunner {
37+
}

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunnerRequired.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.bigquery.runners.multitablerunner;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute only required BigQuery Multitable Sink plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
29+
"stepsdesign", "io.cdap.plugin.common.stepsdesign"},
30+
tags = {"@BigQueryMultiTable_Sink_Required"},
31+
monochrome = true,
32+
plugin = {"pretty", "html:target/cucumber-html-report/bigquerymultitable-sink-required",
33+
"json:target/cucumber-reports/cucumber-bigquerymultitable-sink-required.json",
34+
"junit:target/cucumber-reports/cucumber-bigquerymultitable-sink-required.xml"}
35+
)
36+
public class TestRunnerRequired {
37+
}

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/package-info.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/**
2+
* Package contains the runner for the multitable plugin features.
3+
*/
4+
package io.cdap.plugin.bigquery.runners.multitablerunner;

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright © 2024 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.bigquery.stepsdesign;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cucumber.java.en.Then;
21+
import org.junit.Assert;
22+
23+
import java.io.IOException;
24+
import java.util.Arrays;
25+
import java.util.Collections;
26+
import java.util.List;
27+
28+
/**
29+
* BigQuery MultiTable related common stepDesigns.
30+
*/
31+
public class BigQueryMultiTable {
32+
@Then("Validate data transferred from BigQuery To BigQueryMultiTable is equal")
33+
public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
34+
throws IOException, InterruptedException {
35+
List<String> sourceTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqSourceTable"),
36+
PluginPropertyUtils.pluginProp("bqSourceTable2"));
37+
38+
List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqTargetTable"),
39+
PluginPropertyUtils.pluginProp("bqTargetTable2"));
40+
boolean recordsMatched = BigQueryMultiTableValidation.validateBQToBigQueryMultiTable(sourceTables, targetTables);
41+
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
42+
"of the records in the source table", recordsMatched);
43+
}
44+
45+
@Then("Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal")
46+
public void validateDataTransferredFromBigQueryToBigQueryMultiTableInOneTableIsEqual()
47+
throws IOException, InterruptedException {
48+
boolean recordsMatched = BigQueryMultiTableValidation.
49+
validateBQToBigQueryMultiTable(Collections.singletonList(PluginPropertyUtils.pluginProp("bqSourceTable")),
50+
Collections.singletonList(PluginPropertyUtils.pluginProp("bqTargetTable")));
51+
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
52+
"of the records in the source table", recordsMatched);
53+
}
54+
}

0 commit comments

Comments
 (0)