[DSIP-79][Task] Add datavines task for data quality

docs/docs/en/guide/task/datavines.md

@@ -0,0 +1,30 @@
+# Datavines
+
+## Overview
+
+Use `Datavines Task` to create a datavines-type task and support data quality job in Datavines. When the worker executes `Datavines Task`,
+it will call `Datavines API` to trigger datavines job. Click [here](https://datavane.github.io/datavines-website/) for details about `Datavines`.
+
+## Create Task
+
+- Click Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.
+- Drag <img src="../../../../img/tasks/icons/datavines.png" width="15"/> from the toolbar to the canvas.
+
+## Task Parameter
+
+- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md) `Default Task Parameters` section for default parameters.
+
+|   **Parameter**   |                                            **Description**                                            |
+|-------------------|-------------------------------------------------------------------------------------------------------|
+| Datavines Address | The URL for the Datavines service, e.g., http://localhost:5600.                                       |
+| Datavines Job ID  | The unique job id for a datavines job.                                                                |
+| Datavines token   | The Datawines service access token can be obtained through token management on the Datavines service. |
+| Block on Failure  | When turned on, if the data quality check result is failed, the task result will be set as failed.    |
+
+## Task Example
+
+This example illustrates how to create a datavines task node.
+
+![demo-datavines](../../../../img/tasks/demo/datavines_task.png)
+
+![demo-get-datavines-job-id](../../../../img/tasks/demo/datavines_job_id.png)

docs/docs/zh/guide/task/datavines.md

@@ -0,0 +1,31 @@
+# Datavines
+
+## 综述
+
+`Datavines`任务类型，用于创建并执行 `Datavines` 类型任务来执行 Datavines 中的数据质量检查作业。Worker 执行该任务的时候，会通过 `Datavines API` 触发 `Datavines 的作业`。
+点击 [这里](https://datavane.github.io/datavines-website/) 获取更多关于 `Datavines` 的信息。
+
+## 创建任务
+
+- 点击项目管理-项目名称-工作流定义，点击"创建工作流"按钮，进入DAG编辑页面。
+- 工具栏中拖动 <img src="../../../../img/tasks/icons/datavines.png" width="15"/> 到画板中，即可完成创建。
+
+## 任务参数
+
+- 默认参数说明请参考[DolphinScheduler任务参数附录](appendix.md)`默认任务参数`一栏。
+
+|    **任务参数**     |                        **描述**                        |
+|-----------------|------------------------------------------------------|
+| Datavines 地址    | Datavines 服务的 url，例如：`http://localhost:5600`。        |
+| Datavines 作业 ID | Datavines 作业对应的唯一ID。                                 |
+| Datavines token | Datavines 服务访问 token, 可在 Datavines 服务上的 token 管理中取得。 |
+| 检查失败时阻塞         | 开启时，数据质量检查结果为失败时会将任务结果置为失败。                          |
+
+## 例子
+
+这个示例展示了如何创建 Datavines 任务节点：
+
+![demo-datavines](../../../../img/tasks/demo/datavines_task.png)
+
+![demo-get-datavines-job-id](../../../../img/tasks/demo/datavines_job_id.png)
+

dolphinscheduler-api/src/main/resources/task-type-config.yaml

@@ -47,12 +47,15 @@ task:
     - 'SEATUNNEL'
     - 'DATAX'
     - 'SQOOP'
+  dataQuality:
+    - 'DATAVINES'
   machineLearning:
     - 'JUPYTER'
     - 'MLFLOW'
     - 'OPENMLDB'
     - 'DVC'
     - 'SAGEMAKER'
+    - 'PYTORCH'
     - 'KUBEFLOW'
   other:
     - 'ZEPPELIN'

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/tasks/DatavinesTaskE2ETest.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.e2e.cases.tasks;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest;
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.DatavinesTaskForm;
+import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
+import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
+import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junitpioneer.jupiter.DisableIfTestFails;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+@DolphinScheduler(composeFiles = "docker/datavines-task/docker-compose.yaml")
+@DisableIfTestFails
+public class DatavinesTaskE2ETest extends BaseWorkflowE2ETest {
+
+    private static final String mockServerAddress = "http://mockServer:1080";
+
+    private static final String jobId = "1";
+
+    private static final String token = "test-token";
+
+    @BeforeAll
+    public static void setup() {
+        browser = WebDriverHolder.getWebDriver();
+
+        TenantPage tenantPage = new LoginPage(browser)
+                .login(adminUser)
+                .goToNav(SecurityPage.class)
+                .goToTab(TenantPage.class);
+
+        if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
+            tenantPage
+                    .create(adminUser.getTenant())
+                    .goToNav(SecurityPage.class)
+                    .goToTab(UserPage.class)
+                    .update(adminUser);
+        }
+
+        tenantPage
+                .goToNav(ProjectPage.class)
+                .createProjectUntilSuccess(projectName);
+    }
+
+    @Test
+    @Order(10)
+    void testRunDatavinesTasks_SuccessCase() {
+        WorkflowDefinitionTab workflowDefinitionPage =
+                new ProjectPage(browser)
+                        .goToNav(ProjectPage.class)
+                        .goTo(projectName)
+                        .goToTab(WorkflowDefinitionTab.class);
+
+        String workflowName = "DatavinesSuccessCase";
+        String taskName = "DatavinesSuccessTask";
+        workflowDefinitionPage
+                .createWorkflow()
+                .<DatavinesTaskForm>addTask(WorkflowForm.TaskType.DATAVINES)
+                .address(mockServerAddress)
+                .jobId(jobId)
+                .token(token)
+                .name(taskName)
+                .submit()
+
+                .submit()
+                .name(workflowName)
+                .submit();
+
+        untilWorkflowDefinitionExist(workflowName);
+
+        workflowDefinitionPage.publish(workflowName);
+
+        runWorkflow(workflowName);
+        untilWorkflowInstanceExist(workflowName);
+        WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
+        assertThat(workflowInstance.executionTime()).isEqualTo(1);
+
+        TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
+        assertThat(taskInstance.retryTimes()).isEqualTo(0);
+    }
+
+    @Test
+    @Order(20)
+    void testRunDatavinesTasks_FailureBlockDisabledCase() {
+        WorkflowDefinitionTab workflowDefinitionPage =
+                new ProjectPage(browser)
+                        .goToNav(ProjectPage.class)
+                        .goTo(projectName)
+                        .goToTab(WorkflowDefinitionTab.class);
+
+        String workflowName = "DatavinesFailureBlockDisabledCase";
+        String taskName = "DatavinesFailureBlockDisabledTask";
+        workflowDefinitionPage
+                .createWorkflow()
+                .<DatavinesTaskForm>addTask(WorkflowForm.TaskType.DATAVINES)
+                .address(mockServerAddress)
+                .jobId(jobId)
+                .token(token)
+                .name(taskName)
+                .submit()
+
+                .submit()
+                .name(workflowName)
+                .submit();
+
+        untilWorkflowDefinitionExist(workflowName);
+
+        workflowDefinitionPage.publish(workflowName);
+
+        runWorkflow(workflowName);
+        untilWorkflowInstanceExist(workflowName);
+        WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
+        assertThat(workflowInstance.executionTime()).isEqualTo(1);
+
+        TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
+        assertThat(taskInstance.retryTimes()).isEqualTo(0);
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java

@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.e2e.pages.project.workflow;
 
 import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.DatavinesTaskForm;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.HttpTaskForm;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.JavaTaskForm;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm;
@@ -90,6 +91,8 @@ public <T> T addTask(TaskType type) {
                 return (T) new JavaTaskForm(this);
             case PYTHON:
                 return (T) new PythonTaskForm(this);
+            case DATAVINES:
+                return (T) new DatavinesTaskForm(this);
         }
         throw new UnsupportedOperationException("Unknown task type");
     }
@@ -129,6 +132,7 @@ public enum TaskType {
         SWITCH,
         HTTP,
         JAVA,
-        PYTHON
+        PYTHON,
+        DATAVINES
     }
 }

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/DatavinesTaskForm.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.e2e.pages.project.workflow.task;
+
+import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.FindBy;
+import org.openqa.selenium.support.FindBys;
+import org.openqa.selenium.support.PageFactory;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+
+public final class DatavinesTaskForm extends TaskNodeForm {
+
+    private final WebDriver driver;
+
+    @FindBys({
+            @FindBy(className = "input-datavines-address"),
+            @FindBy(tagName = "input")
+    })
+    private WebElement addressInput;
+
+    @FindBys({
+            @FindBy(className = "input-datavines-job-id"),
+            @FindBy(tagName = "input")
+    })
+    private WebElement jobIdInput;
+
+    @FindBys({
+            @FindBy(className = "input-datavines-token"),
+            @FindBy(tagName = "input")
+    })
+    private WebElement tokenInput;
+
+    public DatavinesTaskForm(WorkflowForm parent) {
+        super(parent);
+        this.driver = parent.driver();
+        PageFactory.initElements(driver, this);
+    }
+
+    public DatavinesTaskForm address(String address) {
+        WebDriverWaitFactory.createWebDriverWait(driver)
+                .until(ExpectedConditions.elementToBeClickable(addressInput));
+        addressInput.sendKeys(address);
+        return this;
+    }
+
+    public DatavinesTaskForm jobId(String jobId) {
+        WebDriverWaitFactory.createWebDriverWait(driver)
+                .until(ExpectedConditions.elementToBeClickable(jobIdInput));
+        jobIdInput.sendKeys(jobId);
+        return this;
+    }
+
+    public DatavinesTaskForm token(String token) {
+        WebDriverWaitFactory.createWebDriverWait(driver)
+                .until(ExpectedConditions.elementToBeClickable(tokenInput));
+        tokenInput.sendKeys(token);
+        return this;
+    }
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datavines-task/docker-compose.yaml

@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+
+services:
+  dolphinscheduler:
+    image: apache/dolphinscheduler-standalone-server:ci
+    environment:
+      MASTER_MAX_CPU_LOAD_AVG: 400
+      MASTER_RESERVED_MEMORY: 0.01
+      WORKER_TENANT_AUTO_CREATE: 'true'
+    ports:
+      - "12345:12345"
+    networks:
+      - e2e
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
+      interval: 5s
+      timeout: 5s
+      retries: 120
+  mockServer:
+    image: mockserver/mockserver:5.14.0
+    environment:
+      MOCKSERVER_INITIALIZATION_JSON_PATH: /config/mockserver-config.json
+    ports:
+      - "1080:1080"
+    networks:
+      - e2e
+    volumes:
+      - type: bind
+        source: ./
+        target: /config
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:1080/" ]
+      interval: 5s
+      timeout: 5s
+      retries: 120
+
+networks:
+  e2e: