Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions deploy/kubernetes/dolphinscheduler/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -965,6 +965,9 @@ api:
# jackson:
# time-zone: UTC
# date-format: "yyyy-MM-dd HH:mm:ss"
# api:
# # Whether to test datasource connectivity before creating or updating a datasource.
# datasource-connection-enable: false
# -- Periodic probe of container liveness. Container will be restarted if the probe fails.
# More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
Expand Down
1 change: 1 addition & 0 deletions docs/docs/en/architecture/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ Location: `api-server/conf/application.yaml`
| casdoor.organization-name | | organization name in Casdoor |
| casdoor.application-name | | application name in Casdoor |
| casdoor.redirect-url | | doplhinscheduler login url |
| api.datasource-connection-enable | false | Whether to test datasource connectivity before creating or updating a datasource |
| api.traffic.control.global.switch | false | traffic control global switch |
| api.traffic.control.max-global-qps-rate | 300 | global max request number per second |
| api.traffic.control.tenant-switch | false | traffic control tenant switch |
Expand Down
1 change: 1 addition & 0 deletions docs/docs/zh/architecture/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
| casdoor.organization-name | | Casdoor中的组织名称 |
| casdoor.application-name | | Casdoor中的应用名称 |
| casdoor.redirect-url | | dolphinscheduler登录URL |
| api.datasource-connection-enable | false | 在创建或更新数据源时是否需要强制测试其连接性 |
| api.traffic.control.global.switch | false | 流量控制全局开关 |
| api.traffic.control.max-global-qps-rate | 300 | 全局最大请求数/秒 |
| api.traffic.control.tenant-switch | false | 流量控制租户开关 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,8 @@ alert:

api:
audit-enable: false
# Whether to test datasource connectivity before creating or updating a datasource.
datasource-connection-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ api:
base-url: http://localhost:12345/dolphinscheduler
ui-url: http://localhost:5173
audit-enable: false
# Whether to test datasource connectivity before creating or updating a datasource.
datasource-connection-enable: false
traffic-control:
global-switch: false
max-global-qps-rate: 300
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public class ApiConfig implements Validator {
private String baseUrl;
private String uiUrl;
private boolean auditEnable = false;
private boolean datasourceConnectionEnable = false;

private TrafficConfiguration trafficControl = new TrafficConfiguration();

Expand All @@ -70,6 +71,7 @@ private void printConfig() {
log.info("API config: baseUrl -> {} ", baseUrl);
log.info("API config: uiUrl -> {} ", uiUrl);
log.info("API config: auditEnable -> {} ", auditEnable);
log.info("API config: datasourceConnectionEnable -> {} ", datasourceConnectionEnable);
log.info("API config: trafficControl -> {} ", trafficControl);
log.info("API config: pythonGateway -> {} ", pythonGateway);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public interface DataSourceService {
* @param loginUser login user
* @param datasourceParam datasource configuration DTO
* @return created {@link DataSource} entity (sensitive fields masked)
* @throws ServiceException if permission denied, security check fails, or connection test fails
* @throws ServiceException if permission denied, security check fails, or configured connection test fails
*/
DataSource createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam);

Expand All @@ -46,7 +46,7 @@ public interface DataSourceService {
* @param loginUser login user
* @param dataSourceParam datasource params
* @return updated {@link DataSource} entity (sensitive fields masked)
* @throws ServiceException if permission denied, security check fails, or connection test fails
* @throws ServiceException if permission denied, security check fails, or configured connection test fails
*/
DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSourceParam);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_DELETE;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_UPDATE;

import org.apache.dolphinscheduler.api.configuration.ApiConfig;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
Expand Down Expand Up @@ -77,6 +78,9 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
@Autowired
private DataSourceUserDao datasourceUserDao;

@Autowired
private ApiConfig apiConfig;

private static final String TABLE = "TABLE";
private static final String VIEW = "VIEW";
private static final String[] TABLE_TYPES = new String[]{TABLE, VIEW};
Expand All @@ -99,6 +103,10 @@ public DataSource createDataSource(User loginUser, BaseDataSourceParamDTO dataso
}
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(datasourceParam);

if (apiConfig.isDatasourceConnectionEnable()) {
checkConnection(datasourceParam.getType(), connectionParam);
}

// build datasource
DataSource dataSource = new DataSource();
Date now = new Date();
Expand Down Expand Up @@ -151,6 +159,10 @@ public DataSource updateDataSource(User loginUser, BaseDataSourceParamDTO dataSo
connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText());
}

if (apiConfig.isDatasourceConnectionEnable()) {
checkConnection(dataSourceParam.getType(), connectionParam);
}

Date now = new Date();

dataSource.setName(dataSourceParam.getName().trim());
Expand Down Expand Up @@ -191,7 +203,7 @@ public BaseDataSourceParamDTO queryDataSource(int id, User loginUser) {
baseDataSourceParamDTO.setId(dataSource.getId());
baseDataSourceParamDTO.setName(dataSource.getName());
baseDataSourceParamDTO.setNote(dataSource.getNote());
baseDataSourceParamDTO.setPassword(getHiddenPassword());
baseDataSourceParamDTO.setPassword("");

return baseDataSourceParamDTO;
}
Expand Down
2 changes: 2 additions & 0 deletions dolphinscheduler-api/src/main/resources/application.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ api:
base-url: http://127.0.0.1:12345/dolphinscheduler
ui-url: http://127.0.0.1:5173
audit-enable: false
# Whether to test datasource connectivity before creating or updating a datasource.
datasource-connection-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ public void testIsAuditEnable() {
Assertions.assertTrue(apiConfig.isAuditEnable());
}

@Test
public void testIsDatasourceConnectionEnable() {
Assertions.assertFalse(apiConfig.isDatasourceConnectionEnable());
}

@Test
public void testGetTrafficControlConfig() {
ApiConfig.TrafficConfiguration trafficControl = apiConfig.getTrafficControl();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE;
import static org.mockito.Mockito.when;

import org.apache.dolphinscheduler.api.configuration.ApiConfig;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
Expand Down Expand Up @@ -96,6 +97,9 @@ public class DataSourceServiceTest {
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;

@Mock
private ApiConfig apiConfig;

@Mock
private IPage<DataSource> dataSourceList;

Expand Down Expand Up @@ -237,6 +241,129 @@ public void updateDataSourceTest() {
}
}

@Test
public void createDataSourceChecksConnectionWhenConfigured() {
User loginUser = getAdminUser();
PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO();
postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
postgreSqlDatasourceParam.setNote("test dataSource");
postgreSqlDatasourceParam.setHost("172.16.133.200");
postgreSqlDatasourceParam.setPort(5432);
postgreSqlDatasourceParam.setUserName("postgres");
postgreSqlDatasourceParam.setPassword("postgres");
postgreSqlDatasourceParam.setName("dataSource01");

when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true);
passResourcePermissionCheckService();
when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName().trim())).thenReturn(null);

try (
MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
.thenReturn(dataSourceProcessor);
when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
.thenReturn(connectionParam);
when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false);

assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE,
() -> dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam));
}
}

@Test
public void updateDataSourceChecksConnectionWhenConfigured() {
User loginUser = getAdminUser();
int dataSourceId = 12;
PostgreSQLDataSourceParamDTO postgreSqlDatasourceParam = new PostgreSQLDataSourceParamDTO();
postgreSqlDatasourceParam.setId(dataSourceId);
postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
postgreSqlDatasourceParam.setNote("test dataSource");
postgreSqlDatasourceParam.setHost("172.16.133.200");
postgreSqlDatasourceParam.setPort(5432);
postgreSqlDatasourceParam.setUserName("postgres");
postgreSqlDatasourceParam.setPassword("postgres");
postgreSqlDatasourceParam.setName("dataSource01-update");

DataSource dataSource = new DataSource();
dataSource.setId(dataSourceId);
dataSource.setName("dataSource01");
dataSource.setType(DbType.POSTGRESQL);

when(apiConfig.isDatasourceConnectionEnable()).thenReturn(true);
passResourcePermissionCheckService();
when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource);
when(dataSourceDao.queryDataSourceByName(postgreSqlDatasourceParam.getName())).thenReturn(null);

try (
MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
.thenReturn(dataSourceProcessor);
when(connectionParam.getPassword()).thenReturn("postgres");
when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
.thenReturn(connectionParam);
when(dataSourceProcessor.checkDataSourceConnectivity(Mockito.any())).thenReturn(false);

assertThrowsServiceException(Status.CONNECTION_TEST_FAILURE,
() -> dataSourceService.updateDataSource(loginUser, postgreSqlDatasourceParam));
}
}

@Test
public void dataSourceDoesNotCheckConnectionByDefault() {
User loginUser = getAdminUser();
int dataSourceId = 12;
PostgreSQLDataSourceParamDTO createDatasourceParam = new PostgreSQLDataSourceParamDTO();
createDatasourceParam.setDatabase("dolphinscheduler");
createDatasourceParam.setNote("test dataSource");
createDatasourceParam.setHost("172.16.133.200");
createDatasourceParam.setPort(5432);
createDatasourceParam.setUserName("postgres");
createDatasourceParam.setPassword("postgres");
createDatasourceParam.setName("dataSource01");

PostgreSQLDataSourceParamDTO updateDatasourceParam = new PostgreSQLDataSourceParamDTO();
updateDatasourceParam.setId(dataSourceId);
updateDatasourceParam.setDatabase("dolphinscheduler");
updateDatasourceParam.setNote("test dataSource");
updateDatasourceParam.setHost("172.16.133.200");
updateDatasourceParam.setPort(5432);
updateDatasourceParam.setUserName("postgres");
updateDatasourceParam.setPassword("postgres");
updateDatasourceParam.setName("dataSource01-update");

DataSource dataSource = new DataSource();
dataSource.setId(dataSourceId);
dataSource.setName("dataSource01");
dataSource.setType(DbType.POSTGRESQL);

when(apiConfig.isDatasourceConnectionEnable()).thenReturn(false);
passResourcePermissionCheckService();
when(dataSourceDao.queryById(dataSourceId)).thenReturn(dataSource);
when(dataSourceDao.queryDataSourceByName(Mockito.anyString())).thenReturn(null);

try (
MockedStatic<DataSourceUtils> mockedStaticDataSourceUtils =
Mockito.mockStatic(DataSourceUtils.class, Mockito.CALLS_REAL_METHODS)) {
DataSourceProcessor dataSourceProcessor = Mockito.mock(DataSourceProcessor.class);
ConnectionParam connectionParam = Mockito.mock(ConnectionParam.class);
mockedStaticDataSourceUtils.when(() -> DataSourceUtils.getDatasourceProcessor(Mockito.any()))
.thenReturn(dataSourceProcessor);
when(connectionParam.getPassword()).thenReturn("postgres");
when(dataSourceProcessor.createConnectionParams(Mockito.any(BaseDataSourceParamDTO.class)))
.thenReturn(connectionParam);

assertDoesNotThrow(() -> dataSourceService.createDataSource(loginUser, createDatasourceParam));
assertDoesNotThrow(() -> dataSourceService.updateDataSource(loginUser, updateDatasourceParam));
Mockito.verify(dataSourceProcessor, Mockito.never()).checkDataSourceConnectivity(Mockito.any());
}
}

@Test
public void testQueryDataSourceListPaging() {

Expand Down
2 changes: 2 additions & 0 deletions dolphinscheduler-api/src/test/resources/application.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ registry:

api:
audit-enable: true
# Whether to test datasource connectivity before creating or updating a datasource.
datasource-connection-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,8 @@ alert:

api:
audit-enable: false
# Whether to test datasource connectivity before creating or updating a datasource.
datasource-connection-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
Expand Down
Loading