From 68a5e89586d1dbd5bb715dd53e5d24ede5867be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 2 Apr 2026 14:08:18 +0800 Subject: [PATCH 01/10] Add separate workflow instance logs --- .../dao/entity/WorkflowInstance.java | 2 + .../resources/sql/dolphinscheduler_h2.sql | 1 + .../resources/sql/dolphinscheduler_mysql.sql | 1 + .../sql/dolphinscheduler_postgresql.sql | 1 + .../mysql/dolphinscheduler_ddl.sql | 19 ++++ .../mysql/dolphinscheduler_dml.sql | 16 ++++ .../postgresql/dolphinscheduler_ddl.sql | 18 ++++ .../postgresql/dolphinscheduler_dml.sql | 16 ++++ .../engine/WorkflowEventBusFireWorker.java | 4 + .../handler/AbstractCommandHandler.java | 5 + .../handler/ExecuteTaskCommandHandler.java | 7 ++ .../handler/ReRunWorkflowCommandHandler.java | 7 ++ .../RecoverFailureTaskCommandHandler.java | 7 ++ .../RecoverSerialWaitCommandHandler.java | 7 ++ .../handler/RunWorkflowCommandHandler.java | 7 ++ .../WorkflowFailoverCommandHandler.java | 7 ++ .../dispatcher/WorkerGroupDispatcher.java | 4 + .../AbstractWorkflowStateAction.java | 4 + .../server/master/failover/TaskFailover.java | 3 + .../master/log/WorkflowLogDiscriminator.java | 52 +++++++++++ .../server/master/log/WorkflowLogFilter.java | 59 ++++++++++++ .../server/master/log/WorkflowLogMarkers.java | 43 +++++++++ .../rpc/TaskExecutorEventListenerImpl.java | 59 ++++++++---- .../rpc/TaskInstanceControllerImpl.java | 6 ++ .../master/runner/WorkflowExecuteContext.java | 7 +- .../server/master/utils/WorkflowLogUtils.java | 92 +++++++++++++++++++ .../src/main/resources/logback-spring.xml | 22 +++++ 27 files changed, 456 insertions(+), 20 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogDiscriminator.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogFilter.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogMarkers.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/WorkflowLogUtils.java diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkflowInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkflowInstance.java index 8abf6a46666d..31caeb7adf1d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkflowInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkflowInstance.java @@ -158,6 +158,8 @@ public class WorkflowInstance { private Date restartTime; + private String logPath; + /** * set the process name with process define version and timestamp * diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 996fce0732b9..d4896e74a68b 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -636,6 +636,7 @@ CREATE TABLE t_ds_workflow_instance var_pool longtext, dry_run int NULL DEFAULT 0, restart_time datetime DEFAULT NULL, + log_path longtext DEFAULT NULL, PRIMARY KEY (id) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 7067cd7429aa..d3fb8a6fea77 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -644,6 +644,7 @@ CREATE TABLE `t_ds_workflow_instance` ( `dry_run` tinyint(4) DEFAULT '0' COMMENT 'dry run flag：0 normal, 1 dry run', `next_workflow_instance_id` int(11) DEFAULT '0' COMMENT 'serial queue next workflowInstanceId', `restart_time` datetime DEFAULT NULL COMMENT 'workflow instance restart time', + `log_path` longtext DEFAULT NULL COMMENT 'workflow instance log path', PRIMARY KEY (`id`), KEY `workflow_instance_index` (`workflow_definition_code`,`id`) USING BTREE, KEY `start_time_index` (`start_time`,`end_time`) USING BTREE diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 36261d7a8c2d..f8d2f0b22519 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -584,6 +584,7 @@ CREATE TABLE t_ds_workflow_instance ( dry_run int DEFAULT '0' , next_workflow_instance_id int DEFAULT '0', restart_time timestamp DEFAULT NULL , + log_path text DEFAULT NULL , PRIMARY KEY (id) ) ; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..d186ab70e3f7 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +ALTER TABLE `t_ds_workflow_instance` +ADD COLUMN `log_path` longtext NULL COMMENT 'workflow instance log path'; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..3d439848b44b --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +ALTER TABLE t_ds_workflow_instance ADD COLUMN IF NOT EXISTS "log_path" text DEFAULT NULL; \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java index 2e1c807edf80..70145fdf28a8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable; import org.apache.dolphinscheduler.server.master.runner.IWorkflowExecuteContext; import org.apache.dolphinscheduler.server.master.utils.ExceptionUtils; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -86,11 +87,14 @@ public void fireAllRegisteredEvent() { final String workflowInstanceName = workflowExecutionRunnable.getName(); try { LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC( + workflowExecutionRunnable.getWorkflowExecuteContext().getWorkflowInstance().getLogPath()); doFireSingleWorkflowEventBus(workflowExecutionRunnable); } catch (Exception ex) { log.error("Fire event failed for WorkflowExecuteRunnable: {}", workflowInstanceName, ex); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java index 2c9df21a870c..1bfc215a695e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java @@ -81,6 +81,7 @@ public WorkflowExecutionRunnable handleCommand(final Command command) { assembleWorkflowInstanceLifecycleListeners(workflowExecuteContextBuilder); assembleWorkflowEventBus(workflowExecuteContextBuilder); assembleWorkflowExecutionGraph(workflowExecuteContextBuilder); + assembleLogPath(workflowExecuteContextBuilder); final WorkflowExecutionRunnableBuilder workflowExecutionRunnableBuilder = WorkflowExecutionRunnableBuilder .builder() @@ -159,4 +160,8 @@ protected void assembleProject( workflowExecuteContextBuilder.setProject(project); } + protected void assembleLogPath(final WorkflowExecuteContextBuilder workflowExecuteContextBuilder) { + workflowExecuteContextBuilder.setLogPath(workflowExecuteContextBuilder.getWorkflowInstance().getLogPath()); + } + } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java index 43afdb09e733..f2393867fe09 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnable; import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnableBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -86,6 +87,12 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setTaskDependType(command.getTaskDependType()); } workflowInstance.setHost(masterConfig.getMasterAddress()); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getRestartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java index 524c7a225a5c..dfbeb49db9fd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; import java.util.List; @@ -74,6 +75,12 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setHost(masterConfig.getMasterAddress()); workflowInstance.setEndTime(null); workflowInstance.setRunTimes(workflowInstance.getRunTimes() + 1); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getRestartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java index 92f5fb4ed0c7..14dc1060432f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java @@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnableBuilder; import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskInstanceFactories; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.ArrayList; import java.util.HashSet; @@ -95,6 +96,12 @@ protected void assembleWorkflowInstance( workflowInstance.setStateWithDesc(WorkflowExecutionStatus.RUNNING_EXECUTION, command.getCommandType().name()); workflowInstance.setCommandType(command.getCommandType()); workflowInstance.setHost(masterConfig.getMasterAddress()); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getStartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java index 09486db8baac..e91332e29cbb 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -45,6 +46,12 @@ protected void assembleWorkflowInstance(WorkflowExecuteContext.WorkflowExecuteCo .orElseThrow(() -> new IllegalArgumentException("Cannot find WorkflowInstance:" + workflowInstanceId)); workflowInstance.setStateWithDesc(WorkflowExecutionStatus.RUNNING_EXECUTION, command.getCommandType().name()); workflowInstance.setHost(masterConfig.getMasterAddress()); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getRestartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java index c56006c89ad8..54a2ba949f26 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java @@ -35,6 +35,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnable; import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnableBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; @@ -77,6 +78,12 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setHost(masterConfig.getMasterAddress()); workflowInstance.setCommandParam(command.getCommandParam()); workflowInstance.setGlobalParams(mergeCommandParamsWithWorkflowParams(command, workflowDefinition)); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getStartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.upsertWorkflowInstance(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java index e18d02b400f8..d64be2e2e9e9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnable; import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskExecutionRunnableBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; import java.util.Map; @@ -89,6 +90,12 @@ protected void assembleWorkflowInstance( workflowInstance.setRestartTime(new Date()); workflowInstance.setState(workflowFailoverCommandParam.getWorkflowExecutionStatus()); workflowInstance.setHost(masterConfig.getMasterAddress()); + workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( + workflowInstance.getRestartTime(), + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId())); + workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java index 51ccd2f9cf11..b37e2d30423b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.dispatcher.event.TaskDispatchableEvent; import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskFailedLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.dolphinscheduler.task.executor.log.TaskExecutorMDCUtils; import java.util.Date; @@ -93,9 +94,12 @@ public void run() { TaskExecutorMDCUtils.MDCAutoClosable ignore = TaskExecutorMDCUtils.logWithMDC(taskExecutionRunnable.getId())) { LogUtils.setWorkflowInstanceIdMDC(taskExecutionRunnable.getTaskInstance().getWorkflowInstanceId()); + WorkflowLogUtils + .setWorkflowInstanceLogFullPathMDC(taskExecutionRunnable.getWorkflowInstance().getLogPath()); doDispatchTask(taskExecutionRunnable); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java index 59e770c4fb1e..7ae98653bd31 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable; import org.apache.dolphinscheduler.server.master.metrics.WorkflowInstanceMetrics; import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.dolphinscheduler.service.alert.WorkflowAlertManager; import org.apache.commons.collections4.CollectionUtils; @@ -108,12 +109,15 @@ protected void triggerTasks(final IWorkflowExecutionRunnable workflowExecutionRu protected void pauseActiveTask(final IWorkflowExecutionRunnable workflowExecutionRunnable) { try { LogUtils.setWorkflowInstanceIdMDC(workflowExecutionRunnable.getId()); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC( + workflowExecutionRunnable.getWorkflowExecuteContext().getWorkflowInstance().getLogPath()); workflowExecutionRunnable .getWorkflowExecutionGraph() .getActiveTaskExecutionRunnable() .forEach(ITaskExecutionRunnable::pause); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/TaskFailover.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/TaskFailover.java index c68b4bb4e6f4..343d44c5bde6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/TaskFailover.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/failover/TaskFailover.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskFailoverLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.springframework.stereotype.Component; @@ -28,8 +29,10 @@ public class TaskFailover { public void failoverTask(final ITaskExecutionRunnable taskExecutionRunnable) { LogUtils.setWorkflowInstanceIdMDC(taskExecutionRunnable.getWorkflowInstance().getId()); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(taskExecutionRunnable.getWorkflowInstance().getLogPath()); taskExecutionRunnable.getWorkflowEventBus().publish(TaskFailoverLifecycleEvent.of(taskExecutionRunnable)); LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogDiscriminator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogDiscriminator.java new file mode 100644 index 000000000000..d8baaea1577f --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogDiscriminator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.log; + +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import org.slf4j.MDC; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.sift.AbstractDiscriminator; + +/** + * Workflow Log Discriminator + */ +@Slf4j +@Getter +@Setter +public class WorkflowLogDiscriminator extends AbstractDiscriminator { + + private String key; + + private String logBase; + + @Override + public String getDiscriminatingValue(ILoggingEvent event) { + String workflowInstanceLogPath = MDC.get(WorkflowLogUtils.WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY); + if (workflowInstanceLogPath == null) { + log.error("The workflow instance log path is null, please check the logback configuration, log: {}", event); + } + return workflowInstanceLogPath; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogFilter.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogFilter.java new file mode 100644 index 000000000000..c9c42fa2d9a3 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogFilter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.log; + +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.slf4j.MDC; +import org.slf4j.Marker; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.Filter; +import ch.qos.logback.core.spi.FilterReply; + +/** + * This class is used to filter the log of the workflow instance. + */ +@Slf4j +public class WorkflowLogFilter extends Filter { + + @Override + public FilterReply decide(ILoggingEvent event) { + String workflowInstanceLogPath = MDC.get(WorkflowLogUtils.WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY); + // If the workflowInstanceLogPath is empty, it means that the log is not related to a workflow instance. + if (StringUtils.isEmpty(workflowInstanceLogPath)) { + return FilterReply.DENY; + } + + final Marker marker = event.getMarker(); + if (marker == null) { + return FilterReply.ACCEPT; + } + if (marker.contains(WorkflowLogMarkers.includeInWorkflowLog())) { + return FilterReply.ACCEPT; + } + if (marker.contains(WorkflowLogMarkers.excludeInWorkflowLog())) { + return FilterReply.DENY; + } + return FilterReply.ACCEPT; + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogMarkers.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogMarkers.java new file mode 100644 index 000000000000..0653e555d2ed --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/log/WorkflowLogMarkers.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.log; + +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; + +public class WorkflowLogMarkers { + + private static final Marker WORKFLOW_LOGGER_EXCLUDE_MARKER = MarkerFactory.getMarker("WORKFLOW_LOGGER_EXCLUDE"); + + private static final Marker WORKFLOW_LOGGER_INCLUDE_MARKER = MarkerFactory.getMarker("WORKFLOW_LOGGER_INCLUDE"); + + /** + * The marker used to exclude logs from the workflow instance log file. + */ + public static Marker excludeInWorkflowLog() { + return WORKFLOW_LOGGER_EXCLUDE_MARKER; + } + + /** + * The marker used to include logs from the workflow instance log file. + */ + public static Marker includeInWorkflowLog() { + return WORKFLOW_LOGGER_INCLUDE_MARKER; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java index 94df43ede4d0..fb71709b7bbe 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.rpc; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.extract.master.ITaskExecutorEventListener; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository; @@ -29,14 +30,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable; import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable; -import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorKilledLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorPausedLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent; -import org.apache.dolphinscheduler.task.executor.events.TaskExecutorSuccessLifecycleEvent; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; @@ -54,7 +48,10 @@ public class TaskExecutorEventListenerImpl implements ITaskExecutorEventListener @Override public void onTaskExecutorDispatched(final TaskExecutorDispatchedLifecycleEvent taskExecutorDispatchedLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorDispatchedLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorDispatchedLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorDispatchedLifecycleEvent); @@ -66,12 +63,16 @@ public void onTaskExecutorDispatched(final TaskExecutorDispatchedLifecycleEvent taskExecutionRunnable.getWorkflowEventBus().publish(taskDispatchedLifecycleEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override public void onTaskExecutorRunning(final TaskExecutorStartedLifecycleEvent taskExecutorStartedLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorStartedLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorStartedLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorStartedLifecycleEvent); @@ -84,31 +85,39 @@ public void onTaskExecutorRunning(final TaskExecutorStartedLifecycleEvent taskEx taskExecutionRunnable.getWorkflowEventBus().publish(taskRunningEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override - public void onTaskExecutorRuntimeContextChanged(final TaskExecutorRuntimeContextChangedLifecycleEvent taskExecutorRuntimeContextChangedLifecycleEventr) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorRuntimeContextChangedLifecycleEventr.getWorkflowInstanceId()); + public void onTaskExecutorRuntimeContextChanged(final TaskExecutorRuntimeContextChangedLifecycleEvent taskExecutorRuntimeContextChangedLifecycleEvent) { + int workflowInstanceId = taskExecutorRuntimeContextChangedLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = - getTaskExecutionRunnable(taskExecutorRuntimeContextChangedLifecycleEventr); + getTaskExecutionRunnable(taskExecutorRuntimeContextChangedLifecycleEvent); final TaskRuntimeContextChangedEvent taskRuntimeContextChangedEvent = TaskRuntimeContextChangedEvent.builder() .taskExecutionRunnable(taskExecutionRunnable) - .runtimeContext(taskExecutorRuntimeContextChangedLifecycleEventr.getAppIds()) + .runtimeContext(taskExecutorRuntimeContextChangedLifecycleEvent.getAppIds()) .build(); taskExecutionRunnable.getWorkflowEventBus().publish(taskRuntimeContextChangedEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskExecutorSuccessLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorSuccessLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorSuccessLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorSuccessLifecycleEvent); @@ -120,12 +129,16 @@ public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskEx taskExecutionRunnable.getWorkflowEventBus().publish(taskSuccessEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override public void onTaskExecutorFailed(final TaskExecutorFailedLifecycleEvent taskExecutorFailedLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorFailedLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorFailedLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorFailedLifecycleEvent); @@ -136,12 +149,16 @@ public void onTaskExecutorFailed(final TaskExecutorFailedLifecycleEvent taskExec taskExecutionRunnable.getWorkflowEventBus().publish(taskFailedEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override public void onTaskExecutorKilled(final TaskExecutorKilledLifecycleEvent taskExecutorKilledLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorKilledLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorKilledLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorKilledLifecycleEvent); @@ -152,12 +169,16 @@ public void onTaskExecutorKilled(final TaskExecutorKilledLifecycleEvent taskExec taskExecutionRunnable.getWorkflowEventBus().publish(taskKilledEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @Override public void onTaskExecutorPaused(final TaskExecutorPausedLifecycleEvent taskExecutorPausedLifecycleEvent) { - LogUtils.setWorkflowInstanceIdMDC(taskExecutorPausedLifecycleEvent.getWorkflowInstanceId()); + int workflowInstanceId = taskExecutorPausedLifecycleEvent.getWorkflowInstanceId(); + LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); + WorkflowInstance workflowInstance = workflowRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); try { final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorPausedLifecycleEvent); @@ -165,6 +186,7 @@ public void onTaskExecutorPaused(final TaskExecutorPausedLifecycleEvent taskExec taskExecutionRunnable.getWorkflowEventBus().publish(taskPausedEvent); } finally { LogUtils.removeWorkflowInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } @@ -184,5 +206,4 @@ private ITaskExecutionRunnable getTaskExecutionRunnable(final IReportableTaskExe } return taskExecutionRunnable; } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java index 16b7910b82d8..e28abf153241 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.rpc; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.extract.master.ITaskInstanceController; import org.apache.dolphinscheduler.extract.master.transportor.TaskGroupSlotAcquireSuccessNotifyRequest; import org.apache.dolphinscheduler.extract.master.transportor.TaskGroupSlotAcquireSuccessNotifyResponse; @@ -25,6 +26,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskDispatchLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable; import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import lombok.extern.slf4j.Slf4j; @@ -46,6 +48,9 @@ public TaskGroupSlotAcquireSuccessNotifyResponse notifyTaskGroupSlotAcquireSucce final int workflowInstanceId = taskGroupSlotAcquireSuccessNotifyRequest.getWorkflowInstanceId(); final int taskInstanceId = taskGroupSlotAcquireSuccessNotifyRequest.getTaskInstanceId(); LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstanceId, taskInstanceId); + WorkflowInstance workflowInstance = + workflowExecutionRunnableMemoryRepository.get(workflowInstanceId).getWorkflowInstance(); + WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); final IWorkflowExecutionRunnable workflowExecutionRunnable = workflowExecutionRunnableMemoryRepository.get(workflowInstanceId); if (workflowExecutionRunnable == null) { @@ -68,6 +73,7 @@ public TaskGroupSlotAcquireSuccessNotifyResponse notifyTaskGroupSlotAcquireSucce return TaskGroupSlotAcquireSuccessNotifyResponse.success(); } finally { LogUtils.removeWorkflowAndTaskInstanceIdMDC(); + WorkflowLogUtils.removeWorkflowInstanceLogFullPathMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java index 0727ff5e9841..8796649c12bd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java @@ -55,6 +55,8 @@ public class WorkflowExecuteContext implements IWorkflowExecuteContext { private final List workflowInstanceLifecycleListeners; + private String logPath; + public static WorkflowExecuteContextBuilder builder() { return new WorkflowExecuteContextBuilder(); } @@ -79,6 +81,8 @@ public static class WorkflowExecuteContextBuilder { private Project project; + private String logPath; + public WorkflowExecuteContextBuilder withCommand(Command command) { this.command = command; return this; @@ -93,7 +97,8 @@ public WorkflowExecuteContext build() { workflowGraph, workflowExecutionGraph, workflowEventBus, - Optional.ofNullable(workflowInstanceLifecycleListeners).orElse(Collections.emptyList())); + Optional.ofNullable(workflowInstanceLifecycleListeners).orElse(Collections.emptyList()), + logPath); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/WorkflowLogUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/WorkflowLogUtils.java new file mode 100644 index 000000000000..db1ea5256669 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/WorkflowLogUtils.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.utils; + +import org.apache.dolphinscheduler.common.constants.DateConstants; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.server.master.log.WorkflowLogDiscriminator; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; +import java.util.Optional; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import ch.qos.logback.classic.sift.SiftingAppender; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.spi.AppenderAttachable; + +@Slf4j +@UtilityClass +public class WorkflowLogUtils { + + private static final Path WORKFLOW_INSTANCE_LOG_BASE_PATH = getWorkflowInstanceLogBasePath(); + public static final String WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY = "workflowInstanceLogFullPath"; + + public static String getWorkflowInstanceLogFullPath(Date workflowStartTime, + Long workflowDefinitionCode, + int workflowDefinitionVersion, + int workflowInstanceId) { + if (WORKFLOW_INSTANCE_LOG_BASE_PATH == null) { + throw new IllegalArgumentException( + "Cannot find the workflow instance log base path, please check your logback.xml file"); + } + final String workflowLogFileName = Paths.get( + String.valueOf(workflowDefinitionCode), + String.valueOf(workflowDefinitionVersion), + String.format("%s.log", workflowInstanceId)).toString(); + return WORKFLOW_INSTANCE_LOG_BASE_PATH + .resolve(DateUtils.format(workflowStartTime, DateConstants.YYYYMMDD, null)) + .resolve(workflowLogFileName) + .toString(); + } + + /** + * Get workflow instance log base absolute path, this is defined in logback.xml + */ + public static Path getWorkflowInstanceLogBasePath() { + return Optional.of(LoggerFactory.getILoggerFactory()) + .map(e -> (AppenderAttachable) (e.getLogger("ROOT"))) + .map(e -> (SiftingAppender) (e.getAppender("WORKFLOWLOGFILE"))) + .map(e -> ((WorkflowLogDiscriminator) (e.getDiscriminator()))) + .map(WorkflowLogDiscriminator::getLogBase) + .map(e -> Paths.get(e).toAbsolutePath()) + .orElse(null); + } + + public static String getWorkflowInstanceLogFullPathMDC() { + return MDC.get(WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY); + } + + public static void setWorkflowInstanceLogFullPathMDC(String workflowInstanceLogFullPath) { + if (workflowInstanceLogFullPath == null) { + log.warn("workflowInstanceLogFullPath is null"); + return; + } + MDC.put(WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY, workflowInstanceLogFullPath); + } + + public static void removeWorkflowInstanceLogFullPathMDC() { + MDC.remove(WORKFLOW_INSTANCE_LOG_FULL_PATH_MDC_KEY); + } +} diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index a4c3a4f22dfe..b16c201335d4 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -51,6 +51,27 @@ + + + + + workflowInstanceLogFullPath + ${log.base}/workflows + + + + ${workflowInstanceLogFullPath} + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{10}:[%line] - %message%n + + UTF-8 + + true + + + + ${log.base}/dolphinscheduler-master.log @@ -75,6 +96,7 @@ + From f38aa56826aed6c32bfe0d5e536c75e5327961e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 16 Apr 2026 19:57:23 +0800 Subject: [PATCH 02/10] Resolve conflicts --- .../server/master/engine/WorkflowEventBusFireWorker.java | 2 +- .../master/engine/task/dispatcher/WorkerGroupDispatcher.java | 2 +- .../server/master/rpc/TaskExecutorEventListenerImpl.java | 4 +--- .../server/master/rpc/TaskInstanceControllerImpl.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java index 36e7efb27586..0b914381c35a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java @@ -88,7 +88,7 @@ public void fireAllRegisteredEvent() { try { LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId); WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC( - workflowExecutionRunnable.getWorkflowExecuteContext().getWorkflowInstance().getLogPath()); + workflowExecution.getWorkflowExecuteContext().getWorkflowInstance().getLogPath()); doFireSingleWorkflowEventBus(workflowExecution); } catch (Exception ex) { log.error("Fire event failed for WorkflowExecuteRunnable: {}", workflowInstanceName, ex); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java index 6606732cc08f..4f0d4d3901dd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/dispatcher/WorkerGroupDispatcher.java @@ -95,7 +95,7 @@ public void run() { TaskExecutorMDCUtils.logWithMDC(taskExecution.getId())) { LogUtils.setWorkflowInstanceIdMDC(taskExecution.getTaskInstance().getWorkflowInstanceId()); WorkflowLogUtils - .setWorkflowInstanceLogFullPathMDC(taskExecutionRunnable.getWorkflowInstance().getLogPath()); + .setWorkflowInstanceLogFullPathMDC(taskExecution.getWorkflowInstance().getLogPath()); doDispatchTask(taskExecution); } finally { LogUtils.removeWorkflowInstanceIdMDC(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java index a08a3f3ad480..42b9f42261a4 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent; import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.workflow.execution.IWorkflowExecution; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent; @@ -38,9 +39,6 @@ import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorSuccessLifecycleEvent; -import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable; -import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java index 9b7c5ae147c4..579d5ecbd7a2 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceControllerImpl.java @@ -49,7 +49,7 @@ public TaskGroupSlotAcquireSuccessNotifyResponse notifyTaskGroupSlotAcquireSucce final int taskInstanceId = taskGroupSlotAcquireSuccessNotifyRequest.getTaskInstanceId(); LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstanceId, taskInstanceId); WorkflowInstance workflowInstance = - workflowExecutionRunnableMemoryRepository.get(workflowInstanceId).getWorkflowInstance(); + workflowExecutionMemoryRepository.get(workflowInstanceId).getWorkflowInstance(); WorkflowLogUtils.setWorkflowInstanceLogFullPathMDC(workflowInstance.getLogPath()); final IWorkflowExecution workflowExecution = workflowExecutionMemoryRepository.get(workflowInstanceId); From c2e475a6a44ae8fce2daa28b1aab27314aa51497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 14 May 2026 18:26:46 +0800 Subject: [PATCH 03/10] set logPath in AbstractCommandHandler --- .../mysql/dolphinscheduler_ddl.sql | 2 +- .../postgresql/dolphinscheduler_ddl.sql | 2 +- .../handler/AbstractCommandHandler.java | 24 ++++++++++++++++--- .../handler/ExecuteTaskCommandHandler.java | 7 ------ .../handler/ReRunWorkflowCommandHandler.java | 7 ------ .../RecoverFailureTaskCommandHandler.java | 7 ------ .../RecoverSerialWaitCommandHandler.java | 7 ------ .../handler/RunWorkflowCommandHandler.java | 7 ------ .../WorkflowFailoverCommandHandler.java | 7 ------ .../master/runner/WorkflowExecuteContext.java | 7 +----- 10 files changed, 24 insertions(+), 53 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql index d186ab70e3f7..55d6005df575 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql @@ -16,4 +16,4 @@ */ ALTER TABLE `t_ds_workflow_instance` -ADD COLUMN `log_path` longtext NULL COMMENT 'workflow instance log path'; \ No newline at end of file +ADD COLUMN `log_path` longtext NULL COMMENT 'workflow instance log path'; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql index 3d439848b44b..57459d10e067 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -15,4 +15,4 @@ * limitations under the License. */ -ALTER TABLE t_ds_workflow_instance ADD COLUMN IF NOT EXISTS "log_path" text DEFAULT NULL; \ No newline at end of file +ALTER TABLE t_ds_workflow_instance ADD COLUMN IF NOT EXISTS "log_path" text DEFAULT NULL; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java index 9d9a02a4b66a..4a55488753fc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/AbstractCommandHandler.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.dao.repository.ProjectDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionLogDao; +import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.extract.master.command.ICommandParam; import org.apache.dolphinscheduler.server.master.engine.WorkflowEventBus; import org.apache.dolphinscheduler.server.master.engine.command.ICommandHandler; @@ -39,10 +40,12 @@ import org.apache.dolphinscheduler.server.master.engine.workflow.listener.IWorkflowLifecycleListener; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.stream.Collectors; @@ -69,6 +72,9 @@ public abstract class AbstractCommandHandler implements ICommandHandler { @Autowired protected ProjectDao projectDao; + @Autowired + protected WorkflowInstanceDao workflowInstanceDao; + @Override public WorkflowExecution handleCommand(final Command command) { final WorkflowExecuteContextBuilder workflowExecuteContextBuilder = WorkflowExecuteContext.builder() @@ -81,7 +87,7 @@ public WorkflowExecution handleCommand(final Command command) { assembleWorkflowInstanceLifecycleListeners(workflowExecuteContextBuilder); assembleWorkflowEventBus(workflowExecuteContextBuilder); assembleWorkflowExecutionGraph(workflowExecuteContextBuilder); - assembleLogPath(workflowExecuteContextBuilder); + assembleWorkflowInstanceLogPath(workflowExecuteContextBuilder); final WorkflowExecutionBuilder workflowExecutionBuilder = WorkflowExecutionBuilder .builder() @@ -160,8 +166,20 @@ protected void assembleProject( workflowExecuteContextBuilder.setProject(project); } - protected void assembleLogPath(final WorkflowExecuteContextBuilder workflowExecuteContextBuilder) { - workflowExecuteContextBuilder.setLogPath(workflowExecuteContextBuilder.getWorkflowInstance().getLogPath()); + protected void assembleWorkflowInstanceLogPath(final WorkflowExecuteContextBuilder workflowExecuteContextBuilder) { + final WorkflowInstance workflowInstance = workflowExecuteContextBuilder.getWorkflowInstance(); + final Date logTime = workflowInstance.getRestartTime() != null + ? workflowInstance.getRestartTime() + : workflowInstance.getStartTime(); + final String logPath = WorkflowLogUtils.getWorkflowInstanceLogFullPath( + logTime, + workflowInstance.getWorkflowDefinitionCode(), + workflowInstance.getWorkflowDefinitionVersion(), + workflowInstance.getId()); + workflowInstance.setLogPath(logPath); + workflowInstanceDao.updateById(workflowInstance); + + workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java index 22a6ee7f066d..846fb7c4b387 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ExecuteTaskCommandHandler.java @@ -36,7 +36,6 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecution; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -87,12 +86,6 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setTaskDependType(command.getTaskDependType()); } workflowInstance.setHost(masterConfig.getMasterAddress()); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getRestartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java index dfbeb49db9fd..524c7a225a5c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/ReRunWorkflowCommandHandler.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; import java.util.List; @@ -75,12 +74,6 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setHost(masterConfig.getMasterAddress()); workflowInstance.setEndTime(null); workflowInstance.setRunTimes(workflowInstance.getRunTimes() + 1); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getRestartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java index 526b008a1fb2..f0a30fb74ff0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverFailureTaskCommandHandler.java @@ -33,7 +33,6 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionBuilder; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskInstanceFactories; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.ArrayList; import java.util.HashSet; @@ -96,12 +95,6 @@ protected void assembleWorkflowInstance( workflowInstance.setStateWithDesc(WorkflowExecutionStatus.RUNNING_EXECUTION, command.getCommandType().name()); workflowInstance.setCommandType(command.getCommandType()); workflowInstance.setHost(masterConfig.getMasterAddress()); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getStartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java index e91332e29cbb..09486db8baac 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RecoverSerialWaitCommandHandler.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -46,12 +45,6 @@ protected void assembleWorkflowInstance(WorkflowExecuteContext.WorkflowExecuteCo .orElseThrow(() -> new IllegalArgumentException("Cannot find WorkflowInstance:" + workflowInstanceId)); workflowInstance.setStateWithDesc(WorkflowExecutionStatus.RUNNING_EXECUTION, command.getCommandType().name()); workflowInstance.setHost(masterConfig.getMasterAddress()); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getRestartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java index 2b63a7092f62..414921a6ba98 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java @@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecution; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import org.apache.commons.collections4.CollectionUtils; @@ -78,12 +77,6 @@ protected void assembleWorkflowInstance(final WorkflowExecuteContextBuilder work workflowInstance.setHost(masterConfig.getMasterAddress()); workflowInstance.setCommandParam(command.getCommandParam()); workflowInstance.setGlobalParams(mergeCommandParamsWithWorkflowParams(command, workflowDefinition)); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getStartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.upsertWorkflowInstance(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java index 9b48cd1bd54c..8234eec7b24c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/WorkflowFailoverCommandHandler.java @@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecution; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; -import org.apache.dolphinscheduler.server.master.utils.WorkflowLogUtils; import java.util.Date; import java.util.Map; @@ -90,12 +89,6 @@ protected void assembleWorkflowInstance( workflowInstance.setRestartTime(new Date()); workflowInstance.setState(workflowFailoverCommandParam.getWorkflowExecutionStatus()); workflowInstance.setHost(masterConfig.getMasterAddress()); - workflowInstance.setLogPath(WorkflowLogUtils.getWorkflowInstanceLogFullPath( - workflowInstance.getRestartTime(), - workflowInstance.getWorkflowDefinitionCode(), - workflowInstance.getWorkflowDefinitionVersion(), - workflowInstance.getId())); - workflowInstanceDao.updateById(workflowInstance); workflowExecuteContextBuilder.setWorkflowInstance(workflowInstance); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java index 8796649c12bd..0727ff5e9841 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteContext.java @@ -55,8 +55,6 @@ public class WorkflowExecuteContext implements IWorkflowExecuteContext { private final List workflowInstanceLifecycleListeners; - private String logPath; - public static WorkflowExecuteContextBuilder builder() { return new WorkflowExecuteContextBuilder(); } @@ -81,8 +79,6 @@ public static class WorkflowExecuteContextBuilder { private Project project; - private String logPath; - public WorkflowExecuteContextBuilder withCommand(Command command) { this.command = command; return this; @@ -97,8 +93,7 @@ public WorkflowExecuteContext build() { workflowGraph, workflowExecutionGraph, workflowEventBus, - Optional.ofNullable(workflowInstanceLifecycleListeners).orElse(Collections.emptyList()), - logPath); + Optional.ofNullable(workflowInstanceLifecycleListeners).orElse(Collections.emptyList())); } } From bf4e76e7e5a6d790d74585c8ceaebf35cd3d92f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 19 May 2026 18:46:23 +0800 Subject: [PATCH 04/10] add query and delete workflow log in backend --- .../controller/WorkflowLoggerController.java | 158 ++++++++++++++ .../dolphinscheduler/api/enums/Status.java | 5 + .../api/executor/logging/LocalLogClient.java | 41 ++++ .../executor/logging/LogClientDelegate.java | 71 +++++++ .../api/executor/logging/RemoteLogClient.java | 10 + .../api/service/WorkflowLoggerService.java | 70 +++++++ .../impl/WorkflowInstanceServiceImpl.java | 14 ++ .../impl/WorkflowLoggerServiceImpl.java | 166 +++++++++++++++ .../executor/logging/LocalLogClientTest.java | 65 ++++++ .../executor/logging/LocalLogClientTest.java~ | 198 ++++++++++++++++++ .../dao/entity/ResponseWorkflowLog.java | 32 +++ .../extract/common/ILogService.java | 13 ++ .../common/service/impl/LogServiceImpl.java | 43 ++++ ...orkflowInstanceLogFileDownloadRequest.java | 34 +++ ...rkflowInstanceLogFileDownloadResponse.java | 37 ++++ .../WorkflowInstanceLogPageQueryRequest.java | 38 ++++ .../WorkflowInstanceLogPageQueryResponse.java | 37 ++++ 17 files changed, 1032 insertions(+) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLoggerService.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseWorkflowLog.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryResponse.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java new file mode 100644 index 000000000000..2d9cc9ebce3e --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.apache.dolphinscheduler.api.enums.Status.DOWNLOAD_WORKFLOW_INSTANCE_LOG_FILE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKFLOW_INSTANCE_LOG_ERROR; + +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.WorkflowLoggerService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.dao.entity.ResponseWorkflowLog; +import org.apache.dolphinscheduler.dao.entity.User; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; + +/** + * Workflow logger controller + */ +@Tag(name = "WORKFLOW_LOGGER_TAG") +@RestController +@RequestMapping("/workflow-log") +public class WorkflowLoggerController extends BaseController { + + @Autowired + private WorkflowLoggerService workflowLoggerService; + + /** + * query workflow log + * + * @param loginUser login user + * @param workflowInstanceId workflow instance id + * @param skipNum skip number + * @param limit limit + * @return workflow log content + */ + @Operation(summary = "queryWorkflowLog", description = "QUERY_WORKFLOW_INSTANCE_LOG_NOTES") + @Parameters({ + @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100")) + }) + @GetMapping(value = "/detail") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_WORKFLOW_INSTANCE_LOG_ERROR) + public Result queryWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "workflowInstanceId") int workflowInstanceId, + @RequestParam(value = "skipLineNum") int skipNum, + @RequestParam(value = "limit") int limit) { + return workflowLoggerService.queryWorkflowLog(loginUser, workflowInstanceId, skipNum, limit); + } + + /** + * download workflow log file + * + * @param loginUser login user + * @param workflowInstanceId workflow instance id + * @return log file content + */ + @Operation(summary = "downloadWorkflowLog", description = "DOWNLOAD_WORKFLOW_INSTANCE_LOG_NOTES") + @Parameters({ + @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) + }) + @GetMapping(value = "/download-log") + @ResponseBody + @ApiException(DOWNLOAD_WORKFLOW_INSTANCE_LOG_FILE_ERROR) + public ResponseEntity downloadWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "workflowInstanceId") int workflowInstanceId) { + byte[] logBytes = workflowLoggerService.getWorkflowLogBytes(loginUser, workflowInstanceId); + return ResponseEntity + .ok() + .header(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + "workflow-" + workflowInstanceId + "-" + System.currentTimeMillis() + + ".log" + "\"") + .body(logBytes); + } + + /** + * query workflow log in specified project + * + * @param loginUser login user + * @param projectCode project code + * @param workflowInstanceId workflow instance id + * @param skipNum skip number + * @param limit limit + * @return workflow log content + */ + @Operation(summary = "queryWorkflowLogInSpecifiedProject", description = "QUERY_WORKFLOW_INSTANCE_LOG_IN_SPECIFIED_PROJECT_NOTES") + @Parameters({ + @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true, schema = @Schema(implementation = long.class)), + @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100")) + }) + @GetMapping(value = "/{projectCode}/detail") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_WORKFLOW_INSTANCE_LOG_ERROR) + public Result queryWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, + @RequestParam(value = "workflowInstanceId") int workflowInstanceId, + @RequestParam(value = "skipLineNum") int skipNum, + @RequestParam(value = "limit") int limit) { + return Result.success( + workflowLoggerService.queryWorkflowLog(loginUser, projectCode, workflowInstanceId, skipNum, limit)); + } + + /** + * download workflow log file in specified project + * + * @param loginUser login user + * @param projectCode project code + * @param workflowInstanceId workflow instance id + * @return log file content + */ + @Operation(summary = "downloadWorkflowLogInSpecifiedProject", description = "DOWNLOAD_WORKFLOW_INSTANCE_LOG_IN_SPECIFIED_PROJECT_NOTES") + @Parameters({ + @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true, schema = @Schema(implementation = long.class)), + @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) + }) + @GetMapping(value = "/{projectCode}/download-log") + @ResponseBody + @ApiException(DOWNLOAD_WORKFLOW_INSTANCE_LOG_FILE_ERROR) + public ResponseEntity downloadWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, + @RequestParam(value = "workflowInstanceId") int workflowInstanceId) { + byte[] logBytes = workflowLoggerService.getWorkflowLogBytes(loginUser, projectCode, workflowInstanceId); + return ResponseEntity + .ok() + .header(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + "workflow-" + workflowInstanceId + "-" + System.currentTimeMillis() + + ".log" + "\"") + .body(logBytes); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 3d24834bd429..b06418311c71 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -295,6 +295,11 @@ public enum Status { UPDATE_PROJECT_PREFERENCE_STATE_ERROR(10303, "Failed to update the state of the project preference", "更新项目偏好设置错误"), VERSION_INFO_STATE_ERROR(10304, "Failed to obtain project version and address", "获取版本信息错误"), + QUERY_WORKFLOW_INSTANCE_LOG_ERROR(10305, "view workflow instance log error: {0}", "查询工作流实例日志错误: {0}"), + DOWNLOAD_WORKFLOW_INSTANCE_LOG_FILE_ERROR(10306, "download workflow instance log file error", "下载工作流日志文件错误"), + WORKFLOW_INSTANCE_NOT_FOUND(10307, "workflow instance not found", "工作流实例不存在"), + WORKFLOW_INSTANCE_HOST_IS_NULL(10308, "workflow instance host is null", "工作流实例host为空"), + OIDC_TOKEN_EXCHANGE_FAILED(15000, "OIDC token exchange failed", "OIDC令牌交换失败"), OIDC_ID_TOKEN_ISSUER_INVALID(15001, "Invalid issuer in OIDC ID token", "OIDC ID令牌的颁发者无效"), OIDC_ID_TOKEN_AUDIENCE_INVALID(15002, "Invalid audience in OIDC ID token", "OIDC ID令牌的受众无效"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClient.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClient.java index ebe8b2e9fea2..994a7418d6dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClient.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClient.java @@ -18,12 +18,17 @@ package org.apache.dolphinscheduler.api.executor.logging; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.extract.base.client.Clients; import org.apache.dolphinscheduler.extract.common.ILogService; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryResponse; import lombok.extern.slf4j.Slf4j; @@ -85,4 +90,40 @@ private ILogService getProxyLogService(TaskInstance taskInstance) { log.debug("Created log service for host: {}", taskInstance.getHost()); return logService; } + + public WorkflowInstanceLogPageQueryResponse getWorkflowPartLog(WorkflowInstance workflowInstance, int skipLineNum, + int limit) { + return getLocalWorkflowPartLog(workflowInstance, skipLineNum, limit); + } + + public WorkflowInstanceLogFileDownloadResponse getWorkflowWholeLog(WorkflowInstance workflowInstance) { + return getLocalWorkflowWholeLog(workflowInstance); + } + + private WorkflowInstanceLogFileDownloadResponse getLocalWorkflowWholeLog(WorkflowInstance workflowInstance) { + WorkflowInstanceLogFileDownloadRequest request = new WorkflowInstanceLogFileDownloadRequest( + workflowInstance.getId(), + workflowInstance.getLogPath()); + return getProxyWorkflowLogService(workflowInstance).getWorkflowInstanceWholeLogFileBytes(request); + } + + private WorkflowInstanceLogPageQueryResponse getLocalWorkflowPartLog(WorkflowInstance workflowInstance, + int skipLineNum, int limit) { + WorkflowInstanceLogPageQueryRequest request = WorkflowInstanceLogPageQueryRequest + .builder() + .workflowInstanceId(workflowInstance.getId()) + .workflowInstanceLogAbsolutePath(workflowInstance.getLogPath()) + .skipLineNum(skipLineNum) + .limit(limit) + .build(); + return getProxyWorkflowLogService(workflowInstance).pageQueryWorkflowInstanceLog(request); + } + + private ILogService getProxyWorkflowLogService(WorkflowInstance workflowInstance) { + ILogService logService = Clients + .withService(ILogService.class) + .withHost(workflowInstance.getHost()); + log.debug("Created log service for host: {}", workflowInstance.getHost()); + return logService; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LogClientDelegate.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LogClientDelegate.java index ac5b6ecb2ba4..1fca8115a316 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LogClientDelegate.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LogClientDelegate.java @@ -18,9 +18,12 @@ package org.apache.dolphinscheduler.api.executor.logging; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.extract.common.transportor.LogResponseStatus; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -109,4 +112,72 @@ private boolean checkNodeExists(TaskInstance taskInstance) { return exists; } + /** + * Retrieves a portion of the log string for a given workflow instance. + * This method first attempts to fetch the log from local storage; if unsuccessful, it tries to obtain the log from remote storage. + * + * @param workflowInstance The workflow instance object, containing information needed for log retrieval. + * @param skipLineNum The number of log lines to skip from the beginning. + * @param limit The maximum number of log lines to retrieve. + * @return A string containing the specified portion of the log. + */ + public String getWorkflowPartLogString(WorkflowInstance workflowInstance, int skipLineNum, int limit) { + checkWorkflowArgs(workflowInstance); + if (checkWorkflowNodeExists(workflowInstance)) { + WorkflowInstanceLogPageQueryResponse response = + localLogClient.getWorkflowPartLog(workflowInstance, skipLineNum, limit); + if (response.getCode() == LogResponseStatus.SUCCESS) { + return response.getLogContent(); + } else { + log.warn("get part log string is not success for workflow instance {}; reason :{}", + workflowInstance.getId(), response.getMessage()); + return remoteLogClient.getWorkflowPartLog(workflowInstance, skipLineNum, limit); + } + } else { + return remoteLogClient.getWorkflowPartLog(workflowInstance, skipLineNum, limit); + } + } + + /** + * Retrieves the complete log content for a given workflow instance as a byte array. + * This method first attempts to fetch the log from local storage; if unsuccessful, it tries to obtain the log from remote storage. + * + * @param workflowInstance The workflow instance object, containing information needed for log retrieval. + * @return A byte array containing the complete log content. + */ + public byte[] getWorkflowWholeLogBytes(WorkflowInstance workflowInstance) { + checkWorkflowArgs(workflowInstance); + if (checkWorkflowNodeExists(workflowInstance)) { + WorkflowInstanceLogFileDownloadResponse response = localLogClient.getWorkflowWholeLog(workflowInstance); + if (response.getCode() == LogResponseStatus.SUCCESS) { + return response.getLogBytes(); + } else { + log.warn("get whole log bytes is not success for workflow instance {}; reason :{}", + workflowInstance.getId(), response.getMessage()); + return remoteLogClient.getWorkflowWholeLog(workflowInstance); + } + } else { + return remoteLogClient.getWorkflowWholeLog(workflowInstance); + } + } + + private static void checkWorkflowArgs(WorkflowInstance workflowInstance) { + if (workflowInstance == null) { + throw new IllegalArgumentException("workflow instance is null"); + } + } + + private boolean checkWorkflowNodeExists(WorkflowInstance workflowInstance) { + String host = workflowInstance.getHost(); + if (host == null || host.isEmpty()) { + log.warn("Host is null or empty for workflow instance {}", workflowInstance.getId()); + return false; + } + boolean exists = registryClient.checkNodeExists(host, RegistryNodeType.MASTER); + if (!exists) { + log.warn("Node {} does not exist for workflow instance {}", host, workflowInstance.getId()); + } + return exists; + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/RemoteLogClient.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/RemoteLogClient.java index 1b3542e96209..318abd02b405 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/RemoteLogClient.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/RemoteLogClient.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.common.utils.LogUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.springframework.stereotype.Component; @@ -51,4 +52,13 @@ public String getPartLog(TaskInstance taskInstance, int skipLineNum, int limit) LogUtils.readPartFileContentFromRemote(taskInstance.getLogPath(), skipLineNum, limit)); } + public byte[] getWorkflowWholeLog(WorkflowInstance workflowInstance) { + return LogUtils.getFileContentBytesFromRemote(workflowInstance.getLogPath()); + } + + public String getWorkflowPartLog(WorkflowInstance workflowInstance, int skipLineNum, int limit) { + return LogUtils.rollViewLogLines( + LogUtils.readPartFileContentFromRemote(workflowInstance.getLogPath(), skipLineNum, limit)); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLoggerService.java new file mode 100644 index 000000000000..02395246cbe6 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkflowLoggerService.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.ResponseWorkflowLog; +import org.apache.dolphinscheduler.dao.entity.User; + +/** + * Workflow logger service + */ +public interface WorkflowLoggerService { + + /** + * query workflow log + * + * @param loginUser login user + * @param workflowInstanceId workflow instance id + * @param skipLineNum skip line number + * @param limit limit + * @return workflow log content + */ + Result queryWorkflowLog(User loginUser, int workflowInstanceId, int skipLineNum, int limit); + + /** + * get workflow log bytes + * + * @param loginUser login user + * @param workflowInstanceId workflow instance id + * @return log byte array + */ + byte[] getWorkflowLogBytes(User loginUser, int workflowInstanceId); + + /** + * query workflow log in specified project + * + * @param loginUser login user + * @param projectCode project code + * @param workflowInstanceId workflow instance id + * @param skipLineNum skip line number + * @param limit limit + * @return log string data + */ + String queryWorkflowLog(User loginUser, long projectCode, int workflowInstanceId, int skipLineNum, int limit); + + /** + * get workflow log bytes in specified project + * + * @param loginUser login user + * @param projectCode project code + * @param workflowInstanceId workflow instance id + * @return log byte array + */ + byte[] getWorkflowLogBytes(User loginUser, long projectCode, int workflowInstanceId); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java index 7dccef947a37..90c7f975b0c0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java @@ -75,6 +75,8 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceMapDao; import org.apache.dolphinscheduler.dao.utils.WorkflowUtils; +import org.apache.dolphinscheduler.extract.base.client.Clients; +import org.apache.dolphinscheduler.extract.common.ILogService; import org.apache.dolphinscheduler.extract.master.command.ICommandParam; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; @@ -818,6 +820,18 @@ public void deleteWorkflowInstanceById(int workflowInstanceId) { deleteSubWorkflowInstanceIfNeeded(workflowInstanceId); // delete alert alertDao.deleteByWorkflowInstanceId(workflowInstanceId); + // delete workflow instance log + WorkflowInstance workflowInstance = processService.findWorkflowInstanceById(workflowInstanceId); + if (workflowInstance != null && StringUtils.isNotBlank(workflowInstance.getLogPath())) { + // Remove workflow instance log failed will not affect the deletion of workflow instance + try { + Clients.withService(ILogService.class) + .withHost(workflowInstance.getHost()) + .removeWorkflowInstanceLog(workflowInstance.getLogPath()); + } catch (Exception ex) { + log.error("Remove workflow instance log error", ex); + } + } // delete workflow instance workflowInstanceDao.deleteById(workflowInstanceId); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java new file mode 100644 index 000000000000..153380feb0e9 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.executor.logging.LogClientDelegate; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.service.WorkflowLoggerService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.dao.entity.ResponseWorkflowLog; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; +import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; + +import org.apache.commons.lang3.StringUtils; + +import java.nio.charset.StandardCharsets; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.common.primitives.Bytes; + +/** + * Workflow logger service impl + */ +@Service +@Slf4j +public class WorkflowLoggerServiceImpl extends BaseServiceImpl implements WorkflowLoggerService { + + private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; + + @Autowired + private WorkflowInstanceDao workflowInstanceDao; + + @Autowired + private ProjectService projectService; + + @Autowired + private LogClientDelegate logClientDelegate; + + @Override + public Result queryWorkflowLog(User loginUser, int workflowInstanceId, int skipLineNum, + int limit) { + WorkflowInstance workflowInstance = workflowInstanceDao.queryById(workflowInstanceId); + if (workflowInstance == null) { + log.error("Workflow instance does not exist, workflowInstanceId:{}.", workflowInstanceId); + return Result.error(Status.WORKFLOW_INSTANCE_NOT_FOUND); + } + if (StringUtils.isBlank(workflowInstance.getHost())) { + log.error("Host of workflow instance is null, workflowInstanceId:{}.", workflowInstanceId); + return Result.error(Status.WORKFLOW_INSTANCE_HOST_IS_NULL); + } + projectService.checkProjectAndAuthThrowException(loginUser, workflowInstance.getProjectCode(), VIEW_LOG); + String log = queryWorkflowLog(workflowInstance, skipLineNum, limit); + int lineNum = log.split("\r\n").length; + return Result.success(new ResponseWorkflowLog(lineNum, log)); + } + + @Override + public byte[] getWorkflowLogBytes(User loginUser, int workflowInstanceId) { + WorkflowInstance workflowInstance = workflowInstanceDao.queryById(workflowInstanceId); + if (workflowInstance == null || StringUtils.isBlank(workflowInstance.getHost())) { + throw new RuntimeException("workflow instance is null or host is null"); + } + projectService.checkProjectAndAuthThrowException(loginUser, workflowInstance.getProjectCode(), DOWNLOAD_LOG); + return getWorkflowLogBytes(workflowInstance); + } + + @Override + public String queryWorkflowLog(User loginUser, long projectCode, int workflowInstanceId, int skipLineNum, + int limit) { + projectService.checkProjectAndAuthThrowException(loginUser, projectCode, VIEW_LOG); + WorkflowInstance workflowInstance = workflowInstanceDao.queryById(workflowInstanceId); + if (workflowInstance == null || StringUtils.isBlank(workflowInstance.getHost())) { + throw new RuntimeException("Workflow instance not found or host is null"); + } + if (projectCode != workflowInstance.getProjectCode()) { + throw new RuntimeException("Workflow instance does not exist in project"); + } + return queryWorkflowLog(workflowInstance, skipLineNum, limit); + } + + @Override + public byte[] getWorkflowLogBytes(User loginUser, long projectCode, int workflowInstanceId) { + projectService.checkProjectAndAuthThrowException(loginUser, projectCode, DOWNLOAD_LOG); + WorkflowInstance workflowInstance = workflowInstanceDao.queryById(workflowInstanceId); + if (workflowInstance == null || StringUtils.isBlank(workflowInstance.getHost())) { + throw new RuntimeException("Workflow instance not found or host is null"); + } + if (projectCode != workflowInstance.getProjectCode()) { + throw new RuntimeException("Workflow instance does not exist in project"); + } + return getWorkflowLogBytes(workflowInstance); + } + + private String queryWorkflowLog(WorkflowInstance workflowInstance, int skipLineNum, int limit) { + final String logPath = workflowInstance.getLogPath(); + log.info("Query workflow instance log, workflowInstanceId:{}, workflowInstanceName:{}, host: {}, logPath:{}", + workflowInstance.getId(), workflowInstance.getName(), workflowInstance.getHost(), logPath); + if (StringUtils.isBlank(logPath)) { + throw new RuntimeException("WorkflowInstanceLogPath is empty"); + } + + StringBuilder sb = new StringBuilder(); + if (skipLineNum == 0) { + String head = String.format(LOG_HEAD_FORMAT, + logPath, + workflowInstance.getHost(), + Constants.SYSTEM_LINE_SEPARATOR); + sb.append(head); + } + + try { + String logContent = logClientDelegate.getWorkflowPartLogString(workflowInstance, skipLineNum, limit); + if (logContent != null) { + sb.append(logContent); + } + return sb.toString(); + } catch (Throwable ex) { + log.error("Query workflow instance log error", ex); + throw new RuntimeException("Query workflow instance log error: " + ex.getMessage(), ex); + } + } + + private byte[] getWorkflowLogBytes(WorkflowInstance workflowInstance) { + String host = workflowInstance.getHost(); + String logPath = workflowInstance.getLogPath(); + + byte[] head = String.format(LOG_HEAD_FORMAT, + logPath, + host, + Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + + byte[] logBytes; + + try { + logBytes = logClientDelegate.getWorkflowWholeLogBytes(workflowInstance); + return Bytes.concat(head, logBytes); + } catch (Exception ex) { + log.error("Download WorkflowInstance: {} Log Error", workflowInstance.getName(), ex); + throw new RuntimeException("Download workflow instance log error", ex); + } + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java index 38ed86900790..a3f7363e355d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.extract.common.ILogService; @@ -31,6 +32,10 @@ import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryResponse; import java.io.IOException; import java.net.ServerSocket; @@ -98,6 +103,39 @@ public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLog public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { } + + @Override + public WorkflowInstanceLogFileDownloadResponse getWorkflowInstanceWholeLogFileBytes(WorkflowInstanceLogFileDownloadRequest workflowInstanceLogFileDownloadRequest) { + if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 1) { + return new WorkflowInstanceLogFileDownloadResponse(new byte[0], LogResponseStatus.SUCCESS, ""); + } else if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 10) { + return new WorkflowInstanceLogFileDownloadResponse("log content".getBytes(), + LogResponseStatus.SUCCESS, + ""); + } + + throw new ServiceException("download error"); + } + + @Override + public WorkflowInstanceLogPageQueryResponse pageQueryWorkflowInstanceLog(WorkflowInstanceLogPageQueryRequest workflowInstanceLogPageQueryRequest) { + if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() != null) { + if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 10) { + return new WorkflowInstanceLogPageQueryResponse("Partial log content", + LogResponseStatus.SUCCESS, + ""); + } + } + + return new WorkflowInstanceLogPageQueryResponse(); + } + + @Override + public void removeWorkflowInstanceLog(String workflowInstanceLogAbsolutePath) { + + } }); springServerMethodInvokerDiscovery.start(); } @@ -134,4 +172,31 @@ public void testGetPartLogSuccess() { assertNotNull(actualResponse); assertEquals("Partial log content", actualResponse.getLogContent()); } + + @Test + public void testGetWorkflowWholeLogSuccess() { + WorkflowInstance workflowInstance = new WorkflowInstance(); + workflowInstance.setHost("127.0.0.1:" + nettyServerPort); + workflowInstance.setId(1); + workflowInstance.setLogPath("/path/to/log"); + + WorkflowInstanceLogFileDownloadResponse actualResponse = localLogClient.getWorkflowWholeLog(workflowInstance); + + assertNotNull(actualResponse); + assertArrayEquals("".getBytes(), actualResponse.getLogBytes()); + } + + @Test + public void testGetWorkflowPartLogSuccess() { + WorkflowInstance workflowInstance = new WorkflowInstance(); + workflowInstance.setHost("127.0.0.1:" + nettyServerPort); + workflowInstance.setId(10); + workflowInstance.setLogPath("/path/to/log"); + + WorkflowInstanceLogPageQueryResponse actualResponse = + localLogClient.getWorkflowPartLog(workflowInstance, 0, 10); + + assertNotNull(actualResponse); + assertEquals("Partial log content", actualResponse.getLogContent()); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ new file mode 100644 index 000000000000..ae1360cedf27 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.logging; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; +import org.apache.dolphinscheduler.extract.common.ILogService; +import org.apache.dolphinscheduler.extract.common.transportor.LogResponseStatus; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; + +import java.io.IOException; +import java.net.ServerSocket; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class LocalLogClientTest { + + @InjectMocks + private LocalLogClient localLogClient; + + private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; + + private int nettyServerPort = 18080; + + @BeforeEach + public void setUp() { + try (ServerSocket s = new ServerSocket(0)) { + nettyServerPort = s.getLocalPort(); + } catch (IOException e) { + return; + } + + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( + NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build()); + springServerMethodInvokerDiscovery.start(); + springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { + + @Override + public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { + if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 1) { + return new TaskInstanceLogFileDownloadResponse(new byte[0], LogResponseStatus.SUCCESS, ""); + } else if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogFileDownloadResponse("log content".getBytes(), LogResponseStatus.SUCCESS, + ""); + } + + throw new ServiceException("download error"); + } + + @Override + public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() != null) { + if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 10) { + return new TaskInstanceLogPageQueryResponse("Partial log content", LogResponseStatus.SUCCESS, + ""); + } + } + + return new TaskInstanceLogPageQueryResponse(); + } + + @Override + public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { + + } + + @Override + public WorkflowInstanceLogFileDownloadResponse getWorkflowInstanceWholeLogFileBytes(WorkflowInstanceLogFileDownloadRequest workflowInstanceLogFileDownloadRequest) { + if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 1) { + return new WorkflowInstanceLogFileDownloadResponse(new byte[0], LogResponseStatus.SUCCESS, ""); + } else if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 10) { + return new WorkflowInstanceLogFileDownloadResponse("log content".getBytes(), + LogResponseStatus.SUCCESS, + ""); + } + + throw new ServiceException("download error"); + } + + @Override + public WorkflowInstanceLogPageQueryResponse pageQueryWorkflowInstanceLog(WorkflowInstanceLogPageQueryRequest workflowInstanceLogPageQueryRequest) { + if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() != null) { + if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 100) { + throw new ServiceException("query log error"); + } else if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 10) { + return new WorkflowInstanceLogPageQueryResponse("Partial log content", + LogResponseStatus.SUCCESS, + ""); + } + } + + return new WorkflowInstanceLogPageQueryResponse(); + } + + @Override + public void removeWorkflowInstanceLog(String workflowInstanceLogAbsolutePath) { + + } + }); + springServerMethodInvokerDiscovery.start(); + } + + @AfterEach + public void tearDown() { + if (springServerMethodInvokerDiscovery != null) { + springServerMethodInvokerDiscovery.close(); + } + } + + @Test + public void testGetWholeLogSuccess() { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setHost("127.0.0.1:" + nettyServerPort); + taskInstance.setId(1); + taskInstance.setLogPath("/path/to/log"); + + TaskInstanceLogFileDownloadResponse actualResponse = localLogClient.getWholeLog(taskInstance); + + assertNotNull(actualResponse); + assertArrayEquals("".getBytes(), actualResponse.getLogBytes()); + } + + @Test + public void testGetPartLogSuccess() { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(10); + taskInstance.setHost("127.0.0.1:" + nettyServerPort); + taskInstance.setLogPath("/path/to/log"); + + TaskInstanceLogPageQueryResponse actualResponse = localLogClient.getPartLog(taskInstance, 0, 10); + + assertNotNull(actualResponse); + assertEquals("Partial log content", actualResponse.getLogContent()); + } + + @Test + public void testGetWorkflowWholeLogSuccess() { + WorkflowInstance workflowInstance = new WorkflowInstance(); + workflowInstance.setHost("127.0.0.1:" + nettyServerPort); + workflowInstance.setId(1); + workflowInstance.setLogPath("/path/to/log"); + + WorkflowInstanceLogFileDownloadResponse actualResponse = localLogClient.getWorkflowWholeLog(workflowInstance); + + assertNotNull(actualResponse); + assertArrayEquals("".getBytes(), actualResponse.getLogBytes()); + } + + @Test + public void testGetWorkflowPartLogSuccess() { + WorkflowInstance workflowInstance = new WorkflowInstance(); + workflowInstance.setHost("127.0.0.1:" + nettyServerPort); + workflowInstance.setId(10); + workflowInstance.setLogPath("/path/to/log"); + + WorkflowInstanceLogPageQueryResponse actualResponse = + localLogClient.getWorkflowPartLog(workflowInstance, 0, 10); + + assertNotNull(actualResponse); + assertEquals("Partial log content", actualResponse.getLogContent()); + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseWorkflowLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseWorkflowLog.java new file mode 100644 index 000000000000..4dd7b753b90a --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseWorkflowLog.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * log of the logger service response + */ +@Data +@AllArgsConstructor +public class ResponseWorkflowLog { + + private int lineNum; + private String message; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java index 4484e519421b..58f9e4f6ab67 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java @@ -23,6 +23,10 @@ import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryResponse; @RpcService public interface ILogService { @@ -36,4 +40,13 @@ public interface ILogService { @RpcMethod void removeTaskInstanceLog(String taskInstanceLogAbsolutePath); + @RpcMethod + WorkflowInstanceLogFileDownloadResponse getWorkflowInstanceWholeLogFileBytes(WorkflowInstanceLogFileDownloadRequest workflowInstanceLogFileDownloadRequest); + + @RpcMethod + WorkflowInstanceLogPageQueryResponse pageQueryWorkflowInstanceLog(WorkflowInstanceLogPageQueryRequest workflowInstanceLogPageQueryRequest); + + @RpcMethod + void removeWorkflowInstanceLog(String workflowInstanceLogAbsolutePath); + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java index bbe2c09d9291..f8175ff763ff 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java @@ -25,6 +25,10 @@ import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.WorkflowInstanceLogPageQueryResponse; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -82,4 +86,43 @@ public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { FileUtils.deleteFile(taskInstanceLogAbsolutePath); } + @Override + public WorkflowInstanceLogFileDownloadResponse getWorkflowInstanceWholeLogFileBytes(WorkflowInstanceLogFileDownloadRequest workflowInstanceLogFileDownloadRequest) { + final WorkflowInstanceLogFileDownloadResponse workflowInstanceLogFileDownloadResponse = + new WorkflowInstanceLogFileDownloadResponse(); + try { + byte[] bytes = LogUtils + .getFileContentBytesFromLocal( + workflowInstanceLogFileDownloadRequest.getWorkflowInstanceLogAbsolutePath()); + workflowInstanceLogFileDownloadResponse.setLogBytes(bytes); + } catch (Exception e) { + workflowInstanceLogFileDownloadResponse.setCode(LogResponseStatus.ERROR); + workflowInstanceLogFileDownloadResponse.setMessage(ExceptionUtils.getRootCauseMessage(e)); + } + return workflowInstanceLogFileDownloadResponse; + } + + @Override + public WorkflowInstanceLogPageQueryResponse pageQueryWorkflowInstanceLog(WorkflowInstanceLogPageQueryRequest workflowInstanceLogPageQueryRequest) { + final WorkflowInstanceLogPageQueryResponse workflowInstanceLogPageQueryResponse = + new WorkflowInstanceLogPageQueryResponse(); + List lines; + try { + lines = LogUtils.readPartFileContentFromLocal( + workflowInstanceLogPageQueryRequest.getWorkflowInstanceLogAbsolutePath(), + workflowInstanceLogPageQueryRequest.getSkipLineNum(), + workflowInstanceLogPageQueryRequest.getLimit()); + workflowInstanceLogPageQueryResponse.setLogContent(LogUtils.rollViewLogLines(lines)); + } catch (Exception e) { + workflowInstanceLogPageQueryResponse.setCode(LogResponseStatus.ERROR); + workflowInstanceLogPageQueryResponse.setMessage(ExceptionUtils.getMessage(e)); + } + return workflowInstanceLogPageQueryResponse; + } + + @Override + public void removeWorkflowInstanceLog(String workflowInstanceLogAbsolutePath) { + FileUtils.deleteFile(workflowInstanceLogAbsolutePath); + } + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadRequest.java new file mode 100644 index 000000000000..fca7833151bf --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadRequest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.transportor; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class WorkflowInstanceLogFileDownloadRequest { + + private Integer workflowInstanceId; + + private String workflowInstanceLogAbsolutePath; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadResponse.java new file mode 100644 index 000000000000..18a8f7c3cade --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogFileDownloadResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.transportor; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class WorkflowInstanceLogFileDownloadResponse { + + private byte[] logBytes; + + private LogResponseStatus code = LogResponseStatus.SUCCESS; + + private String message; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryRequest.java new file mode 100644 index 000000000000..15f6ea1c6e31 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryRequest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.transportor; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class WorkflowInstanceLogPageQueryRequest { + + private Integer workflowInstanceId; + + private String workflowInstanceLogAbsolutePath; + + private Integer skipLineNum; + + private Integer limit; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryResponse.java new file mode 100644 index 000000000000..dce7cced7e73 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/transportor/WorkflowInstanceLogPageQueryResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.transportor; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class WorkflowInstanceLogPageQueryResponse { + + private String logContent; + + private LogResponseStatus code = LogResponseStatus.SUCCESS; + + private String message; + +} From c81b96109685d691c378f0f04e21917bacf20759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 19 May 2026 19:04:28 +0800 Subject: [PATCH 05/10] update import --- .../api/controller/WorkflowLoggerController.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java index 2d9cc9ebce3e..f378b1e93266 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java @@ -31,6 +31,14 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; From ca022f7caa6440ce9d5e22c42bd145c15401e64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 21 May 2026 19:15:41 +0800 Subject: [PATCH 06/10] add query and delete workflow log in frontend --- .../controller/WorkflowLoggerController.java | 60 +--------------- .../impl/WorkflowLoggerServiceImpl.java | 2 +- .../src/locales/en_US/project.ts | 2 + .../src/locales/zh_CN/project.ts | 2 + .../modules/workflow-instances/types.ts | 1 + .../src/service/modules/workflow-log/index.ts | 32 +++++++++ .../src/service/modules/workflow-log/types.ts | 27 +++++++ .../instance/components/table-action.tsx | 56 ++++++++++++++- .../projects/workflow/instance/index.tsx | 55 +++++++++++++- .../projects/workflow/instance/use-table.ts | 72 +++++++++++++++++-- 10 files changed, 241 insertions(+), 68 deletions(-) create mode 100644 dolphinscheduler-ui/src/service/modules/workflow-log/index.ts create mode 100644 dolphinscheduler-ui/src/service/modules/workflow-log/types.ts diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java index f378b1e93266..7b545ed88005 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkflowLoggerController.java @@ -32,7 +32,6 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -102,65 +101,8 @@ public ResponseEntity downloadWorkflowLog(@Parameter(hidden = true) @RequestAttr return ResponseEntity .ok() .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + "workflow-" + workflowInstanceId + "-" + System.currentTimeMillis() - + ".log" + "\"") + "attachment; filename=\"" + "workflow-" + workflowInstanceId + ".log" + "\"") .body(logBytes); } - /** - * query workflow log in specified project - * - * @param loginUser login user - * @param projectCode project code - * @param workflowInstanceId workflow instance id - * @param skipNum skip number - * @param limit limit - * @return workflow log content - */ - @Operation(summary = "queryWorkflowLogInSpecifiedProject", description = "QUERY_WORKFLOW_INSTANCE_LOG_IN_SPECIFIED_PROJECT_NOTES") - @Parameters({ - @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true, schema = @Schema(implementation = long.class)), - @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), - @Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")), - @Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100")) - }) - @GetMapping(value = "/{projectCode}/detail") - @ResponseStatus(HttpStatus.OK) - @ApiException(QUERY_WORKFLOW_INSTANCE_LOG_ERROR) - public Result queryWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, - @RequestParam(value = "workflowInstanceId") int workflowInstanceId, - @RequestParam(value = "skipLineNum") int skipNum, - @RequestParam(value = "limit") int limit) { - return Result.success( - workflowLoggerService.queryWorkflowLog(loginUser, projectCode, workflowInstanceId, skipNum, limit)); - } - - /** - * download workflow log file in specified project - * - * @param loginUser login user - * @param projectCode project code - * @param workflowInstanceId workflow instance id - * @return log file content - */ - @Operation(summary = "downloadWorkflowLogInSpecifiedProject", description = "DOWNLOAD_WORKFLOW_INSTANCE_LOG_IN_SPECIFIED_PROJECT_NOTES") - @Parameters({ - @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true, schema = @Schema(implementation = long.class)), - @Parameter(name = "workflowInstanceId", description = "WORKFLOW_INSTANCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) - }) - @GetMapping(value = "/{projectCode}/download-log") - @ResponseBody - @ApiException(DOWNLOAD_WORKFLOW_INSTANCE_LOG_FILE_ERROR) - public ResponseEntity downloadWorkflowLog(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode, - @RequestParam(value = "workflowInstanceId") int workflowInstanceId) { - byte[] logBytes = workflowLoggerService.getWorkflowLogBytes(loginUser, projectCode, workflowInstanceId); - return ResponseEntity - .ok() - .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + "workflow-" + workflowInstanceId + "-" + System.currentTimeMillis() - + ".log" + "\"") - .body(logBytes); - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java index 153380feb0e9..6399a094d9cf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowLoggerServiceImpl.java @@ -74,7 +74,7 @@ public Result queryWorkflowLog(User loginUser, int workflow } projectService.checkProjectAndAuthThrowException(loginUser, workflowInstance.getProjectCode(), VIEW_LOG); String log = queryWorkflowLog(workflowInstance, skipLineNum, limit); - int lineNum = log.split("\r\n").length; + int lineNum = log.split("\\r\\n").length; return Result.success(new ResponseWorkflowLog(lineNum, log)); } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 40e553d4dd1d..b326e98e708d 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -181,6 +181,8 @@ export default { recovery_failed: 'Recovery Failed', gantt: 'Gantt', name: 'Name', + auto_refresh: 'Auto Refresh', + no_log_content: 'No log content', all_status: 'AllStatus', submit_success: 'Submitted successfully', running: 'Running', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index abaa84a34b1a..4e61e3003ae3 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -179,6 +179,8 @@ export default { recovery_failed: '重跑失败任务', gantt: '甘特图', name: '名称', + auto_refresh: '自动刷新', + no_log_content: '暂无日志内容', all_status: '全部状态', submit_success: '提交成功', running: '正在运行', diff --git a/dolphinscheduler-ui/src/service/modules/workflow-instances/types.ts b/dolphinscheduler-ui/src/service/modules/workflow-instances/types.ts index 52bc977fa237..1ee7aa1eaac4 100644 --- a/dolphinscheduler-ui/src/service/modules/workflow-instances/types.ts +++ b/dolphinscheduler-ui/src/service/modules/workflow-instances/types.ts @@ -97,6 +97,7 @@ interface IWorkflowInstance { dryRun: number executorName: string host: string + logPath?: string count?: number disabled?: boolean buttonType?: string diff --git a/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts b/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts new file mode 100644 index 000000000000..13274150c0c5 --- /dev/null +++ b/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { axios } from '@/service/service' +import utils from '@/utils' +import { WorkflowLogIdReq, WorkflowLogReq } from './types' + +export function queryWorkflowLogDetail(params: WorkflowLogReq): any { + return axios({ + url: '/workflow-log/detail', + method: 'get', + params + }) +} + +export function downloadWorkflowLog(params: WorkflowLogIdReq): void { + utils.downloadFile('workflow-log/download-log', params) +} \ No newline at end of file diff --git a/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts b/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts new file mode 100644 index 000000000000..7cc2803da0da --- /dev/null +++ b/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface WorkflowLogIdReq { + workflowInstanceId: number +} + +interface WorkflowLogReq extends WorkflowLogIdReq { + limit: number + skipLineNum: number +} + +export { WorkflowLogIdReq, WorkflowLogReq } \ No newline at end of file diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/table-action.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/table-action.tsx index fffcaddf0e53..02dadd17295e 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/table-action.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/table-action.tsx @@ -25,7 +25,9 @@ import { CloseCircleOutlined, PauseCircleOutlined, ControlOutlined, - PlayCircleOutlined + PlayCircleOutlined, + AlignLeftOutlined, + DownloadOutlined } from '@vicons/antd' import { useI18n } from 'vue-i18n' import { useRouter } from 'vue-router' @@ -48,7 +50,9 @@ export default defineComponent({ 'reStore', 'stop', 'suspend', - 'deleteInstance' + 'deleteInstance', + 'viewLog', + 'downloadLog' ], setup(props, ctx) { const router: Router = useRouter() @@ -89,6 +93,14 @@ export default defineComponent({ ctx.emit('deleteInstance') } + const handleViewLog = () => { + ctx.emit('viewLog', props.row) + } + + const handleDownloadLog = () => { + ctx.emit('downloadLog', props.row) + } + return { handleEdit, handleReRun, @@ -97,6 +109,8 @@ export default defineComponent({ handleSuspend, handleDeleteInstance, handleGantt, + handleViewLog, + handleDownloadLog, ...toRefs(props) } }, @@ -295,6 +309,44 @@ export default defineComponent({ ) }} + + {{ + default: () => t('project.workflow.view_log'), + trigger: () => ( + + + + + + ) + }} + + + {{ + default: () => t('project.workflow.download_log'), + trigger: () => ( + + + + + + ) + }} + ) } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/index.tsx index c4de57aabaa1..53e45f5389b6 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/index.tsx @@ -27,6 +27,7 @@ import { } from 'naive-ui' import { useTable } from './use-table' import Card from '@/components/card' +import LogModal from '@/components/log-modal' import WorkflowInstanceCondition from './components/workflow-instance-condition' import type { IWorkflowInstanceSearch } from './types' import totalCount from '@/utils/tableTotalCount' @@ -35,8 +36,15 @@ export default defineComponent({ name: 'WorkflowInstanceList', setup() { let setIntervalP: number - const { variables, createColumns, getTableData, batchDeleteInstance } = - useTable() + const { + variables, + createColumns, + getTableData, + batchDeleteInstance, + viewLog, + downloadLog, + fetchLog + } = useTable() const requestData = () => { getTableData() @@ -64,6 +72,17 @@ export default defineComponent({ batchDeleteInstance() } + const onConfirmModal = () => { + variables.showLogModalRef = false + } + + const refreshLogs = () => { + variables.logRef = '' + variables.limit = 1000 + variables.skipLineNum = 0 + fetchLog(variables.currentLogRow) + } + onMounted(() => { createColumns(variables) requestData() @@ -82,11 +101,33 @@ export default defineComponent({ clearInterval(setIntervalP) }) + watch( + () => variables.showLogModalRef, + () => { + if (variables.showLogModalRef) { + variables.logRef = '' + variables.limit = 1000 + variables.skipLineNum = 0 + fetchLog(variables.currentLogRow) + } else { + variables.logRef = '' + variables.logLoadingRef = true + variables.currentLogRow = null + variables.skipLineNum = 0 + variables.limit = 1000 + } + } + ) + return { requestData, handleSearch, handleChangePageSize, handleBatchDelete, + onConfirmModal, + refreshLogs, + viewLog, + downloadLog, ...toRefs(variables) } }, @@ -150,6 +191,16 @@ export default defineComponent({ }} + ) } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts b/dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts index 02b7ba8a7479..611de39934da 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/use-table.ts @@ -16,8 +16,9 @@ */ import _ from 'lodash' -import { reactive, h, ref } from 'vue' +import { reactive, h, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' +import { useAsyncState } from '@vueuse/core' import { useRouter } from 'vue-router' import ButtonLink from '@/components/button-link' import { RowKey } from 'naive-ui/lib/data-table/src/interface' @@ -27,6 +28,7 @@ import { deleteWorkflowInstanceById, batchDeleteWorkflowInstanceByIds } from '@/service/modules/workflow-instances' +import { queryWorkflowLogDetail, downloadWorkflowLog } from '@/service/modules/workflow-log' import { execute } from '@/service/modules/executors' import TableAction from './components/table-action' import { @@ -69,7 +71,14 @@ export function useTable() { .workflowDefinitionCode ? ref(Number(router.currentRoute.value.query.workflowDefinitionCode)) : ref(), - loadingRef: ref(false) + loadingRef: ref(false), + // Log related variables + showLogModalRef: ref(false), + logRef: ref(''), + logLoadingRef: ref(true), + currentLogRow: ref(null), + skipLineNum: ref(0), + limit: ref(1000) }) const createColumns = (variables: any) => { @@ -239,7 +248,9 @@ export function useTable() { }) } }, - onDeleteInstance: () => deleteInstance(_row.id) + onDeleteInstance: () => deleteInstance(_row.id), + onViewLog: () => viewLog(_row), + onDownloadLog: () => downloadLog(_row) }) } ] @@ -353,14 +364,67 @@ export function useTable() { }) } + /** + * View workflow log + */ + const viewLog = (row: IWorkflowInstance) => { + variables.currentLogRow = row + variables.showLogModalRef = true + } + + /** + * Fetch workflow log + */ + const fetchLog = (row: any) => { + const { state } = useAsyncState( + queryWorkflowLogDetail({ + workflowInstanceId: Number(row.id), + limit: variables.limit, + skipLineNum: variables.skipLineNum + }).then((res: any) => { + variables.logRef += res?.message || '' + if (res?.message && res.message !== '') { + variables.skipLineNum += res.lineNum + fetchLog(row) + } else { + variables.logLoadingRef = false + } + }), + {} + ) + + return state + } + + /** + * Download workflow log + */ + const downloadLog = (row: IWorkflowInstance) => { + downloadWorkflowLog({ workflowInstanceId: row.id }) + } + + /** + * Close log modal + */ + const closeLogModal = () => { + variables.showLogModalRef = false + variables.logRef = '' + variables.currentLogRow = null + } + return { variables, createColumns, getTableData, - batchDeleteInstance + batchDeleteInstance, + viewLog, + downloadLog, + closeLogModal, + fetchLog } } + export function renderWorkflowStateCell( state: IWorkflowExecutionState, t: Function From c6ff5c9d8d1bb43dc4c121360d143ed683525c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 21 May 2026 19:21:31 +0800 Subject: [PATCH 07/10] remove locates --- dolphinscheduler-ui/src/locales/en_US/project.ts | 2 -- dolphinscheduler-ui/src/locales/zh_CN/project.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index b326e98e708d..40e553d4dd1d 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -181,8 +181,6 @@ export default { recovery_failed: 'Recovery Failed', gantt: 'Gantt', name: 'Name', - auto_refresh: 'Auto Refresh', - no_log_content: 'No log content', all_status: 'AllStatus', submit_success: 'Submitted successfully', running: 'Running', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 4e61e3003ae3..abaa84a34b1a 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -179,8 +179,6 @@ export default { recovery_failed: '重跑失败任务', gantt: '甘特图', name: '名称', - auto_refresh: '自动刷新', - no_log_content: '暂无日志内容', all_status: '全部状态', submit_success: '提交成功', running: '正在运行', From 26d09c0d23ca41341d7e38f7495ced24857e4449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 21 May 2026 20:36:08 +0800 Subject: [PATCH 08/10] Delete redundant file --- .../executor/logging/LocalLogClientTest.java~ | 198 ------------------ 1 file changed, 198 deletions(-) delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ deleted file mode 100644 index ae1360cedf27..000000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClientTest.java~ +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.executor.logging; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -import org.apache.dolphinscheduler.api.exceptions.ServiceException; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; -import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; -import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; -import org.apache.dolphinscheduler.extract.common.ILogService; -import org.apache.dolphinscheduler.extract.common.transportor.LogResponseStatus; -import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; -import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; -import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; -import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; - -import java.io.IOException; -import java.net.ServerSocket; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class LocalLogClientTest { - - @InjectMocks - private LocalLogClient localLogClient; - - private SpringServerMethodInvokerDiscovery springServerMethodInvokerDiscovery; - - private int nettyServerPort = 18080; - - @BeforeEach - public void setUp() { - try (ServerSocket s = new ServerSocket(0)) { - nettyServerPort = s.getLocalPort(); - } catch (IOException e) { - return; - } - - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( - NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build()); - springServerMethodInvokerDiscovery.start(); - springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { - - @Override - public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { - if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 1) { - return new TaskInstanceLogFileDownloadResponse(new byte[0], LogResponseStatus.SUCCESS, ""); - } else if (taskInstanceLogFileDownloadRequest.getTaskInstanceId() == 10) { - return new TaskInstanceLogFileDownloadResponse("log content".getBytes(), LogResponseStatus.SUCCESS, - ""); - } - - throw new ServiceException("download error"); - } - - @Override - public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { - if (taskInstanceLogPageQueryRequest.getTaskInstanceId() != null) { - if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 100) { - throw new ServiceException("query log error"); - } else if (taskInstanceLogPageQueryRequest.getTaskInstanceId() == 10) { - return new TaskInstanceLogPageQueryResponse("Partial log content", LogResponseStatus.SUCCESS, - ""); - } - } - - return new TaskInstanceLogPageQueryResponse(); - } - - @Override - public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { - - } - - @Override - public WorkflowInstanceLogFileDownloadResponse getWorkflowInstanceWholeLogFileBytes(WorkflowInstanceLogFileDownloadRequest workflowInstanceLogFileDownloadRequest) { - if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 1) { - return new WorkflowInstanceLogFileDownloadResponse(new byte[0], LogResponseStatus.SUCCESS, ""); - } else if (workflowInstanceLogFileDownloadRequest.getWorkflowInstanceId() == 10) { - return new WorkflowInstanceLogFileDownloadResponse("log content".getBytes(), - LogResponseStatus.SUCCESS, - ""); - } - - throw new ServiceException("download error"); - } - - @Override - public WorkflowInstanceLogPageQueryResponse pageQueryWorkflowInstanceLog(WorkflowInstanceLogPageQueryRequest workflowInstanceLogPageQueryRequest) { - if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() != null) { - if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 100) { - throw new ServiceException("query log error"); - } else if (workflowInstanceLogPageQueryRequest.getWorkflowInstanceId() == 10) { - return new WorkflowInstanceLogPageQueryResponse("Partial log content", - LogResponseStatus.SUCCESS, - ""); - } - } - - return new WorkflowInstanceLogPageQueryResponse(); - } - - @Override - public void removeWorkflowInstanceLog(String workflowInstanceLogAbsolutePath) { - - } - }); - springServerMethodInvokerDiscovery.start(); - } - - @AfterEach - public void tearDown() { - if (springServerMethodInvokerDiscovery != null) { - springServerMethodInvokerDiscovery.close(); - } - } - - @Test - public void testGetWholeLogSuccess() { - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setHost("127.0.0.1:" + nettyServerPort); - taskInstance.setId(1); - taskInstance.setLogPath("/path/to/log"); - - TaskInstanceLogFileDownloadResponse actualResponse = localLogClient.getWholeLog(taskInstance); - - assertNotNull(actualResponse); - assertArrayEquals("".getBytes(), actualResponse.getLogBytes()); - } - - @Test - public void testGetPartLogSuccess() { - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(10); - taskInstance.setHost("127.0.0.1:" + nettyServerPort); - taskInstance.setLogPath("/path/to/log"); - - TaskInstanceLogPageQueryResponse actualResponse = localLogClient.getPartLog(taskInstance, 0, 10); - - assertNotNull(actualResponse); - assertEquals("Partial log content", actualResponse.getLogContent()); - } - - @Test - public void testGetWorkflowWholeLogSuccess() { - WorkflowInstance workflowInstance = new WorkflowInstance(); - workflowInstance.setHost("127.0.0.1:" + nettyServerPort); - workflowInstance.setId(1); - workflowInstance.setLogPath("/path/to/log"); - - WorkflowInstanceLogFileDownloadResponse actualResponse = localLogClient.getWorkflowWholeLog(workflowInstance); - - assertNotNull(actualResponse); - assertArrayEquals("".getBytes(), actualResponse.getLogBytes()); - } - - @Test - public void testGetWorkflowPartLogSuccess() { - WorkflowInstance workflowInstance = new WorkflowInstance(); - workflowInstance.setHost("127.0.0.1:" + nettyServerPort); - workflowInstance.setId(10); - workflowInstance.setLogPath("/path/to/log"); - - WorkflowInstanceLogPageQueryResponse actualResponse = - localLogClient.getWorkflowPartLog(workflowInstance, 0, 10); - - assertNotNull(actualResponse); - assertEquals("Partial log content", actualResponse.getLogContent()); - } -} From 4062178aabf33571c72342bbd2de73d26d53de6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 22 May 2026 09:30:54 +0800 Subject: [PATCH 09/10] Add blank lines --- dolphinscheduler-ui/src/service/modules/workflow-log/index.ts | 2 +- dolphinscheduler-ui/src/service/modules/workflow-log/types.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts b/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts index 13274150c0c5..994d7d251e76 100644 --- a/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts +++ b/dolphinscheduler-ui/src/service/modules/workflow-log/index.ts @@ -29,4 +29,4 @@ export function queryWorkflowLogDetail(params: WorkflowLogReq): any { export function downloadWorkflowLog(params: WorkflowLogIdReq): void { utils.downloadFile('workflow-log/download-log', params) -} \ No newline at end of file +} diff --git a/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts b/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts index 7cc2803da0da..831971cfc75c 100644 --- a/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts +++ b/dolphinscheduler-ui/src/service/modules/workflow-log/types.ts @@ -24,4 +24,4 @@ interface WorkflowLogReq extends WorkflowLogIdReq { skipLineNum: number } -export { WorkflowLogIdReq, WorkflowLogReq } \ No newline at end of file +export { WorkflowLogIdReq, WorkflowLogReq } From 9f06e8b227f611729272e4c48cf75b26267be995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 9 Jun 2026 11:50:48 +0800 Subject: [PATCH 10/10] adapt to 3.5.0 --- .../{3.4.2_schema => 3.5.0_schema}/mysql/dolphinscheduler_ddl.sql | 0 .../{3.4.2_schema => 3.5.0_schema}/mysql/dolphinscheduler_dml.sql | 0 .../postgresql/dolphinscheduler_ddl.sql | 0 .../postgresql/dolphinscheduler_dml.sql | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{3.4.2_schema => 3.5.0_schema}/mysql/dolphinscheduler_ddl.sql (100%) rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{3.4.2_schema => 3.5.0_schema}/mysql/dolphinscheduler_dml.sql (100%) rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{3.4.2_schema => 3.5.0_schema}/postgresql/dolphinscheduler_ddl.sql (100%) rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{3.4.2_schema => 3.5.0_schema}/postgresql/dolphinscheduler_dml.sql (100%) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql similarity index 100% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_ddl.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql similarity index 100% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/mysql/dolphinscheduler_dml.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql similarity index 100% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_ddl.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql similarity index 100% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.2_schema/postgresql/dolphinscheduler_dml.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql