From fb5a23fe2f639c121edff2df74efa208d1d132e9 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 10:29:27 -0600 Subject: [PATCH 1/6] GitHub Action to verify that newly added files have the license header. This includes Python scripts for validation as well as a GitHub Action that runs them and comments on PRs if license headers are missing. Signed-off-by: David Venable --- .github/scripts/check-license-headers.py | 149 +++++++++++++++++++++ .github/scripts/get-new-files.py | 55 ++++++++ .github/workflows/license-header-check.yml | 86 ++++++++++++ 3 files changed, 290 insertions(+) create mode 100755 .github/scripts/check-license-headers.py create mode 100755 .github/scripts/get-new-files.py create mode 100644 .github/workflows/license-header-check.yml diff --git a/.github/scripts/check-license-headers.py b/.github/scripts/check-license-headers.py new file mode 100755 index 0000000000..b46ebbd114 --- /dev/null +++ b/.github/scripts/check-license-headers.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +""" +License Header Compliance Checker for OpenSearch Data Prepper + +This script checks that files contain the required license headers +as specified in CONTRIBUTING.md. + +Usage: + python check-license-headers.py file1.java file2.py ... + echo "file1.java\nfile2.py" | python check-license-headers.py +""" + +import os +import sys +from pathlib import Path +from typing import List + +# File extensions that require license headers +SUPPORTED_EXTENSIONS = { + '.java', '.groovy', '.gradle', # Java ecosystem + '.py', # Python + '.sh', '.bash', '.zsh', # Shell scripts + '.yaml', '.yml', # YAML files + '.properties', # Properties files +} + +def needs_license_header(file_path: str) -> bool: + """Check if a file needs a license header based on its extension.""" + path = Path(file_path) + return path.suffix.lower() in SUPPORTED_EXTENSIONS + +def check_file_header(file_path: str) -> bool: + """Check if a file has the required complete license header.""" + if not Path(file_path).exists(): + return True + + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + # Read first 15 lines to check for license header + lines = [] + for i, line in enumerate(f): + if i >= 15: # Only check first 15 lines + break + lines.append(line) + + content = ''.join(lines) + + # Check for all 5 required license header components + required_components = [ + 'Copyright OpenSearch Contributors', + 'SPDX-License-Identifier: Apache-2.0', + 'The OpenSearch Contributors require contributions made to', + 'this file be licensed under the Apache-2.0 license or a', + 'compatible open source license.' + ] + + # All components must be present + for component in required_components: + if component not in content: + return False + + return True + + except Exception as e: + print(f"Error reading file {file_path}: {e}", file=sys.stderr) + return True # Skip files we can't read + +def get_files_to_check() -> List[str]: + """Get files to check from command line args or stdin.""" + if len(sys.argv) > 1: + # Files provided as command line arguments + return sys.argv[1:] + else: + # Read files from stdin + files = [] + for line in sys.stdin: + file_path = line.strip() + if file_path: + files.append(file_path) + return files + +def main(): + """Main function to check license headers.""" + files_to_check = get_files_to_check() + + if not files_to_check: + print("No files to check", file=sys.stderr) + return + + print(f"Checking {len(files_to_check)} files for license headers.") + + violations = [] + + for file_path in files_to_check: + print(f"Checking: {file_path}") + + if not Path(file_path).exists(): + print(f" File not found: {file_path}") + continue + + # Skip if doesn't need header + if not needs_license_header(file_path): + print(f" Skipped (no header needed): {file_path}") + continue + + # Check header + if not check_file_header(file_path): + violations.append(f"- `{file_path}`") + print(f" ❌ Missing license header: {file_path}") + else: + print(f" ✅ Header OK: {file_path}") + + # Output results + if violations: + print(f"\n❌ Found {len(violations)} license header violations:") + + violation_text = '\n'.join(violations) + + # Set output for GitHub Actions + github_output = os.environ.get('GITHUB_OUTPUT') + if github_output: + with open(github_output, 'a') as f: + f.write(f"violations< + comment.user.type === 'Bot' && + comment.body.includes('License Header Violations Found') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } + env: + VIOLATIONS: ${{ steps.license-check.outputs.violations }} \ No newline at end of file From 84a92aa5ea67ff911725e581422a924f9dba2a21 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 11:22:48 -0600 Subject: [PATCH 2/6] Update the comment on successful resolution of headers. Signed-off-by: David Venable --- .github/workflows/license-header-check.yml | 33 +++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml index ed744b0b0e..e49bfea131 100644 --- a/.github/workflows/license-header-check.yml +++ b/.github/workflows/license-header-check.yml @@ -83,4 +83,35 @@ jobs: }); } env: - VIOLATIONS: ${{ steps.license-check.outputs.violations }} \ No newline at end of file + VIOLATIONS: ${{ steps.license-check.outputs.violations }} + + - name: Update PR comment (all violations resolved) + if: success() + uses: actions/github-script@v8 + with: + script: | + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + (comment.body.includes('License Header Violations Found') || comment.body.includes('License Header Check Passed')) + ); + + if (botComment) { + const successBody = [ + '## ✅ License Header Check Passed', + '', + 'All newly added files have proper license headers. Great work! 🎉' + ].join('\n'); + + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: successBody + }); + } \ No newline at end of file From 43b45b2664d651230f5b12b2ccc749800eecad1c Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 10:34:05 -0600 Subject: [PATCH 3/6] Add test files for license header compliance testing - Good files with complete license headers (4 files): * Java: TestProcessor.java * Python: setup-environment.py * YAML: docker-compose.yml * Properties: application.properties - Bad files with missing/incomplete headers (5 files): * Java: NewFeature.java (no header) * Python: monitor.py (short header) * YAML: data-prepper-config.yaml (no header) * Properties: dev.properties (no header) * Shell: compile.sh (incomplete header) These files test the license header compliance system across all supported file types and various violation scenarios. --- config/docker/docker-compose.yml | 18 +++++++++++++++++ config/environments/dev.properties | 5 +++++ .../dataprepper/core/NewFeature.java | 11 ++++++++++ .../plugins/test/TestProcessor.java | 16 +++++++++++++++ .../kubernetes/data-prepper-config.yaml | 13 ++++++++++++ .../trace-analytics/application.properties | 13 ++++++++++++ scripts/compile.sh | 10 ++++++++++ scripts/deployment/setup-environment.py | 20 +++++++++++++++++++ tools/monitoring/monitor.py | 19 ++++++++++++++++++ 9 files changed, 125 insertions(+) create mode 100644 config/docker/docker-compose.yml create mode 100644 config/environments/dev.properties create mode 100644 data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java create mode 100644 data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java create mode 100644 deployment/kubernetes/data-prepper-config.yaml create mode 100644 examples/trace-analytics/application.properties create mode 100755 scripts/compile.sh create mode 100644 scripts/deployment/setup-environment.py create mode 100644 tools/monitoring/monitor.py diff --git a/config/docker/docker-compose.yml b/config/docker/docker-compose.yml new file mode 100644 index 0000000000..ee41342231 --- /dev/null +++ b/config/docker/docker-compose.yml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +version: '3.8' + +services: + data-prepper: + image: opensearchproject/data-prepper:latest + ports: + - "4900:4900" + volumes: + - ./pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml \ No newline at end of file diff --git a/config/environments/dev.properties b/config/environments/dev.properties new file mode 100644 index 0000000000..9035016fe2 --- /dev/null +++ b/config/environments/dev.properties @@ -0,0 +1,5 @@ +# Development environment configuration +environment=development +debug.enabled=true +log.level=DEBUG +data-prepper.workers=2 \ No newline at end of file diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java new file mode 100644 index 0000000000..8a4dde22e0 --- /dev/null +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java @@ -0,0 +1,11 @@ +package org.opensearch.dataprepper.core; + +/** + * This file is missing the license header completely. + */ +public class NewFeature { + + public void doSomething() { + System.out.println("This file has no license header!"); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java b/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java new file mode 100644 index 0000000000..7b08756c87 --- /dev/null +++ b/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugins.test; + +import org.opensearch.dataprepper.model.processor.Processor; + +public class TestProcessor implements Processor { + // This is a good Java file with complete license header +} \ No newline at end of file diff --git a/deployment/kubernetes/data-prepper-config.yaml b/deployment/kubernetes/data-prepper-config.yaml new file mode 100644 index 0000000000..8474569b9c --- /dev/null +++ b/deployment/kubernetes/data-prepper-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: data-prepper-config + namespace: opensearch +data: + pipelines.yaml: | + simple-pipeline: + source: + http: + port: 2021 + sink: + - stdout: \ No newline at end of file diff --git a/examples/trace-analytics/application.properties b/examples/trace-analytics/application.properties new file mode 100644 index 0000000000..5f7233a891 --- /dev/null +++ b/examples/trace-analytics/application.properties @@ -0,0 +1,13 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +# Application configuration for trace analytics example +server.port=8080 +logging.level.org.opensearch.dataprepper=DEBUG +data-prepper.pipeline.workers=4 \ No newline at end of file diff --git a/scripts/compile.sh b/scripts/compile.sh new file mode 100755 index 0000000000..c1211ac776 --- /dev/null +++ b/scripts/compile.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright OpenSearch Contributors + +# This shell script has incomplete license header +set -e + +echo "Starting compilation..." +./gradlew clean build + +echo "Compilation completed successfully!" \ No newline at end of file diff --git a/scripts/deployment/setup-environment.py b/scripts/deployment/setup-environment.py new file mode 100644 index 0000000000..103538f7e8 --- /dev/null +++ b/scripts/deployment/setup-environment.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +""" +Environment setup script for Data Prepper deployment. +""" + +def setup_environment(): + """Set up the deployment environment.""" + print("Setting up environment...") + +if __name__ == "__main__": + setup_environment() \ No newline at end of file diff --git a/tools/monitoring/monitor.py b/tools/monitoring/monitor.py new file mode 100644 index 0000000000..43a569acaa --- /dev/null +++ b/tools/monitoring/monitor.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +""" +This file has only the short header (missing the 3 additional lines). +""" + +import sys +import time + +def monitor_system(): + """Monitor system resources.""" + while True: + print("Monitoring...") + time.sleep(60) + +if __name__ == "__main__": + monitor_system() \ No newline at end of file From 600fefd588eaacf1f09da53ce533e5a382f28578 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 11:14:01 -0600 Subject: [PATCH 4/6] Fix license headers on all test files Added complete license headers to all files that were flagged: - data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java - tools/monitoring/monitor.py - deployment/kubernetes/data-prepper-config.yaml - config/environments/dev.properties - scripts/compile.sh All files now have the complete 5-line license header as required. --- config/environments/dev.properties | 9 +++++++++ .../java/org/opensearch/dataprepper/core/NewFeature.java | 9 +++++++++ deployment/kubernetes/data-prepper-config.yaml | 9 +++++++++ scripts/compile.sh | 7 +++++++ tools/monitoring/monitor.py | 6 ++++++ 5 files changed, 40 insertions(+) diff --git a/config/environments/dev.properties b/config/environments/dev.properties index 9035016fe2..8e67ed03ca 100644 --- a/config/environments/dev.properties +++ b/config/environments/dev.properties @@ -1,3 +1,12 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + # Development environment configuration environment=development debug.enabled=true diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java index 8a4dde22e0..801df84333 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java @@ -1,3 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.dataprepper.core; /** diff --git a/deployment/kubernetes/data-prepper-config.yaml b/deployment/kubernetes/data-prepper-config.yaml index 8474569b9c..d2fd2c9c78 100644 --- a/deployment/kubernetes/data-prepper-config.yaml +++ b/deployment/kubernetes/data-prepper-config.yaml @@ -1,3 +1,12 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + apiVersion: v1 kind: ConfigMap metadata: diff --git a/scripts/compile.sh b/scripts/compile.sh index c1211ac776..91cdaf72d2 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -1,5 +1,12 @@ #!/bin/bash +# # Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# # This shell script has incomplete license header set -e diff --git a/tools/monitoring/monitor.py b/tools/monitoring/monitor.py index 43a569acaa..1df78a6068 100644 --- a/tools/monitoring/monitor.py +++ b/tools/monitoring/monitor.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 +# # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# """ This file has only the short header (missing the 3 additional lines). From 0ca652b6ec077cd192f76c72365d7a2362ff4850 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 11:19:17 -0600 Subject: [PATCH 5/6] One bad license Signed-off-by: David Venable --- config/environments/dev.properties | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/environments/dev.properties b/config/environments/dev.properties index 8e67ed03ca..c269bb0986 100644 --- a/config/environments/dev.properties +++ b/config/environments/dev.properties @@ -2,10 +2,6 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 # -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# # Development environment configuration environment=development From d69e804467e026df36a12ad63ae5ec34c3db7349 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 6 Jan 2026 11:30:25 -0600 Subject: [PATCH 6/6] Fix last copyright Signed-off-by: David Venable --- config/environments/dev.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/environments/dev.properties b/config/environments/dev.properties index c269bb0986..8e67ed03ca 100644 --- a/config/environments/dev.properties +++ b/config/environments/dev.properties @@ -2,6 +2,10 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 # +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# # Development environment configuration environment=development