diff --git a/.github/scripts/check-license-headers.py b/.github/scripts/check-license-headers.py new file mode 100755 index 0000000000..b46ebbd114 --- /dev/null +++ b/.github/scripts/check-license-headers.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +""" +License Header Compliance Checker for OpenSearch Data Prepper + +This script checks that files contain the required license headers +as specified in CONTRIBUTING.md. + +Usage: + python check-license-headers.py file1.java file2.py ... + echo "file1.java\nfile2.py" | python check-license-headers.py +""" + +import os +import sys +from pathlib import Path +from typing import List + +# File extensions that require license headers +SUPPORTED_EXTENSIONS = { + '.java', '.groovy', '.gradle', # Java ecosystem + '.py', # Python + '.sh', '.bash', '.zsh', # Shell scripts + '.yaml', '.yml', # YAML files + '.properties', # Properties files +} + +def needs_license_header(file_path: str) -> bool: + """Check if a file needs a license header based on its extension.""" + path = Path(file_path) + return path.suffix.lower() in SUPPORTED_EXTENSIONS + +def check_file_header(file_path: str) -> bool: + """Check if a file has the required complete license header.""" + if not Path(file_path).exists(): + return True + + try: + with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: + # Read first 15 lines to check for license header + lines = [] + for i, line in enumerate(f): + if i >= 15: # Only check first 15 lines + break + lines.append(line) + + content = ''.join(lines) + + # Check for all 5 required license header components + required_components = [ + 'Copyright OpenSearch Contributors', + 'SPDX-License-Identifier: Apache-2.0', + 'The OpenSearch Contributors require contributions made to', + 'this file be licensed under the Apache-2.0 license or a', + 'compatible open source license.' + ] + + # All components must be present + for component in required_components: + if component not in content: + return False + + return True + + except Exception as e: + print(f"Error reading file {file_path}: {e}", file=sys.stderr) + return True # Skip files we can't read + +def get_files_to_check() -> List[str]: + """Get files to check from command line args or stdin.""" + if len(sys.argv) > 1: + # Files provided as command line arguments + return sys.argv[1:] + else: + # Read files from stdin + files = [] + for line in sys.stdin: + file_path = line.strip() + if file_path: + files.append(file_path) + return files + +def main(): + """Main function to check license headers.""" + files_to_check = get_files_to_check() + + if not files_to_check: + print("No files to check", file=sys.stderr) + return + + print(f"Checking {len(files_to_check)} files for license headers.") + + violations = [] + + for file_path in files_to_check: + print(f"Checking: {file_path}") + + if not Path(file_path).exists(): + print(f" File not found: {file_path}") + continue + + # Skip if doesn't need header + if not needs_license_header(file_path): + print(f" Skipped (no header needed): {file_path}") + continue + + # Check header + if not check_file_header(file_path): + violations.append(f"- `{file_path}`") + print(f" ❌ Missing license header: {file_path}") + else: + print(f" ✅ Header OK: {file_path}") + + # Output results + if violations: + print(f"\n❌ Found {len(violations)} license header violations:") + + violation_text = '\n'.join(violations) + + # Set output for GitHub Actions + github_output = os.environ.get('GITHUB_OUTPUT') + if github_output: + with open(github_output, 'a') as f: + f.write(f"violations< + comment.user.type === 'Bot' && + comment.body.includes('License Header Violations Found') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } + env: + VIOLATIONS: ${{ steps.license-check.outputs.violations }} + + - name: Update PR comment (all violations resolved) + if: success() + uses: actions/github-script@v8 + with: + script: | + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + (comment.body.includes('License Header Violations Found') || comment.body.includes('License Header Check Passed')) + ); + + if (botComment) { + const successBody = [ + '## ✅ License Header Check Passed', + '', + 'All newly added files have proper license headers. Great work! 🎉' + ].join('\n'); + + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: successBody + }); + } \ No newline at end of file diff --git a/config/docker/docker-compose.yml b/config/docker/docker-compose.yml new file mode 100644 index 0000000000..ee41342231 --- /dev/null +++ b/config/docker/docker-compose.yml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +version: '3.8' + +services: + data-prepper: + image: opensearchproject/data-prepper:latest + ports: + - "4900:4900" + volumes: + - ./pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml \ No newline at end of file diff --git a/config/environments/dev.properties b/config/environments/dev.properties new file mode 100644 index 0000000000..8e67ed03ca --- /dev/null +++ b/config/environments/dev.properties @@ -0,0 +1,14 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +# Development environment configuration +environment=development +debug.enabled=true +log.level=DEBUG +data-prepper.workers=2 \ No newline at end of file diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java new file mode 100644 index 0000000000..801df84333 --- /dev/null +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/NewFeature.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.core; + +/** + * This file is missing the license header completely. + */ +public class NewFeature { + + public void doSomething() { + System.out.println("This file has no license header!"); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java b/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java new file mode 100644 index 0000000000..7b08756c87 --- /dev/null +++ b/data-prepper-plugins/test-plugin/src/main/java/org/opensearch/dataprepper/plugins/test/TestProcessor.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugins.test; + +import org.opensearch.dataprepper.model.processor.Processor; + +public class TestProcessor implements Processor { + // This is a good Java file with complete license header +} \ No newline at end of file diff --git a/deployment/kubernetes/data-prepper-config.yaml b/deployment/kubernetes/data-prepper-config.yaml new file mode 100644 index 0000000000..d2fd2c9c78 --- /dev/null +++ b/deployment/kubernetes/data-prepper-config.yaml @@ -0,0 +1,22 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: data-prepper-config + namespace: opensearch +data: + pipelines.yaml: | + simple-pipeline: + source: + http: + port: 2021 + sink: + - stdout: \ No newline at end of file diff --git a/examples/trace-analytics/application.properties b/examples/trace-analytics/application.properties new file mode 100644 index 0000000000..5f7233a891 --- /dev/null +++ b/examples/trace-analytics/application.properties @@ -0,0 +1,13 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +# Application configuration for trace analytics example +server.port=8080 +logging.level.org.opensearch.dataprepper=DEBUG +data-prepper.pipeline.workers=4 \ No newline at end of file diff --git a/scripts/compile.sh b/scripts/compile.sh new file mode 100755 index 0000000000..91cdaf72d2 --- /dev/null +++ b/scripts/compile.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +# This shell script has incomplete license header +set -e + +echo "Starting compilation..." +./gradlew clean build + +echo "Compilation completed successfully!" \ No newline at end of file diff --git a/scripts/deployment/setup-environment.py b/scripts/deployment/setup-environment.py new file mode 100644 index 0000000000..103538f7e8 --- /dev/null +++ b/scripts/deployment/setup-environment.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +""" +Environment setup script for Data Prepper deployment. +""" + +def setup_environment(): + """Set up the deployment environment.""" + print("Setting up environment...") + +if __name__ == "__main__": + setup_environment() \ No newline at end of file diff --git a/tools/monitoring/monitor.py b/tools/monitoring/monitor.py new file mode 100644 index 0000000000..1df78a6068 --- /dev/null +++ b/tools/monitoring/monitor.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +""" +This file has only the short header (missing the 3 additional lines). +""" + +import sys +import time + +def monitor_system(): + """Monitor system resources.""" + while True: + print("Monitoring...") + time.sleep(60) + +if __name__ == "__main__": + monitor_system() \ No newline at end of file