Add Helm hook annotations to RBAC resources #109
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Helm Chart CI/CD | |
| on: | |
| push: | |
| branches: | |
| - '**' | |
| tags: | |
| - 'v*' | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: harbor.lag0.com.br # Change this to your OCI registry | |
| CHART_NAME: headplane | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-cases: ${{ steps.get-test-cases.outputs.test-cases }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get test cases | |
| id: get-test-cases | |
| run: | | |
| test_cases=$(find test/test-cases -name "*.yaml" -type f | sort | jq -R -s -c 'split("\n")[:-1]') | |
| echo "test-cases=$test_cases" >> $GITHUB_OUTPUT | |
| echo "Found test cases: $test_cases" | |
| test: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| test-file: ${{ fromJson(needs.setup.outputs.test-cases) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: v3.12.3 | |
| - name: Install yq | |
| run: | | |
| echo "=== Installing yq ===" | |
| wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| chmod +x /usr/local/bin/yq | |
| echo "✓ yq installed successfully " | |
| - name: Install k3s | |
| run: | | |
| echo "=== Installing k3s ===" | |
| curl -sfL https://get.k3s.io | sh - | |
| sudo chmod 644 /etc/rancher/k3s/k3s.yaml | |
| mkdir -p ~/.kube | |
| sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config | |
| sudo chown $USER:$USER ~/.kube/config | |
| export KUBECONFIG=~/.kube/config | |
| echo "✓ k3s installed successfully" | |
| - name: Wait for k3s to be ready | |
| run: | | |
| echo "=== Waiting for k3s to be ready ===" | |
| timeout 60s bash -c 'until kubectl get nodes; do sleep 2; done' | |
| echo "✓ k3s is ready" | |
| - name: Create test directories | |
| run: | | |
| echo "=== Creating test directories ===" | |
| mkdir -p test/output test/logs | |
| echo "✓ Test directories created successfully" | |
| - name: Update Helm dependencies | |
| run: | | |
| if [ ! "$(ls -A charts)" ]; then | |
| echo "=== Updating Helm dependencies ===" | |
| helm dependency update . | |
| echo "✓ Helm dependencies updated successfully" | |
| else | |
| echo "=== Using existing dependencies ===" | |
| echo "✓ Using existing dependencies" | |
| fi | |
| - name: Copy test templates | |
| run: | | |
| echo "=== Copying test templates ===" | |
| if [ -d "test/templates" ]; then | |
| cp -r test/templates/* templates/ | |
| echo "✓ Test templates copied successfully" | |
| else | |
| echo "No test templates directory found" | |
| fi | |
| - name: Generate template | |
| id: generate | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| echo "=== Generating template for: $test_name ===" | |
| helm template rybbit . -f "${{ matrix.test-file }}" > "test/output/${test_name}-output.yaml" | |
| echo "✓ Template generated successfully for $test_name" | |
| - name: Validate YAML syntax | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| echo "=== Validating YAML for: $test_name ===" | |
| if ! yq eval '.' "test/output/${test_name}-output.yaml" > /dev/null 2>&1; then | |
| echo "Invalid YAML syntax in ${test_name}-output.yaml" | |
| yq eval '.' "test/output/${test_name}-output.yaml" | |
| exit 1 | |
| fi | |
| echo "✓ YAML syntax validated successfully for $test_name" | |
| - name: Check for unrendered templates | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| echo "=== Checking for unrendered templates in: $test_name ===" | |
| if grep -q "{{.*}}" "test/output/${test_name}-output.yaml"; then | |
| echo "Found unrendered template variables in ${test_name}-output.yaml" | |
| cat "test/output/${test_name}-output.yaml" | |
| exit 1 | |
| fi | |
| echo "✓ No unrendered templates found in $test_name" | |
| - name: Validate Kubernetes resources | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| echo "=== Validating Kubernetes resources for: $test_name ===" | |
| # Check if the test file contains ServiceMonitor resources | |
| if grep -q "kind: ServiceMonitor" "test/output/${test_name}-output.yaml"; then | |
| # Check if Prometheus Operator CRDs are installed | |
| if ! kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1; then | |
| echo "Skipping ServiceMonitor validation as Prometheus Operator CRDs are not installed" | |
| # Remove ServiceMonitor resources temporarily for validation | |
| yq eval 'select(.kind != "ServiceMonitor")' "test/output/${test_name}-output.yaml" > "test/output/${test_name}-output-temp.yaml" | |
| # Validate remaining resources | |
| if [ -s "test/output/${test_name}-output-temp.yaml" ]; then | |
| kubectl apply --dry-run=client -f "test/output/${test_name}-output-temp.yaml" | |
| else | |
| echo "No resources to validate after removing ServiceMonitor resources" | |
| fi | |
| rm "test/output/${test_name}-output-temp.yaml" | |
| else | |
| kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" | |
| fi | |
| else | |
| kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" | |
| fi | |
| echo "✓ Kubernetes resources validated successfully for $test_name" | |
| - name: Check required resources | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| echo "=== Checking required resources for: $test_name ===" | |
| if ! grep -q "kind: Deployment\|kind: Service\|kind: ConfigMap\|kind: Secret" "test/output/${test_name}-output.yaml"; then | |
| echo "Missing required Kubernetes resources in ${test_name}-output.yaml" | |
| exit 1 | |
| fi | |
| echo "✓ Required resources found in $test_name" | |
| - name: Deploy and test resources | |
| run: | | |
| test_name=$(basename "${{ matrix.test-file }}" .yaml) | |
| namespace="test-${test_name}-$(date +%m%d%H%M)" | |
| namespace=$(echo "$namespace" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | cut -c 1-63 | sed 's/-$//') | |
| echo "=== Testing $test_name in namespace: $namespace ===" | |
| kubectl create namespace "$namespace" | |
| # Apply resources and monitor status | |
| echo "Applying resources..." | |
| helm template rybbit . -f "${{ matrix.test-file }}" --namespace "$namespace" | kubectl apply -f - -n "$namespace" | |
| # Give resources time to start scheduling | |
| echo "Waiting for resources to start scheduling..." | |
| sleep 3 | |
| # Immediate check of events and pod status | |
| echo "=== Initial Status Check ===" | |
| echo "Deployments:" | |
| kubectl get deployments -n "$namespace" -o wide | |
| echo -e "\nJobs:" | |
| kubectl get jobs -n "$namespace" -o wide | |
| echo -e "\nPods:" | |
| kubectl get pods -n "$namespace" -o wide | |
| echo -e "\nRecent Events:" | |
| kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 | |
| # Monitor deployments and jobs with continuous updates | |
| echo -e "\nMonitoring deployments and jobs..." | |
| start_time=$(date +%s) | |
| timeout_seconds=60 | |
| while true; do | |
| current_time=$(date +%s) | |
| elapsed=$((current_time - start_time)) | |
| remaining=$((timeout_seconds - elapsed)) | |
| if [ $elapsed -ge $timeout_seconds ]; then | |
| echo "[$test_name] Timeout reached (${timeout_seconds}s). Test FAILED!" | |
| echo "[$test_name] Final status before timeout:" | |
| kubectl get deployments,jobs,pods -n "$namespace" -o wide | |
| echo "[$test_name] Recent events:" | |
| kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 10 | |
| exit 1 | |
| fi | |
| echo "[$test_name] === Current Status (${elapsed}s elapsed, ${remaining}s remaining) ===" | |
| kubectl get deployments,jobs,pods -n "$namespace" -o wide | |
| # Check if all deployments are ready | |
| deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) | |
| deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) | |
| # Check if all jobs are completed | |
| job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) | |
| job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) | |
| # If no jobs exist, set job counts to 0 | |
| if [ "$job_total_count" -eq 0 ]; then | |
| job_completed_count=0 | |
| job_total_count=0 | |
| fi | |
| echo "[$test_name] Status: Deployments $deployment_ready_count/$deployment_total_count ready, Jobs $job_completed_count/$job_total_count completed" | |
| # Proactively capture logs for pods that might be having issues | |
| for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do | |
| pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') | |
| # Only capture logs from pods that are running or have failed | |
| if [ "$pod_status" = "Running" ] || [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ]; then | |
| echo "[$test_name] Logs for $pod (status: $pod_status):" | |
| kubectl logs -n "$namespace" "$pod" --all-containers --tail=5 || echo "No logs available for $pod" | |
| elif [ "$pod_status" = "Pending" ]; then | |
| # For pending pods, just show the status without trying to get logs | |
| echo "[$test_name] Pod $pod is still pending - waiting for container to start..." | |
| fi | |
| done | |
| # Check if all deployments are ready AND all jobs are completed | |
| deployments_ready=false | |
| jobs_completed=false | |
| # Check if deployments are ready (if any exist) | |
| if [ "$deployment_total_count" -eq 0 ]; then | |
| deployments_ready=true | |
| elif [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ]; then | |
| deployments_ready=true | |
| fi | |
| # Check if jobs are completed (if any exist) | |
| if [ "$job_total_count" -eq 0 ]; then | |
| jobs_completed=true | |
| elif [ "$job_completed_count" -eq "$job_total_count" ] && [ "$job_total_count" -gt 0 ]; then | |
| jobs_completed=true | |
| fi | |
| # Additional check: ensure all containers in running pods are ready | |
| containers_ready=true | |
| for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}'); do | |
| ready_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[?(@.ready==true)].name}' | wc -w) | |
| total_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[*].name}' | wc -w) | |
| if [ "$ready_containers" -ne "$total_containers" ]; then | |
| containers_ready=false | |
| echo "[$test_name] Pod $pod has $ready_containers/$total_containers containers ready" | |
| break | |
| fi | |
| done | |
| if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then | |
| echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" | |
| # # Verify API key replacement if OIDC is enabled | |
| # if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then | |
| # CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) | |
| # # Only verify API key replacement if OIDC is enabled | |
| # if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then | |
| # echo "[$test_name] Verifying API key replacement in headplane-config secret..." | |
| # if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then | |
| # API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') | |
| # if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then | |
| # echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." | |
| # else | |
| # echo "[$test_name] ❌ API key replacement verification failed" | |
| # echo "[$test_name] Current API key: $API_KEY" | |
| # echo "[$test_name] Expected: Real API token (not placeholder)" | |
| # exit 1 | |
| # fi | |
| # else | |
| # echo "[$test_name] ❌ No headscale_api_key found in config" | |
| # exit 1 | |
| # fi | |
| # else | |
| # echo "[$test_name] OIDC not enabled, skipping API key verification" | |
| # fi | |
| # fi | |
| break | |
| fi | |
| # Check for any failed pods | |
| if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then | |
| echo "[$test_name] Found failed or terminating pods. Checking details..." | |
| kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating|Failed" | |
| echo "[$test_name] Recent events:" | |
| kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" | tail -5 | |
| echo "[$test_name] Pod logs:" | |
| for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do | |
| echo "[$test_name] --- $pod logs ---" | |
| kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" | |
| done | |
| exit 1 | |
| fi | |
| # Check for any pods not in Running state | |
| non_running_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}') | |
| if [ -n "$non_running_pods" ]; then | |
| should_fail=false | |
| for pod in $non_running_pods; do | |
| pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') | |
| # Only fail if the pod is in a failed state, not if it's still pending or succeeded | |
| if [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ] || [ "$pod_status" = "CrashLoopBackOff" ]; then | |
| should_fail=true | |
| echo "[$test_name] Pod $pod has failed status: $pod_status" | |
| kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" | |
| elif [ "$pod_status" = "Pending" ]; then | |
| echo "[$test_name] Pod $pod is still pending - normal during startup" | |
| elif [ "$pod_status" = "Running" ]; then | |
| echo "[$test_name] Pod $pod has succeeded - this is expected for unfinished jobs" | |
| elif [ "$pod_status" = "Succeeded" ]; then | |
| echo "[$test_name] Pod $pod has succeeded - this is expected for completed jobs" | |
| else | |
| echo "[$test_name] Pod $pod has unexpected status: $pod_status" | |
| should_fail=true | |
| fi | |
| done | |
| if [ "$should_fail" = true ]; then | |
| exit 1 | |
| fi | |
| fi | |
| # Check for failed jobs | |
| failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') | |
| if [ -n "$failed_jobs" ]; then | |
| echo "[$test_name] Found failed jobs: $failed_jobs" | |
| for job in $failed_jobs; do | |
| echo "[$test_name] Job $job pod logs:" | |
| job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') | |
| for pod in $job_pods; do | |
| echo "[$test_name] --- $pod logs ---" | |
| kubectl logs -n "$namespace" "$pod" --tail=10 || echo "No logs available" | |
| done | |
| done | |
| exit 1 | |
| fi | |
| sleep 10 | |
| done | |
| # Cleanup with force delete | |
| echo "=== Cleaning up ===" | |
| kubectl delete namespace "$namespace" --force --grace-period=0 | |
| echo "✓ Cleanup completed for $test_name" | |
| publish: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| needs: [setup, test] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: set version | |
| run: | | |
| ref=${GITHUB_REF##*/} | |
| echo "VERSION=${ref#v}" >> $GITHUB_ENV | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: v3.12.3 | |
| - name: Login to OCI Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ secrets.REGISTRY_USERNAME }} | |
| password: ${{ secrets.REGISTRY_PASSWORD }} | |
| - name: Print env.VERSION | |
| run: | | |
| echo "env.VERSION: ${{ env.VERSION }}" | |
| - name: Package and Push Helm Chart | |
| run: | | |
| echo "=== Updating dependencies ===" | |
| helm dependency update . | |
| echo "✓ Dependencies updated successfully" | |
| echo "=== Packaging chart ===" | |
| helm package --version ${{ env.VERSION }} . | |
| echo "✓ Chart packaged successfully" | |
| echo "=== Pushing to OCI registry ===" | |
| helm push headplane-*.tgz oci://${{ env.REGISTRY }}/library |