From 4074ad1d0bc0ae90338825b006ad401f2892c894 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:03:39 -0300 Subject: [PATCH 01/46] Add test/test-cases --- .github/workflows/publish-helm-chart.yml | 195 +++++++++++++++++++++++ test/test-cases/00-lag0.yaml | 82 ++++++++++ test/test-cases/01-definetelynobody.yaml | 23 +++ 3 files changed, 300 insertions(+) create mode 100644 test/test-cases/00-lag0.yaml create mode 100644 test/test-cases/01-definetelynobody.yaml diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 236eeb8..200b705 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -2,6 +2,9 @@ name: Helm Chart CI/CD on: push: + branches: + - main + - staging tags: - 'v*' workflow_dispatch: @@ -11,6 +14,198 @@ env: CHART_NAME: headplane jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.12.3 + + - name: Install yq + run: | + echo "=== Installing yq ===" + wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod +x /usr/local/bin/yq + echo "✓ yq installed successfully " + + - name: Install k3s + run: | + echo "=== Installing k3s ===" + curl -sfL https://get.k3s.io | sh - + sudo chmod 644 /etc/rancher/k3s/k3s.yaml + mkdir -p ~/.kube + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo chown $USER:$USER ~/.kube/config + export KUBECONFIG=~/.kube/config + echo "✓ k3s installed successfully" + + - name: Wait for k3s to be ready + run: | + echo "=== Waiting for k3s to be ready ===" + timeout 60s bash -c 'until kubectl get nodes; do sleep 2; done' + echo "✓ k3s is ready" + + - name: Create test directories + run: | + echo "=== Creating test directories ===" + mkdir -p test/output test/logs + echo "✓ Test directories created successfully" + + - name: Update Helm dependencies + run: | + if [ ! "$(ls -A charts)" ]; then + echo "=== Updating Helm dependencies ===" + helm dependency update . + echo "✓ Helm dependencies updated successfully" + else + echo "=== Using existing dependencies ===" + echo "✓ Using existing dependencies" + fi + + - name: Generate templates + id: generate + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + echo "=== Generating template for: $test_name ===" + helm template rybbit . -f "$test_file" > "test/output/${test_name}-output.yaml" + echo "✓ Template generated successfully for $test_name" + done + + - name: Validate YAML syntax + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + echo "=== Validating YAML for: $test_name ===" + if ! yq eval '.' "test/output/${test_name}-output.yaml" > /dev/null 2>&1; then + echo "Invalid YAML syntax in ${test_name}-output.yaml" + yq eval '.' "test/output/${test_name}-output.yaml" + exit 1 + fi + echo "✓ YAML syntax validated successfully for $test_name" + done + + - name: Check for unrendered templates + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + echo "=== Checking for unrendered templates in: $test_name ===" + if grep -q "{{.*}}" "test/output/${test_name}-output.yaml"; then + echo "Found unrendered template variables in ${test_name}-output.yaml" + cat "test/output/${test_name}-output.yaml" + exit 1 + fi + echo "✓ No unrendered templates found in $test_name" + done + + - name: Validate Kubernetes resources + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + echo "=== Validating Kubernetes resources for: $test_name ===" + + # Check if the test file contains ServiceMonitor resources + if grep -q "kind: ServiceMonitor" "test/output/${test_name}-output.yaml"; then + # Check if Prometheus Operator CRDs are installed + if ! kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1; then + echo "Skipping ServiceMonitor validation as Prometheus Operator CRDs are not installed" + # Remove ServiceMonitor resources temporarily for validation + yq eval 'select(.kind != "ServiceMonitor")' "test/output/${test_name}-output.yaml" > "test/output/${test_name}-output-temp.yaml" + # Validate remaining resources + if [ -s "test/output/${test_name}-output-temp.yaml" ]; then + kubectl apply --dry-run=client -f "test/output/${test_name}-output-temp.yaml" + else + echo "No resources to validate after removing ServiceMonitor resources" + fi + rm "test/output/${test_name}-output-temp.yaml" + else + kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" + fi + else + kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" + fi + echo "✓ Kubernetes resources validated successfully for $test_name" + done + + - name: Check required resources + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + echo "=== Checking required resources for: $test_name ===" + if ! grep -q "kind: Deployment\|kind: Service\|kind: ConfigMap\|kind: Secret" "test/output/${test_name}-output.yaml"; then + echo "Missing required Kubernetes resources in ${test_name}-output.yaml" + exit 1 + fi + echo "✓ Required resources found in $test_name" + done + + - name: Deploy and test resources + run: | + for test_file in test/test-cases/*.yaml; do + test_name=$(basename "$test_file" .yaml) + namespace="test-${test_name}-$(date +%m%d%H%M)" + namespace=$(echo "$namespace" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | cut -c 1-63 | sed 's/-$//') + + echo "=== Testing $test_name in namespace: $namespace ===" + kubectl create namespace "$namespace" + + # Apply resources and monitor status + echo "Applying resources..." + helm template rybbit . -f "$test_file" --namespace "$namespace" | kubectl apply -f - -n "$namespace" + + # Immediate check of events and pod status + echo "=== Initial Status Check ===" + echo "Deployments:" + kubectl get deployments -n "$namespace" -o wide + + echo -e "\nPods:" + kubectl get pods -n "$namespace" -o wide + + echo -e "\nRecent Events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 + + # Monitor deployments with continuous updates + echo -e "\nMonitoring deployments..." + while true; do + echo "=== Current Status ===" + kubectl get deployments,pods -n "$namespace" -o wide + + # Check if all deployments are ready + ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + + if [ "$ready_count" -eq "$total_count" ] && [ "$total_count" -gt 0 ]; then + echo "All deployments are ready!" + break + fi + + # Check for any failed pods + if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating"; then + echo "Found failed or terminating pods. Checking details..." + kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" + echo -e "\nPod Events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" + echo -e "\nPod Details:" + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do + echo -e "\n=== $pod ===" + kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" + done + exit 1 + fi + + sleep 5 + done + + # Cleanup with force delete + echo "=== Cleaning up ===" + kubectl delete namespace "$namespace" --force --grace-period=0 + echo "✓ Cleanup completed for $test_name" + done + publish: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest diff --git a/test/test-cases/00-lag0.yaml b/test/test-cases/00-lag0.yaml new file mode 100644 index 0000000..8af843e --- /dev/null +++ b/test/test-cases/00-lag0.yaml @@ -0,0 +1,82 @@ +headplane: + image: ghcr.io/tale/headplane:0.6.0 + config: + server: + host: "0.0.0.0" + port: 3000 + cookie_secure: true + headscale: + url: "https://vpn.lag0.com.br" + config_path: "/etc/headscale/config.yaml" + config_strict: "true" + integration: + kubernetes: + enabled: true + validate_manifest: false + pod_name: "headplane-0" + secret: + name: headplane-secret + create: true +headscale: + image: headscale/headscale:0.26.0 + config: + server_url: https://headscale.lag0.com.br + listen_addr: 0.0.0.0:8080 + metrics_listen_addr: 0.0.0.0:9090 + grpc_listen_addr: 0.0.0.0:50443 + grpc_allow_insecure: false + policy: + mode: database + prefixes: + v4: 100.64.0.0/10 + v6: fd7a:115c:a1e0::/48 + allocation: sequential + database: + type: sqlite + debug: false + sqlite: + path: /etc/headscale/db.sqlite + noise: + private_key_path: /etc/headscale/noise_private.key + derp: + server: + enabled: true + region_id: 999 + region_code: "headscale" + region_name: "Headscale Embedded DERP" + stun_listen_addr: "0.0.0.0:3478" + private_key_path: /var/lib/headscale/derp_server_private.key + automatically_add_embedded_derp_region: true + ipv4: 1.2.3.4 + ipv6: 2001:db8::1 + urls: + - https://controlplane.tailscale.com/derpmap/default + paths: [] + dns: + magic_dns: true + base_domain: clients.lag0.com.br + nameservers: + global: + - 1.1.1.1 + - 8.8.8.8 +relay: + enabled: false +pvc: + enabled: true + name: headscale-config + accessModes: + - ReadWriteOnce + storage: 1Gi + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +# storageClassName: default + +ingress: + enabled: false + className: nginx + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-production" + labels: [] + headplaneDomain: "headscale.lag0.com.br" + headscaleDomain: "vpn.lag0.com.br" + tlsSecretName: "headplane-tls" diff --git a/test/test-cases/01-definetelynobody.yaml b/test/test-cases/01-definetelynobody.yaml new file mode 100644 index 0000000..8a5fcc7 --- /dev/null +++ b/test/test-cases/01-definetelynobody.yaml @@ -0,0 +1,23 @@ +headplane: + config: + headscale: + url: "https://vpn.test.example.com" + secret: + server: + cookie_secret: "test-cookie-secret-12345" + oidc: + client_id: "test-headplane-client-id" + enabled: true + issuer: "https://auth.test.example.com" + redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" +headscale: + config: + server_url: "https://vpn.test.example.com" + dns: + base_domain: "test.vpn" + oidc: + client_id: "test-headscale-client-id" + enabled: true + issuer: "https://auth.test.example.com" +pvc: + storageClassName: "test-class-sc" \ No newline at end of file From ec7a42843e93b541524795b8174cb746f97f1b09 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:10:33 -0300 Subject: [PATCH 02/46] add jobs logic to gha test --- .github/workflows/publish-helm-chart.yml | 48 ++++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 200b705..47c541f 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -162,24 +162,41 @@ jobs: echo "Deployments:" kubectl get deployments -n "$namespace" -o wide + echo -e "\nJobs:" + kubectl get jobs -n "$namespace" -o wide + echo -e "\nPods:" kubectl get pods -n "$namespace" -o wide echo -e "\nRecent Events:" kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 - # Monitor deployments with continuous updates - echo -e "\nMonitoring deployments..." + # Monitor deployments and jobs with continuous updates + echo -e "\nMonitoring deployments and jobs..." while true; do echo "=== Current Status ===" - kubectl get deployments,pods -n "$namespace" -o wide + kubectl get deployments,jobs,pods -n "$namespace" -o wide # Check if all deployments are ready - ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + + # Check if all jobs are completed + job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) + job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) + + # If no jobs exist, set job counts to 0 + if [ "$job_total_count" -eq 0 ]; then + job_completed_count=0 + job_total_count=0 + fi + + echo "Deployments: $deployment_ready_count/$deployment_total_count ready" + echo "Jobs: $job_completed_count/$job_total_count completed" - if [ "$ready_count" -eq "$total_count" ] && [ "$total_count" -gt 0 ]; then - echo "All deployments are ready!" + # Check if all deployments are ready AND all jobs are completed + if [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ] && [ "$job_completed_count" -eq "$job_total_count" ]; then + echo "All deployments are ready and all jobs are completed!" break fi @@ -197,6 +214,23 @@ jobs: exit 1 fi + # Check for failed jobs + failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') + if [ -n "$failed_jobs" ]; then + echo "Found failed jobs: $failed_jobs" + for job in $failed_jobs; do + echo -e "\n=== Job $job details ===" + kubectl describe job -n "$namespace" "$job" + echo -e "\n=== Job $job pod logs ===" + job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') + for pod in $job_pods; do + echo "--- Pod $pod logs ---" + kubectl logs -n "$namespace" "$pod" || echo "No logs available" + done + done + exit 1 + fi + sleep 5 done From a9838f187a33343b26e7d55a4b135e5a78df4e16 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:16:59 -0300 Subject: [PATCH 03/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 90 +++--------------------- 1 file changed, 10 insertions(+), 80 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 47c541f..f278e66 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -153,91 +153,21 @@ jobs: echo "=== Testing $test_name in namespace: $namespace ===" kubectl create namespace "$namespace" - # Apply resources and monitor status - echo "Applying resources..." + # Apply resources helm template rybbit . -f "$test_file" --namespace "$namespace" | kubectl apply -f - -n "$namespace" - # Immediate check of events and pod status - echo "=== Initial Status Check ===" - echo "Deployments:" - kubectl get deployments -n "$namespace" -o wide + # Wait for resources to be ready + echo "Waiting for resources to be ready..." + kubectl wait --for=condition=available --timeout=40s deployment --all -n "$namespace" 2>/dev/null || true + kubectl wait --for=condition=complete --timeout=40s job --all -n "$namespace" 2>/dev/null || true - echo -e "\nJobs:" - kubectl get jobs -n "$namespace" -o wide + # Quick status check + echo "Final status:" + kubectl get deployments,jobs,pods -n "$namespace" -o wide - echo -e "\nPods:" - kubectl get pods -n "$namespace" -o wide - - echo -e "\nRecent Events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 - - # Monitor deployments and jobs with continuous updates - echo -e "\nMonitoring deployments and jobs..." - while true; do - echo "=== Current Status ===" - kubectl get deployments,jobs,pods -n "$namespace" -o wide - - # Check if all deployments are ready - deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - - # Check if all jobs are completed - job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) - job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) - - # If no jobs exist, set job counts to 0 - if [ "$job_total_count" -eq 0 ]; then - job_completed_count=0 - job_total_count=0 - fi - - echo "Deployments: $deployment_ready_count/$deployment_total_count ready" - echo "Jobs: $job_completed_count/$job_total_count completed" - - # Check if all deployments are ready AND all jobs are completed - if [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ] && [ "$job_completed_count" -eq "$job_total_count" ]; then - echo "All deployments are ready and all jobs are completed!" - break - fi - - # Check for any failed pods - if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating"; then - echo "Found failed or terminating pods. Checking details..." - kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" - echo -e "\nPod Events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" - echo -e "\nPod Details:" - for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do - echo -e "\n=== $pod ===" - kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" - done - exit 1 - fi - - # Check for failed jobs - failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') - if [ -n "$failed_jobs" ]; then - echo "Found failed jobs: $failed_jobs" - for job in $failed_jobs; do - echo -e "\n=== Job $job details ===" - kubectl describe job -n "$namespace" "$job" - echo -e "\n=== Job $job pod logs ===" - job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') - for pod in $job_pods; do - echo "--- Pod $pod logs ---" - kubectl logs -n "$namespace" "$pod" || echo "No logs available" - done - done - exit 1 - fi - - sleep 5 - done - - # Cleanup with force delete - echo "=== Cleaning up ===" + # Cleanup kubectl delete namespace "$namespace" --force --grace-period=0 - echo "✓ Cleanup completed for $test_name" + echo "✓ Test completed for $test_name" done publish: From c69d3f67ba252ad35e4bca93360ad74f788349e5 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:43:23 -0300 Subject: [PATCH 04/46] add AzSiAz test case --- .../{00-lag0.yaml => 00-lag0-test-case.yaml} | 0 ...aml => 01-definetelynobody-test-case.yaml} | 0 test/test-cases/02-AzSiAz-test-case.yaml | 27 +++++++++++++++++++ 3 files changed, 27 insertions(+) rename test/test-cases/{00-lag0.yaml => 00-lag0-test-case.yaml} (100%) rename test/test-cases/{01-definetelynobody.yaml => 01-definetelynobody-test-case.yaml} (100%) create mode 100644 test/test-cases/02-AzSiAz-test-case.yaml diff --git a/test/test-cases/00-lag0.yaml b/test/test-cases/00-lag0-test-case.yaml similarity index 100% rename from test/test-cases/00-lag0.yaml rename to test/test-cases/00-lag0-test-case.yaml diff --git a/test/test-cases/01-definetelynobody.yaml b/test/test-cases/01-definetelynobody-test-case.yaml similarity index 100% rename from test/test-cases/01-definetelynobody.yaml rename to test/test-cases/01-definetelynobody-test-case.yaml diff --git a/test/test-cases/02-AzSiAz-test-case.yaml b/test/test-cases/02-AzSiAz-test-case.yaml new file mode 100644 index 0000000..3006246 --- /dev/null +++ b/test/test-cases/02-AzSiAz-test-case.yaml @@ -0,0 +1,27 @@ +# Test case for PR #3 +headplane: + config: + headscale: + url: "https://vpn.test.example.com" + secret: + server: + cookie_secret: "test-cookie-secret-12345" + oidc: + client_id: "test-headplane-client-id" + enabled: true + issuer: "https://auth.test.example.com" + redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" +headscale: + config: + server_url: "https://vpn.test.example.com" + dns: + base_domain: "test.vpn" + oidc: + pkce: + enabled: true + method: S256 + client_id: "test-headscale-client-id" + enabled: true + issuer: "https://auth.test.example.com" +pvc: + storageClassName: "test-class-sc" \ No newline at end of file From 45a5955c67379ec4e758825c59c5aa1d8cfbd64f Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:46:36 -0300 Subject: [PATCH 05/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 107 ++++++++++++++++++++--- 1 file changed, 97 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index f278e66..406bb78 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -153,21 +153,108 @@ jobs: echo "=== Testing $test_name in namespace: $namespace ===" kubectl create namespace "$namespace" - # Apply resources + # Apply resources and monitor status + echo "Applying resources..." helm template rybbit . -f "$test_file" --namespace "$namespace" | kubectl apply -f - -n "$namespace" - # Wait for resources to be ready - echo "Waiting for resources to be ready..." - kubectl wait --for=condition=available --timeout=40s deployment --all -n "$namespace" 2>/dev/null || true - kubectl wait --for=condition=complete --timeout=40s job --all -n "$namespace" 2>/dev/null || true + # Immediate check of events and pod status + echo "=== Initial Status Check ===" + echo "Deployments:" + kubectl get deployments -n "$namespace" -o wide - # Quick status check - echo "Final status:" - kubectl get deployments,jobs,pods -n "$namespace" -o wide + echo -e "\nJobs:" + kubectl get jobs -n "$namespace" -o wide - # Cleanup + echo -e "\nPods:" + kubectl get pods -n "$namespace" -o wide + + echo -e "\nRecent Events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 + + # Monitor deployments and jobs with continuous updates + echo -e "\nMonitoring deployments and jobs..." + while true; do + echo "=== Current Status ===" + kubectl get deployments,jobs,pods -n "$namespace" -o wide + + # Check if all deployments are ready + deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + + # Check if all jobs are completed + job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) + job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) + + # If no jobs exist, set job counts to 0 + if [ "$job_total_count" -eq 0 ]; then + job_completed_count=0 + job_total_count=0 + fi + + echo "Deployments: $deployment_ready_count/$deployment_total_count ready" + echo "Jobs: $job_completed_count/$job_total_count completed" + + # Check if all deployments are ready AND all jobs are completed + deployments_ready=false + jobs_completed=false + + # Check if deployments are ready (if any exist) + if [ "$deployment_total_count" -eq 0 ]; then + deployments_ready=true + elif [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ]; then + deployments_ready=true + fi + + # Check if jobs are completed (if any exist) + if [ "$job_total_count" -eq 0 ]; then + jobs_completed=true + elif [ "$job_completed_count" -eq "$job_total_count" ] && [ "$job_total_count" -gt 0 ]; then + jobs_completed=true + fi + + if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ]; then + echo "All deployments are ready and all jobs are completed!" + break + fi + + # Check for any failed pods + if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating"; then + echo "Found failed or terminating pods. Checking details..." + kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" + echo -e "\nPod Events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" + echo -e "\nPod Details:" + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do + echo -e "\n=== $pod ===" + kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" + done + exit 1 + fi + + # Check for failed jobs + failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') + if [ -n "$failed_jobs" ]; then + echo "Found failed jobs: $failed_jobs" + for job in $failed_jobs; do + echo -e "\n=== Job $job details ===" + kubectl describe job -n "$namespace" "$job" + echo -e "\n=== Job $job pod logs ===" + job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') + for pod in $job_pods; do + echo "--- Pod $pod logs ---" + kubectl logs -n "$namespace" "$pod" || echo "No logs available" + done + done + exit 1 + fi + + sleep 5 + done + + # Cleanup with force delete + echo "=== Cleaning up ===" kubectl delete namespace "$namespace" --force --grace-period=0 - echo "✓ Test completed for $test_name" + echo "✓ Cleanup completed for $test_name" done publish: From 51ba3b940d48415342ed67f7f96ca1c5a167dcf8 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:51:53 -0300 Subject: [PATCH 06/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 406bb78..30acfc2 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -173,8 +173,19 @@ jobs: # Monitor deployments and jobs with continuous updates echo -e "\nMonitoring deployments and jobs..." + start_time=$(date +%s) + timeout_seconds=30 + while true; do - echo "=== Current Status ===" + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + + if [ $elapsed -ge $timeout_seconds ]; then + echo "Timeout reached (${timeout_seconds}s). Proceeding to cleanup." + break + fi + + echo "=== Current Status (${elapsed}s elapsed) ===" kubectl get deployments,jobs,pods -n "$namespace" -o wide # Check if all deployments are ready From 23c394984d6d6fd49b3370c7de3734b0a9a826e0 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:53:59 -0300 Subject: [PATCH 07/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 27 ++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 30acfc2..61b6c62 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -179,13 +179,14 @@ jobs: while true; do current_time=$(date +%s) elapsed=$((current_time - start_time)) + remaining=$((timeout_seconds - elapsed)) if [ $elapsed -ge $timeout_seconds ]; then - echo "Timeout reached (${timeout_seconds}s). Proceeding to cleanup." + echo "[$test_name] Timeout reached (${timeout_seconds}s). Proceeding to cleanup." break fi - echo "=== Current Status (${elapsed}s elapsed) ===" + echo "[$test_name] === Current Status (${elapsed}s elapsed, ${remaining}s remaining) ===" kubectl get deployments,jobs,pods -n "$namespace" -o wide # Check if all deployments are ready @@ -202,8 +203,8 @@ jobs: job_total_count=0 fi - echo "Deployments: $deployment_ready_count/$deployment_total_count ready" - echo "Jobs: $job_completed_count/$job_total_count completed" + echo "[$test_name] Deployments: $deployment_ready_count/$deployment_total_count ready" + echo "[$test_name] Jobs: $job_completed_count/$job_total_count completed" # Check if all deployments are ready AND all jobs are completed deployments_ready=false @@ -224,19 +225,19 @@ jobs: fi if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ]; then - echo "All deployments are ready and all jobs are completed!" + echo "[$test_name] All deployments are ready and all jobs are completed!" break fi # Check for any failed pods if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating"; then - echo "Found failed or terminating pods. Checking details..." + echo "[$test_name] Found failed or terminating pods. Checking details..." kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" - echo -e "\nPod Events:" + echo -e "\n[$test_name] Pod Events:" kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" - echo -e "\nPod Details:" + echo -e "\n[$test_name] Pod Details:" for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do - echo -e "\n=== $pod ===" + echo -e "\n[$test_name] === $pod ===" kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" done exit 1 @@ -245,14 +246,14 @@ jobs: # Check for failed jobs failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') if [ -n "$failed_jobs" ]; then - echo "Found failed jobs: $failed_jobs" + echo "[$test_name] Found failed jobs: $failed_jobs" for job in $failed_jobs; do - echo -e "\n=== Job $job details ===" + echo -e "\n[$test_name] === Job $job details ===" kubectl describe job -n "$namespace" "$job" - echo -e "\n=== Job $job pod logs ===" + echo -e "\n[$test_name] === Job $job pod logs ===" job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') for pod in $job_pods; do - echo "--- Pod $pod logs ---" + echo "[$test_name] --- Pod $pod logs ---" kubectl logs -n "$namespace" "$pod" || echo "No logs available" done done From 120bf684bbb414164b654ee6f151791d69c6485a Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 16:58:32 -0300 Subject: [PATCH 08/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 61b6c62..c8e3d2f 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -182,8 +182,12 @@ jobs: remaining=$((timeout_seconds - elapsed)) if [ $elapsed -ge $timeout_seconds ]; then - echo "[$test_name] Timeout reached (${timeout_seconds}s). Proceeding to cleanup." - break + echo "[$test_name] Timeout reached (${timeout_seconds}s). Test FAILED!" + echo "[$test_name] Final status before timeout:" + kubectl get deployments,jobs,pods -n "$namespace" -o wide + echo "[$test_name] Recent events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 10 + exit 1 fi echo "[$test_name] === Current Status (${elapsed}s elapsed, ${remaining}s remaining) ===" From 236cdab5d5ff291bdb9451e6a1b1567c50397146 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:07:06 -0300 Subject: [PATCH 09/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index c8e3d2f..73e8fde 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -2,11 +2,8 @@ name: Helm Chart CI/CD on: push: - branches: - - main - - staging tags: - - 'v*' + - 'v*' workflow_dispatch: env: From 8eb6095c9710d68b6e288cf1e683e26630583f96 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:08:52 -0300 Subject: [PATCH 10/46] add future test case folder --- test/{test-cases => future-test-cases}/02-AzSiAz-test-case.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{test-cases => future-test-cases}/02-AzSiAz-test-case.yaml (100%) diff --git a/test/test-cases/02-AzSiAz-test-case.yaml b/test/future-test-cases/02-AzSiAz-test-case.yaml similarity index 100% rename from test/test-cases/02-AzSiAz-test-case.yaml rename to test/future-test-cases/02-AzSiAz-test-case.yaml From 3daf0dcb166063193aa85d8b3df7e70fa9b8a5d3 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:13:28 -0300 Subject: [PATCH 11/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 73e8fde..488f92a 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -2,6 +2,8 @@ name: Helm Chart CI/CD on: push: + branches: + - '**' tags: - 'v*' workflow_dispatch: From 76dc188b1779e92e5a6e4de1e17d6e5e10cd34e2 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:15:54 -0300 Subject: [PATCH 12/46] Update 01-definetelynobody-test-case.yaml --- test/test-cases/01-definetelynobody-test-case.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test-cases/01-definetelynobody-test-case.yaml b/test/test-cases/01-definetelynobody-test-case.yaml index 8a5fcc7..d48cdc4 100644 --- a/test/test-cases/01-definetelynobody-test-case.yaml +++ b/test/test-cases/01-definetelynobody-test-case.yaml @@ -18,6 +18,4 @@ headscale: oidc: client_id: "test-headscale-client-id" enabled: true - issuer: "https://auth.test.example.com" -pvc: - storageClassName: "test-class-sc" \ No newline at end of file + issuer: "https://auth.test.example.com" \ No newline at end of file From 14963c546a910c344f0d4a2dd448056cf6bd816a Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:33:42 -0300 Subject: [PATCH 13/46] add test secret template --- .github/workflows/publish-helm-chart.yml | 10 ++++++++++ test/templates/secret-oidc.yaml | 17 +++++++++++++++++ .../01-definetelynobody-test-case.yaml | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 test/templates/secret-oidc.yaml diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 488f92a..4fe1983 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -65,6 +65,16 @@ jobs: echo "✓ Using existing dependencies" fi + - name: Copy test templates + run: | + echo "=== Copying test templates ===" + if [ -d "test/templates" ]; then + cp -r test/templates/* templates/ + echo "✓ Test templates copied successfully" + else + echo "No test templates directory found" + fi + - name: Generate templates id: generate run: | diff --git a/test/templates/secret-oidc.yaml b/test/templates/secret-oidc.yaml new file mode 100644 index 0000000..7bf6f80 --- /dev/null +++ b/test/templates/secret-oidc.yaml @@ -0,0 +1,17 @@ +--- +{{- if or .Values.headplane.oidc.enabled .Values.headscale.oidc.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.headplane.oidc.secret_name | default .Values.headscale.oidc.secret_name | default "oidc-secrets" }} +type: Opaque +stringData: + {{- if .Values.headplane.oidc.enabled }} + HEADPLANE_OIDC__CLIENT_SECRET: "test-headplane-oidc-client-secret" + HEADPLANE_OIDC__CLIENT_ID: {{ .Values.headplane.oidc.client_id | default "test-headplane-oidc-client-id" | quote }} + {{- end }} + {{- if .Values.headscale.oidc.enabled }} + HEADSCALE_OIDC__CLIENT_SECRET: "test-headscale-oidc-client-secret" + HEADSCALE_OIDC__CLIENT_ID: {{ .Values.headscale.oidc.client_id | default "test-headscale-oidc-client-id" | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/test/test-cases/01-definetelynobody-test-case.yaml b/test/test-cases/01-definetelynobody-test-case.yaml index d48cdc4..6377f4e 100644 --- a/test/test-cases/01-definetelynobody-test-case.yaml +++ b/test/test-cases/01-definetelynobody-test-case.yaml @@ -6,7 +6,7 @@ headplane: server: cookie_secret: "test-cookie-secret-12345" oidc: - client_id: "test-headplane-client-id" + # client_id: "test-headplane-client-id" enabled: true issuer: "https://auth.test.example.com" redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" From 47288bf5f0ebb8181c291cc551c895d65f44ea35 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:36:12 -0300 Subject: [PATCH 14/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 4fe1983..40d91f8 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -248,10 +248,12 @@ jobs: kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" echo -e "\n[$test_name] Pod Events:" kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" - echo -e "\n[$test_name] Pod Details:" + echo -e "\n[$test_name] Pod Details and Logs:" for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do echo -e "\n[$test_name] === $pod ===" kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" + echo -e "\n[$test_name] === $pod logs ===" + kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" done exit 1 fi From 63c740d1dd2004d10ddd815e109e712ede31845d Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:39:31 -0300 Subject: [PATCH 15/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 40d91f8..a4a1643 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -243,9 +243,9 @@ jobs: fi # Check for any failed pods - if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating"; then + if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then echo "[$test_name] Found failed or terminating pods. Checking details..." - kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating" + kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating|Failed" echo -e "\n[$test_name] Pod Events:" kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" echo -e "\n[$test_name] Pod Details and Logs:" @@ -254,6 +254,20 @@ jobs: kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" echo -e "\n[$test_name] === $pod logs ===" kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" + echo -e "\n[$test_name] === $pod previous logs (if any) ===" + kubectl logs -n "$namespace" "$pod" --all-containers --previous || echo "No previous logs available for $pod" + done + exit 1 + fi + + # Proactively capture logs for any pods that might be having issues + failed_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Failed")].metadata.name}') + if [ -n "$failed_pods" ]; then + echo "[$test_name] Found pods in Failed state: $failed_pods" + for pod in $failed_pods; do + echo -e "\n[$test_name] === $pod logs (Failed state) ===" + kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" + kubectl logs -n "$namespace" "$pod" --all-containers --previous || echo "No previous logs available for $pod" done exit 1 fi From 7969d9af12dae19034c207c6db4165b4f246e827 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 17:46:29 -0300 Subject: [PATCH 16/46] add mock oidc server to pipeline --- .../02-AzSiAz-test-case.yaml | 4 +- test/templates/mock-oidc-server.yaml | 88 +++++++++++++++++++ .../01-definetelynobody-test-case.yaml | 4 +- 3 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 test/templates/mock-oidc-server.yaml diff --git a/test/future-test-cases/02-AzSiAz-test-case.yaml b/test/future-test-cases/02-AzSiAz-test-case.yaml index 3006246..da92d5d 100644 --- a/test/future-test-cases/02-AzSiAz-test-case.yaml +++ b/test/future-test-cases/02-AzSiAz-test-case.yaml @@ -9,7 +9,7 @@ headplane: oidc: client_id: "test-headplane-client-id" enabled: true - issuer: "https://auth.test.example.com" + issuer: "http://mock-oidc-server" redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" headscale: config: @@ -22,6 +22,6 @@ headscale: method: S256 client_id: "test-headscale-client-id" enabled: true - issuer: "https://auth.test.example.com" + issuer: "http://mock-oidc-server" pvc: storageClassName: "test-class-sc" \ No newline at end of file diff --git a/test/templates/mock-oidc-server.yaml b/test/templates/mock-oidc-server.yaml new file mode 100644 index 0000000..ae1a545 --- /dev/null +++ b/test/templates/mock-oidc-server.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: mock-oidc-server +spec: + selector: + app: mock-oidc-server + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-oidc-server +spec: + replicas: 1 + selector: + matchLabels: + app: mock-oidc-server + template: + metadata: + labels: + app: mock-oidc-server + spec: + containers: + - name: mock-oidc-server + image: python:3.9-alpine + ports: + - containerPort: 8080 + command: + - python + - -c + - | + from http.server import HTTPServer, BaseHTTPRequestHandler + import json + + class MockOIDCHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + + if self.path == '/.well-known/openid-configuration': + response = { + "issuer": "https://auth.test.example.com", + "authorization_endpoint": "https://auth.test.example.com/oauth/authorize", + "token_endpoint": "https://auth.test.example.com/oauth/token", + "userinfo_endpoint": "https://auth.test.example.com/userinfo", + "jwks_uri": "https://auth.test.example.com/.well-known/jwks.json", + "response_types_supported": ["code", "token", "id_token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "scopes_supported": ["openid", "profile", "email"], + "token_endpoint_auth_methods_supported": ["client_secret_post", "client_secret_basic"], + "claims_supported": ["sub", "iss", "name", "email"] + } + elif self.path == '/.well-known/jwks.json': + response = { + "keys": [ + { + "kty": "RSA", + "kid": "test-key", + "use": "sig", + "alg": "RS256", + "n": "test-modulus", + "e": "AQAB" + } + ] + } + elif self.path == '/userinfo': + response = { + "sub": "test-user-id", + "name": "Test User", + "email": "test@example.com" + } + else: + self.send_response(404) + self.end_headers() + return + + self.wfile.write(json.dumps(response).encode()) + + server = HTTPServer(('0.0.0.0', 8080), MockOIDCHandler) + print("Mock OIDC server running on port 8080") + server.serve_forever() \ No newline at end of file diff --git a/test/test-cases/01-definetelynobody-test-case.yaml b/test/test-cases/01-definetelynobody-test-case.yaml index 6377f4e..2d280bf 100644 --- a/test/test-cases/01-definetelynobody-test-case.yaml +++ b/test/test-cases/01-definetelynobody-test-case.yaml @@ -8,7 +8,7 @@ headplane: oidc: # client_id: "test-headplane-client-id" enabled: true - issuer: "https://auth.test.example.com" + issuer: "http://mock-oidc-server" redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" headscale: config: @@ -18,4 +18,4 @@ headscale: oidc: client_id: "test-headscale-client-id" enabled: true - issuer: "https://auth.test.example.com" \ No newline at end of file + issuer: "http://mock-oidc-server" \ No newline at end of file From f6af62d8978b0583a8ca80d3e2797da152f264cc Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 18:01:42 -0300 Subject: [PATCH 17/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 25 ++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index a4a1643..73737b5 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -219,6 +219,16 @@ jobs: echo "[$test_name] Deployments: $deployment_ready_count/$deployment_total_count ready" echo "[$test_name] Jobs: $job_completed_count/$job_total_count completed" + # Proactively capture logs for pods that might be having issues + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do + pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') + if [ "$pod_status" = "Pending" ] || [ "$pod_status" = "Unknown" ]; then + echo "[$test_name] === Proactive log capture for $pod (status: $pod_status) ===" + kubectl describe pod -n "$namespace" "$pod" + kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" + fi + done + # Check if all deployments are ready AND all jobs are completed deployments_ready=false jobs_completed=false @@ -260,12 +270,15 @@ jobs: exit 1 fi - # Proactively capture logs for any pods that might be having issues - failed_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Failed")].metadata.name}') - if [ -n "$failed_pods" ]; then - echo "[$test_name] Found pods in Failed state: $failed_pods" - for pod in $failed_pods; do - echo -e "\n[$test_name] === $pod logs (Failed state) ===" + # Check for any pods not in Running state + non_running_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}') + if [ -n "$non_running_pods" ]; then + echo "[$test_name] Found pods not in Running state: $non_running_pods" + for pod in $non_running_pods; do + pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') + echo -e "\n[$test_name] === $pod (status: $pod_status) ===" + kubectl describe pod -n "$namespace" "$pod" + echo -e "\n[$test_name] === $pod logs ===" kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" kubectl logs -n "$namespace" "$pod" --all-containers --previous || echo "No previous logs available for $pod" done From 075c87308e0391e00c23d46f9c13f87969f1c027 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 18:08:54 -0300 Subject: [PATCH 18/46] Fix oidc mock --- .github/workflows/publish-helm-chart.yml | 2 +- test/templates/mock-oidc-server.yaml | 140 +++++++++++++---------- 2 files changed, 82 insertions(+), 60 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 73737b5..68c0441 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -224,7 +224,7 @@ jobs: pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') if [ "$pod_status" = "Pending" ] || [ "$pod_status" = "Unknown" ]; then echo "[$test_name] === Proactive log capture for $pod (status: $pod_status) ===" - kubectl describe pod -n "$namespace" "$pod" + # kubectl describe pod -n "$namespace" "$pod" kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" fi done diff --git a/test/templates/mock-oidc-server.yaml b/test/templates/mock-oidc-server.yaml index ae1a545..f4d4ec1 100644 --- a/test/templates/mock-oidc-server.yaml +++ b/test/templates/mock-oidc-server.yaml @@ -9,7 +9,77 @@ spec: ports: - protocol: TCP port: 80 - targetPort: 8080 + targetPort: 80 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mock-oidc-config +data: + openid-configuration: | + { + "issuer": "http://mock-oidc-server", + "authorization_endpoint": "http://mock-oidc-server/oauth/authorize", + "token_endpoint": "http://mock-oidc-server/oauth/token", + "userinfo_endpoint": "http://mock-oidc-server/userinfo", + "jwks_uri": "http://mock-oidc-server/.well-known/jwks.json", + "response_types_supported": ["code", "token", "id_token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "scopes_supported": ["openid", "profile", "email"], + "token_endpoint_auth_methods_supported": ["client_secret_post", "client_secret_basic"], + "claims_supported": ["sub", "iss", "name", "email"] + } + jwks.json: | + { + "keys": [ + { + "kty": "RSA", + "kid": "test-key", + "use": "sig", + "alg": "RS256", + "n": "test-modulus", + "e": "AQAB" + } + ] + } + userinfo: | + { + "sub": "test-user-id", + "name": "Test User", + "email": "test@example.com" + } + nginx.conf: | + events { + worker_connections 1024; + } + http { + server { + listen 80; + + location /.well-known/openid-configuration { + add_header Content-Type application/json; + add_header Access-Control-Allow-Origin *; + return 200 '{"issuer":"http://mock-oidc-server","authorization_endpoint":"http://mock-oidc-server/oauth/authorize","token_endpoint":"http://mock-oidc-server/oauth/token","userinfo_endpoint":"http://mock-oidc-server/userinfo","jwks_uri":"http://mock-oidc-server/.well-known/jwks.json","response_types_supported":["code","token","id_token"],"subject_types_supported":["public"],"id_token_signing_alg_values_supported":["RS256"],"scopes_supported":["openid","profile","email"],"token_endpoint_auth_methods_supported":["client_secret_post","client_secret_basic"],"claims_supported":["sub","iss","name","email"]}'; + } + + location /.well-known/jwks.json { + add_header Content-Type application/json; + add_header Access-Control-Allow-Origin *; + return 200 '{"keys":[{"kty":"RSA","kid":"test-key","use":"sig","alg":"RS256","n":"test-modulus","e":"AQAB"}]}'; + } + + location /userinfo { + add_header Content-Type application/json; + add_header Access-Control-Allow-Origin *; + return 200 '{"sub":"test-user-id","name":"Test User","email":"test@example.com"}'; + } + + location / { + return 404; + } + } + } --- apiVersion: apps/v1 kind: Deployment @@ -27,62 +97,14 @@ spec: spec: containers: - name: mock-oidc-server - image: python:3.9-alpine + image: nginx:alpine ports: - - containerPort: 8080 - command: - - python - - -c - - | - from http.server import HTTPServer, BaseHTTPRequestHandler - import json - - class MockOIDCHandler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - - if self.path == '/.well-known/openid-configuration': - response = { - "issuer": "https://auth.test.example.com", - "authorization_endpoint": "https://auth.test.example.com/oauth/authorize", - "token_endpoint": "https://auth.test.example.com/oauth/token", - "userinfo_endpoint": "https://auth.test.example.com/userinfo", - "jwks_uri": "https://auth.test.example.com/.well-known/jwks.json", - "response_types_supported": ["code", "token", "id_token"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["RS256"], - "scopes_supported": ["openid", "profile", "email"], - "token_endpoint_auth_methods_supported": ["client_secret_post", "client_secret_basic"], - "claims_supported": ["sub", "iss", "name", "email"] - } - elif self.path == '/.well-known/jwks.json': - response = { - "keys": [ - { - "kty": "RSA", - "kid": "test-key", - "use": "sig", - "alg": "RS256", - "n": "test-modulus", - "e": "AQAB" - } - ] - } - elif self.path == '/userinfo': - response = { - "sub": "test-user-id", - "name": "Test User", - "email": "test@example.com" - } - else: - self.send_response(404) - self.end_headers() - return - - self.wfile.write(json.dumps(response).encode()) - - server = HTTPServer(('0.0.0.0', 8080), MockOIDCHandler) - print("Mock OIDC server running on port 8080") - server.serve_forever() \ No newline at end of file + - containerPort: 80 + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + volumes: + - name: nginx-config + configMap: + name: mock-oidc-config \ No newline at end of file From 21b6cc8d6784236e3718ea3aa20e10bb34689edc Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 18:38:40 -0300 Subject: [PATCH 19/46] ct attempt --- .github/workflows/publish-helm-chart.yml | 84 +++++++++++++++--------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 68c0441..9816c29 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -166,6 +166,10 @@ jobs: echo "Applying resources..." helm template rybbit . -f "$test_file" --namespace "$namespace" | kubectl apply -f - -n "$namespace" + # Give resources time to start scheduling + echo "Waiting for resources to start scheduling..." + sleep 3 + # Immediate check of events and pod status echo "=== Initial Status Check ===" echo "Deployments:" @@ -183,7 +187,7 @@ jobs: # Monitor deployments and jobs with continuous updates echo -e "\nMonitoring deployments and jobs..." start_time=$(date +%s) - timeout_seconds=30 + timeout_seconds=60 while true; do current_time=$(date +%s) @@ -216,16 +220,18 @@ jobs: job_total_count=0 fi - echo "[$test_name] Deployments: $deployment_ready_count/$deployment_total_count ready" - echo "[$test_name] Jobs: $job_completed_count/$job_total_count completed" + echo "[$test_name] Status: Deployments $deployment_ready_count/$deployment_total_count ready, Jobs $job_completed_count/$job_total_count completed" # Proactively capture logs for pods that might be having issues for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') - if [ "$pod_status" = "Pending" ] || [ "$pod_status" = "Unknown" ]; then - echo "[$test_name] === Proactive log capture for $pod (status: $pod_status) ===" - # kubectl describe pod -n "$namespace" "$pod" - kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" + # Only capture logs from pods that are running or have failed + if [ "$pod_status" = "Running" ] || [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ]; then + echo "[$test_name] Logs for $pod (status: $pod_status):" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=5 || echo "No logs available for $pod" + elif [ "$pod_status" = "Pending" ]; then + # For pending pods, just show the status without trying to get logs + echo "[$test_name] Pod $pod is still pending - waiting for container to start..." fi done @@ -247,8 +253,20 @@ jobs: jobs_completed=true fi - if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ]; then - echo "[$test_name] All deployments are ready and all jobs are completed!" + # Additional check: ensure all containers in running pods are ready + containers_ready=true + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}'); do + ready_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[?(@.ready==true)].name}' | wc -w) + total_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[*].name}' | wc -w) + if [ "$ready_containers" -ne "$total_containers" ]; then + containers_ready=false + echo "[$test_name] Pod $pod has $ready_containers/$total_containers containers ready" + break + fi + done + + if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then + echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" break fi @@ -256,16 +274,12 @@ jobs: if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then echo "[$test_name] Found failed or terminating pods. Checking details..." kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating|Failed" - echo -e "\n[$test_name] Pod Events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" - echo -e "\n[$test_name] Pod Details and Logs:" + echo "[$test_name] Recent events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" | tail -5 + echo "[$test_name] Pod logs:" for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do - echo -e "\n[$test_name] === $pod ===" - kubectl describe pod -n "$namespace" "$pod" | grep -A 10 "Events:" - echo -e "\n[$test_name] === $pod logs ===" - kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" - echo -e "\n[$test_name] === $pod previous logs (if any) ===" - kubectl logs -n "$namespace" "$pod" --all-containers --previous || echo "No previous logs available for $pod" + echo "[$test_name] --- $pod logs ---" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" done exit 1 fi @@ -273,16 +287,26 @@ jobs: # Check for any pods not in Running state non_running_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}') if [ -n "$non_running_pods" ]; then - echo "[$test_name] Found pods not in Running state: $non_running_pods" + should_fail=false for pod in $non_running_pods; do pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') - echo -e "\n[$test_name] === $pod (status: $pod_status) ===" - kubectl describe pod -n "$namespace" "$pod" - echo -e "\n[$test_name] === $pod logs ===" - kubectl logs -n "$namespace" "$pod" --all-containers || echo "No logs available for $pod" - kubectl logs -n "$namespace" "$pod" --all-containers --previous || echo "No previous logs available for $pod" + + # Only fail if the pod is in a failed state, not if it's still pending + if [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ] || [ "$pod_status" = "CrashLoopBackOff" ]; then + should_fail=true + echo "[$test_name] Pod $pod has failed status: $pod_status" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" + elif [ "$pod_status" = "Pending" ]; then + echo "[$test_name] Pod $pod is still pending - normal during startup" + else + echo "[$test_name] Pod $pod has unexpected status: $pod_status" + should_fail=true + fi done - exit 1 + + if [ "$should_fail" = true ]; then + exit 1 + fi fi # Check for failed jobs @@ -290,19 +314,17 @@ jobs: if [ -n "$failed_jobs" ]; then echo "[$test_name] Found failed jobs: $failed_jobs" for job in $failed_jobs; do - echo -e "\n[$test_name] === Job $job details ===" - kubectl describe job -n "$namespace" "$job" - echo -e "\n[$test_name] === Job $job pod logs ===" + echo "[$test_name] Job $job pod logs:" job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') for pod in $job_pods; do - echo "[$test_name] --- Pod $pod logs ---" - kubectl logs -n "$namespace" "$pod" || echo "No logs available" + echo "[$test_name] --- $pod logs ---" + kubectl logs -n "$namespace" "$pod" --tail=10 || echo "No logs available" done done exit 1 fi - sleep 5 + sleep 10 done # Cleanup with force delete From 96d2c6b723d2228cdc65e6b2fba4346cee00015a Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:03:02 -0300 Subject: [PATCH 20/46] add api-token reference to headplane --- templates/statefulset-headplane.yaml | 2 ++ test/test-cases/01-definetelynobody-test-case.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 6d4b6bf..9c0000c 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -39,6 +39,8 @@ spec: {{- if .Values.headplane.oidc.enabled }} - secretRef: name: {{ .Values.headplane.oidc.secret_name }} + - secretRef: + name: headscale-api-token {{- end }} {{- with .Values.headplane.envFrom }} {{- toYaml . | nindent 10 }} diff --git a/test/test-cases/01-definetelynobody-test-case.yaml b/test/test-cases/01-definetelynobody-test-case.yaml index 2d280bf..2f9d2c4 100644 --- a/test/test-cases/01-definetelynobody-test-case.yaml +++ b/test/test-cases/01-definetelynobody-test-case.yaml @@ -4,7 +4,7 @@ headplane: url: "https://vpn.test.example.com" secret: server: - cookie_secret: "test-cookie-secret-12345" + cookie_secret: "test-cookie-secret-12345678901234567890123456789012" oidc: # client_id: "test-headplane-client-id" enabled: true From e58d0b07d7c9eb9d604aa8de3eef59a141d127a1 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:05:13 -0300 Subject: [PATCH 21/46] Add future test case to test-cases --- test/{future-test-cases => test-cases}/02-AzSiAz-test-case.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{future-test-cases => test-cases}/02-AzSiAz-test-case.yaml (100%) diff --git a/test/future-test-cases/02-AzSiAz-test-case.yaml b/test/test-cases/02-AzSiAz-test-case.yaml similarity index 100% rename from test/future-test-cases/02-AzSiAz-test-case.yaml rename to test/test-cases/02-AzSiAz-test-case.yaml From cdf370a9c961ff2372b2a5d7e91f5f5a4069aea6 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:08:15 -0300 Subject: [PATCH 22/46] Update 02-AzSiAz-test-case.yaml --- test/test-cases/02-AzSiAz-test-case.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test-cases/02-AzSiAz-test-case.yaml b/test/test-cases/02-AzSiAz-test-case.yaml index da92d5d..7450e2d 100644 --- a/test/test-cases/02-AzSiAz-test-case.yaml +++ b/test/test-cases/02-AzSiAz-test-case.yaml @@ -22,6 +22,4 @@ headscale: method: S256 client_id: "test-headscale-client-id" enabled: true - issuer: "http://mock-oidc-server" -pvc: - storageClassName: "test-class-sc" \ No newline at end of file + issuer: "http://mock-oidc-server" \ No newline at end of file From 191023e179f3127532874536fab32c30026b1ca7 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:15:15 -0300 Subject: [PATCH 23/46] Actions jobs from sequential to parallel --- .github/workflows/publish-helm-chart.yml | 436 ++++++++++++----------- 1 file changed, 222 insertions(+), 214 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 9816c29..2791e88 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -13,8 +13,28 @@ env: CHART_NAME: headplane jobs: + setup: + runs-on: ubuntu-latest + outputs: + test-cases: ${{ steps.get-test-cases.outputs.test-cases }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get test cases + id: get-test-cases + run: | + test_cases=$(find test/test-cases -name "*.yaml" -type f | sort | jq -R -s -c 'split("\n")[:-1]') + echo "test-cases=$test_cases" >> $GITHUB_OUTPUT + echo "Found test cases: $test_cases" + test: + needs: setup runs-on: ubuntu-latest + strategy: + matrix: + test-file: ${{ fromJson(needs.setup.outputs.test-cases) }} + fail-fast: false steps: - name: Checkout uses: actions/checkout@v4 @@ -75,263 +95,251 @@ jobs: echo "No test templates directory found" fi - - name: Generate templates + - name: Generate template id: generate run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - echo "=== Generating template for: $test_name ===" - helm template rybbit . -f "$test_file" > "test/output/${test_name}-output.yaml" - echo "✓ Template generated successfully for $test_name" - done + test_name=$(basename "${{ matrix.test-file }}" .yaml) + echo "=== Generating template for: $test_name ===" + helm template rybbit . -f "${{ matrix.test-file }}" > "test/output/${test_name}-output.yaml" + echo "✓ Template generated successfully for $test_name" - name: Validate YAML syntax run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - echo "=== Validating YAML for: $test_name ===" - if ! yq eval '.' "test/output/${test_name}-output.yaml" > /dev/null 2>&1; then - echo "Invalid YAML syntax in ${test_name}-output.yaml" - yq eval '.' "test/output/${test_name}-output.yaml" - exit 1 - fi - echo "✓ YAML syntax validated successfully for $test_name" - done + test_name=$(basename "${{ matrix.test-file }}" .yaml) + echo "=== Validating YAML for: $test_name ===" + if ! yq eval '.' "test/output/${test_name}-output.yaml" > /dev/null 2>&1; then + echo "Invalid YAML syntax in ${test_name}-output.yaml" + yq eval '.' "test/output/${test_name}-output.yaml" + exit 1 + fi + echo "✓ YAML syntax validated successfully for $test_name" - name: Check for unrendered templates run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - echo "=== Checking for unrendered templates in: $test_name ===" - if grep -q "{{.*}}" "test/output/${test_name}-output.yaml"; then - echo "Found unrendered template variables in ${test_name}-output.yaml" - cat "test/output/${test_name}-output.yaml" - exit 1 - fi - echo "✓ No unrendered templates found in $test_name" - done + test_name=$(basename "${{ matrix.test-file }}" .yaml) + echo "=== Checking for unrendered templates in: $test_name ===" + if grep -q "{{.*}}" "test/output/${test_name}-output.yaml"; then + echo "Found unrendered template variables in ${test_name}-output.yaml" + cat "test/output/${test_name}-output.yaml" + exit 1 + fi + echo "✓ No unrendered templates found in $test_name" - name: Validate Kubernetes resources run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - echo "=== Validating Kubernetes resources for: $test_name ===" - - # Check if the test file contains ServiceMonitor resources - if grep -q "kind: ServiceMonitor" "test/output/${test_name}-output.yaml"; then - # Check if Prometheus Operator CRDs are installed - if ! kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1; then - echo "Skipping ServiceMonitor validation as Prometheus Operator CRDs are not installed" - # Remove ServiceMonitor resources temporarily for validation - yq eval 'select(.kind != "ServiceMonitor")' "test/output/${test_name}-output.yaml" > "test/output/${test_name}-output-temp.yaml" - # Validate remaining resources - if [ -s "test/output/${test_name}-output-temp.yaml" ]; then - kubectl apply --dry-run=client -f "test/output/${test_name}-output-temp.yaml" - else - echo "No resources to validate after removing ServiceMonitor resources" - fi - rm "test/output/${test_name}-output-temp.yaml" + test_name=$(basename "${{ matrix.test-file }}" .yaml) + echo "=== Validating Kubernetes resources for: $test_name ===" + + # Check if the test file contains ServiceMonitor resources + if grep -q "kind: ServiceMonitor" "test/output/${test_name}-output.yaml"; then + # Check if Prometheus Operator CRDs are installed + if ! kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1; then + echo "Skipping ServiceMonitor validation as Prometheus Operator CRDs are not installed" + # Remove ServiceMonitor resources temporarily for validation + yq eval 'select(.kind != "ServiceMonitor")' "test/output/${test_name}-output.yaml" > "test/output/${test_name}-output-temp.yaml" + # Validate remaining resources + if [ -s "test/output/${test_name}-output-temp.yaml" ]; then + kubectl apply --dry-run=client -f "test/output/${test_name}-output-temp.yaml" else - kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" + echo "No resources to validate after removing ServiceMonitor resources" fi + rm "test/output/${test_name}-output-temp.yaml" else kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" fi - echo "✓ Kubernetes resources validated successfully for $test_name" - done + else + kubectl apply --dry-run=client -f "test/output/${test_name}-output.yaml" + fi + echo "✓ Kubernetes resources validated successfully for $test_name" - name: Check required resources run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - echo "=== Checking required resources for: $test_name ===" - if ! grep -q "kind: Deployment\|kind: Service\|kind: ConfigMap\|kind: Secret" "test/output/${test_name}-output.yaml"; then - echo "Missing required Kubernetes resources in ${test_name}-output.yaml" - exit 1 - fi - echo "✓ Required resources found in $test_name" - done + test_name=$(basename "${{ matrix.test-file }}" .yaml) + echo "=== Checking required resources for: $test_name ===" + if ! grep -q "kind: Deployment\|kind: Service\|kind: ConfigMap\|kind: Secret" "test/output/${test_name}-output.yaml"; then + echo "Missing required Kubernetes resources in ${test_name}-output.yaml" + exit 1 + fi + echo "✓ Required resources found in $test_name" - name: Deploy and test resources run: | - for test_file in test/test-cases/*.yaml; do - test_name=$(basename "$test_file" .yaml) - namespace="test-${test_name}-$(date +%m%d%H%M)" - namespace=$(echo "$namespace" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | cut -c 1-63 | sed 's/-$//') + test_name=$(basename "${{ matrix.test-file }}" .yaml) + namespace="test-${test_name}-$(date +%m%d%H%M)" + namespace=$(echo "$namespace" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | cut -c 1-63 | sed 's/-$//') + + echo "=== Testing $test_name in namespace: $namespace ===" + kubectl create namespace "$namespace" + + # Apply resources and monitor status + echo "Applying resources..." + helm template rybbit . -f "${{ matrix.test-file }}" --namespace "$namespace" | kubectl apply -f - -n "$namespace" + + # Give resources time to start scheduling + echo "Waiting for resources to start scheduling..." + sleep 3 + + # Immediate check of events and pod status + echo "=== Initial Status Check ===" + echo "Deployments:" + kubectl get deployments -n "$namespace" -o wide + + echo -e "\nJobs:" + kubectl get jobs -n "$namespace" -o wide + + echo -e "\nPods:" + kubectl get pods -n "$namespace" -o wide + + echo -e "\nRecent Events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 + + # Monitor deployments and jobs with continuous updates + echo -e "\nMonitoring deployments and jobs..." + start_time=$(date +%s) + timeout_seconds=60 + + while true; do + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + remaining=$((timeout_seconds - elapsed)) - echo "=== Testing $test_name in namespace: $namespace ===" - kubectl create namespace "$namespace" + if [ $elapsed -ge $timeout_seconds ]; then + echo "[$test_name] Timeout reached (${timeout_seconds}s). Test FAILED!" + echo "[$test_name] Final status before timeout:" + kubectl get deployments,jobs,pods -n "$namespace" -o wide + echo "[$test_name] Recent events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 10 + exit 1 + fi - # Apply resources and monitor status - echo "Applying resources..." - helm template rybbit . -f "$test_file" --namespace "$namespace" | kubectl apply -f - -n "$namespace" + echo "[$test_name] === Current Status (${elapsed}s elapsed, ${remaining}s remaining) ===" + kubectl get deployments,jobs,pods -n "$namespace" -o wide - # Give resources time to start scheduling - echo "Waiting for resources to start scheduling..." - sleep 3 + # Check if all deployments are ready + deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) + deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - # Immediate check of events and pod status - echo "=== Initial Status Check ===" - echo "Deployments:" - kubectl get deployments -n "$namespace" -o wide + # Check if all jobs are completed + job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) + job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) - echo -e "\nJobs:" - kubectl get jobs -n "$namespace" -o wide + # If no jobs exist, set job counts to 0 + if [ "$job_total_count" -eq 0 ]; then + job_completed_count=0 + job_total_count=0 + fi - echo -e "\nPods:" - kubectl get pods -n "$namespace" -o wide + echo "[$test_name] Status: Deployments $deployment_ready_count/$deployment_total_count ready, Jobs $job_completed_count/$job_total_count completed" - echo -e "\nRecent Events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 20 + # Proactively capture logs for pods that might be having issues + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do + pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') + # Only capture logs from pods that are running or have failed + if [ "$pod_status" = "Running" ] || [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ]; then + echo "[$test_name] Logs for $pod (status: $pod_status):" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=5 || echo "No logs available for $pod" + elif [ "$pod_status" = "Pending" ]; then + # For pending pods, just show the status without trying to get logs + echo "[$test_name] Pod $pod is still pending - waiting for container to start..." + fi + done - # Monitor deployments and jobs with continuous updates - echo -e "\nMonitoring deployments and jobs..." - start_time=$(date +%s) - timeout_seconds=60 + # Check if all deployments are ready AND all jobs are completed + deployments_ready=false + jobs_completed=false - while true; do - current_time=$(date +%s) - elapsed=$((current_time - start_time)) - remaining=$((timeout_seconds - elapsed)) - - if [ $elapsed -ge $timeout_seconds ]; then - echo "[$test_name] Timeout reached (${timeout_seconds}s). Test FAILED!" - echo "[$test_name] Final status before timeout:" - kubectl get deployments,jobs,pods -n "$namespace" -o wide - echo "[$test_name] Recent events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | tail -n 10 - exit 1 - fi - - echo "[$test_name] === Current Status (${elapsed}s elapsed, ${remaining}s remaining) ===" - kubectl get deployments,jobs,pods -n "$namespace" -o wide - - # Check if all deployments are ready - deployment_ready_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.readyReplicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - deployment_total_count=$(kubectl get deployments -n "$namespace" -o jsonpath='{.items[*].status.replicas}' | tr ' ' '\n' | grep -v "^$" | wc -l) - - # Check if all jobs are completed - job_completed_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].status.succeeded}' | tr ' ' '\n' | grep -v "^$" | wc -l) - job_total_count=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[*].spec.completions}' | tr ' ' '\n' | grep -v "^$" | wc -l) - - # If no jobs exist, set job counts to 0 - if [ "$job_total_count" -eq 0 ]; then - job_completed_count=0 - job_total_count=0 + # Check if deployments are ready (if any exist) + if [ "$deployment_total_count" -eq 0 ]; then + deployments_ready=true + elif [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ]; then + deployments_ready=true + fi + + # Check if jobs are completed (if any exist) + if [ "$job_total_count" -eq 0 ]; then + jobs_completed=true + elif [ "$job_completed_count" -eq "$job_total_count" ] && [ "$job_total_count" -gt 0 ]; then + jobs_completed=true + fi + + # Additional check: ensure all containers in running pods are ready + containers_ready=true + for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}'); do + ready_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[?(@.ready==true)].name}' | wc -w) + total_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[*].name}' | wc -w) + if [ "$ready_containers" -ne "$total_containers" ]; then + containers_ready=false + echo "[$test_name] Pod $pod has $ready_containers/$total_containers containers ready" + break fi - - echo "[$test_name] Status: Deployments $deployment_ready_count/$deployment_total_count ready, Jobs $job_completed_count/$job_total_count completed" - - # Proactively capture logs for pods that might be having issues + done + + if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then + echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" + break + fi + + # Check for any failed pods + if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then + echo "[$test_name] Found failed or terminating pods. Checking details..." + kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating|Failed" + echo "[$test_name] Recent events:" + kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" | tail -5 + echo "[$test_name] Pod logs:" for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do + echo "[$test_name] --- $pod logs ---" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" + done + exit 1 + fi + + # Check for any pods not in Running state + non_running_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}') + if [ -n "$non_running_pods" ]; then + should_fail=false + for pod in $non_running_pods; do pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') - # Only capture logs from pods that are running or have failed - if [ "$pod_status" = "Running" ] || [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ]; then - echo "[$test_name] Logs for $pod (status: $pod_status):" - kubectl logs -n "$namespace" "$pod" --all-containers --tail=5 || echo "No logs available for $pod" + + # Only fail if the pod is in a failed state, not if it's still pending + if [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ] || [ "$pod_status" = "CrashLoopBackOff" ]; then + should_fail=true + echo "[$test_name] Pod $pod has failed status: $pod_status" + kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" elif [ "$pod_status" = "Pending" ]; then - # For pending pods, just show the status without trying to get logs - echo "[$test_name] Pod $pod is still pending - waiting for container to start..." - fi - done - - # Check if all deployments are ready AND all jobs are completed - deployments_ready=false - jobs_completed=false - - # Check if deployments are ready (if any exist) - if [ "$deployment_total_count" -eq 0 ]; then - deployments_ready=true - elif [ "$deployment_ready_count" -eq "$deployment_total_count" ] && [ "$deployment_total_count" -gt 0 ]; then - deployments_ready=true - fi - - # Check if jobs are completed (if any exist) - if [ "$job_total_count" -eq 0 ]; then - jobs_completed=true - elif [ "$job_completed_count" -eq "$job_total_count" ] && [ "$job_total_count" -gt 0 ]; then - jobs_completed=true - fi - - # Additional check: ensure all containers in running pods are ready - containers_ready=true - for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}'); do - ready_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[?(@.ready==true)].name}' | wc -w) - total_containers=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[*].name}' | wc -w) - if [ "$ready_containers" -ne "$total_containers" ]; then - containers_ready=false - echo "[$test_name] Pod $pod has $ready_containers/$total_containers containers ready" - break + echo "[$test_name] Pod $pod is still pending - normal during startup" + else + echo "[$test_name] Pod $pod has unexpected status: $pod_status" + should_fail=true fi done - if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then - echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" - break - fi - - # Check for any failed pods - if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then - echo "[$test_name] Found failed or terminating pods. Checking details..." - kubectl get pods -n "$namespace" | grep -E "Error|CrashLoopBackOff|ImagePullBackOff|Terminating|Failed" - echo "[$test_name] Recent events:" - kubectl get events -n "$namespace" --sort-by='.lastTimestamp' | grep -E "Error|Warning|Failed" | tail -5 - echo "[$test_name] Pod logs:" - for pod in $(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do - echo "[$test_name] --- $pod logs ---" - kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" - done + if [ "$should_fail" = true ]; then exit 1 fi - - # Check for any pods not in Running state - non_running_pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[?(@.status.phase!="Running")].metadata.name}') - if [ -n "$non_running_pods" ]; then - should_fail=false - for pod in $non_running_pods; do - pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') - - # Only fail if the pod is in a failed state, not if it's still pending - if [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ] || [ "$pod_status" = "CrashLoopBackOff" ]; then - should_fail=true - echo "[$test_name] Pod $pod has failed status: $pod_status" - kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" - elif [ "$pod_status" = "Pending" ]; then - echo "[$test_name] Pod $pod is still pending - normal during startup" - else - echo "[$test_name] Pod $pod has unexpected status: $pod_status" - should_fail=true - fi - done - - if [ "$should_fail" = true ]; then - exit 1 - fi - fi - - # Check for failed jobs - failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') - if [ -n "$failed_jobs" ]; then - echo "[$test_name] Found failed jobs: $failed_jobs" - for job in $failed_jobs; do - echo "[$test_name] Job $job pod logs:" - job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') - for pod in $job_pods; do - echo "[$test_name] --- $pod logs ---" - kubectl logs -n "$namespace" "$pod" --tail=10 || echo "No logs available" - done + fi + + # Check for failed jobs + failed_jobs=$(kubectl get jobs -n "$namespace" -o jsonpath='{.items[?(@.status.failed>0)].metadata.name}') + if [ -n "$failed_jobs" ]; then + echo "[$test_name] Found failed jobs: $failed_jobs" + for job in $failed_jobs; do + echo "[$test_name] Job $job pod logs:" + job_pods=$(kubectl get pods -n "$namespace" -l job-name="$job" -o jsonpath='{.items[*].metadata.name}') + for pod in $job_pods; do + echo "[$test_name] --- $pod logs ---" + kubectl logs -n "$namespace" "$pod" --tail=10 || echo "No logs available" done - exit 1 - fi - - sleep 10 - done + done + exit 1 + fi - # Cleanup with force delete - echo "=== Cleaning up ===" - kubectl delete namespace "$namespace" --force --grace-period=0 - echo "✓ Cleanup completed for $test_name" + sleep 10 done + + # Cleanup with force delete + echo "=== Cleaning up ===" + kubectl delete namespace "$namespace" --force --grace-period=0 + echo "✓ Cleanup completed for $test_name" publish: if: startsWith(github.ref, 'refs/tags/') From 6e356770976e00a041f3f072d293e27833cd1886 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:18:48 -0300 Subject: [PATCH 24/46] Handle succeeded pod status in Helm chart workflow Updated the publish-helm-chart GitHub Actions workflow to recognize 'Succeeded' pod status as expected for completed jobs, preventing false failures during status checks. --- .github/workflows/publish-helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 2791e88..f1aa942 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -300,13 +300,15 @@ jobs: for pod in $non_running_pods; do pod_status=$(kubectl get pod -n "$namespace" "$pod" -o jsonpath='{.status.phase}') - # Only fail if the pod is in a failed state, not if it's still pending + # Only fail if the pod is in a failed state, not if it's still pending or succeeded if [ "$pod_status" = "Failed" ] || [ "$pod_status" = "Error" ] || [ "$pod_status" = "CrashLoopBackOff" ]; then should_fail=true echo "[$test_name] Pod $pod has failed status: $pod_status" kubectl logs -n "$namespace" "$pod" --all-containers --tail=10 || echo "No logs available for $pod" elif [ "$pod_status" = "Pending" ]; then echo "[$test_name] Pod $pod is still pending - normal during startup" + elif [ "$pod_status" = "Succeeded" ]; then + echo "[$test_name] Pod $pod has succeeded - this is expected for completed jobs" else echo "[$test_name] Pod $pod has unexpected status: $pod_status" should_fail=true From e2e92c0f41113395b842171bceb81cb90288aab4 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:23:41 -0300 Subject: [PATCH 25/46] Add check and creation for headscale-api-token secret The workflow now checks for the existence of the headscale-api-token secret in the target namespace. If it does not exist, a temporary secret is created to allow pod startup, ensuring dependent resources can initialize properly. --- .github/workflows/publish-helm-chart.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index f1aa942..55402a0 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -175,6 +175,16 @@ jobs: echo "Applying resources..." helm template rybbit . -f "${{ matrix.test-file }}" --namespace "$namespace" | kubectl apply -f - -n "$namespace" + # Handle headscale-api-token secret dependency + echo "Checking for headscale-api-token secret dependency..." + if kubectl get secret headscale-api-token -n "$namespace" >/dev/null 2>&1; then + echo "headscale-api-token secret already exists" + else + echo "Creating temporary headscale-api-token secret to allow pod startup..." + kubectl create secret generic headscale-api-token --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY="temporary-token" -n "$namespace" + echo "Temporary secret created - will be replaced by job when headscale is ready" + fi + # Give resources time to start scheduling echo "Waiting for resources to start scheduling..." sleep 3 From 3625e0c144282966ac4faac929f1076da56dcab8 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:38:11 -0300 Subject: [PATCH 26/46] Update 02-AzSiAz-test-case.yaml --- test/test-cases/02-AzSiAz-test-case.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test-cases/02-AzSiAz-test-case.yaml b/test/test-cases/02-AzSiAz-test-case.yaml index 7450e2d..9411f8d 100644 --- a/test/test-cases/02-AzSiAz-test-case.yaml +++ b/test/test-cases/02-AzSiAz-test-case.yaml @@ -5,12 +5,13 @@ headplane: url: "https://vpn.test.example.com" secret: server: - cookie_secret: "test-cookie-secret-12345" + cookie_secret: "test-cookie-secret-12345678901234567890123456789012" oidc: client_id: "test-headplane-client-id" enabled: true issuer: "http://mock-oidc-server" redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" + headscale_api_key: "test-headscale-api-key" headscale: config: server_url: "https://vpn.test.example.com" From 87a5bba6c185794f56815a2f454e21bbbc8fcd5c Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:45:48 -0300 Subject: [PATCH 27/46] Update 02-AzSiAz-test-case.yaml --- test/test-cases/02-AzSiAz-test-case.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test-cases/02-AzSiAz-test-case.yaml b/test/test-cases/02-AzSiAz-test-case.yaml index 9411f8d..51ba081 100644 --- a/test/test-cases/02-AzSiAz-test-case.yaml +++ b/test/test-cases/02-AzSiAz-test-case.yaml @@ -3,15 +3,11 @@ headplane: config: headscale: url: "https://vpn.test.example.com" - secret: - server: - cookie_secret: "test-cookie-secret-12345678901234567890123456789012" oidc: client_id: "test-headplane-client-id" enabled: true issuer: "http://mock-oidc-server" redirect_uri: "https://headplane.test.example.com/admin/oidc/callback" - headscale_api_key: "test-headscale-api-key" headscale: config: server_url: "https://vpn.test.example.com" From cccd7a87b1e4e47d56ddbf3e7886482c7e9753db Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:53:52 -0300 Subject: [PATCH 28/46] Refactor Headscale API token handling in Helm chart Replaces the separate headscale-api-token secret with direct injection of the generated API token into the headplane-config secret. Updates the job logic to always generate and update the config secret, removes legacy secret checks, and cleans up related references in manifests and test cases. --- .github/workflows/publish-helm-chart.yml | 10 --------- templates/job.yaml | 22 +++++++++++-------- templates/secret-headplane.yaml | 1 + templates/statefulset-headplane.yaml | 2 -- .../01-definetelynobody-test-case.yaml | 3 --- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 55402a0..f1aa942 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -175,16 +175,6 @@ jobs: echo "Applying resources..." helm template rybbit . -f "${{ matrix.test-file }}" --namespace "$namespace" | kubectl apply -f - -n "$namespace" - # Handle headscale-api-token secret dependency - echo "Checking for headscale-api-token secret dependency..." - if kubectl get secret headscale-api-token -n "$namespace" >/dev/null 2>&1; then - echo "headscale-api-token secret already exists" - else - echo "Creating temporary headscale-api-token secret to allow pod startup..." - kubectl create secret generic headscale-api-token --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY="temporary-token" -n "$namespace" - echo "Temporary secret created - will be replaced by job when headscale is ready" - fi - # Give resources time to start scheduling echo "Waiting for resources to start scheduling..." sleep 3 diff --git a/templates/job.yaml b/templates/job.yaml index 27300b5..04f1c82 100644 --- a/templates/job.yaml +++ b/templates/job.yaml @@ -23,13 +23,7 @@ spec: sleep 1 done - echo "Checking if Secret 'headscale-api-token' exists..." - if kubectl get secret headscale-api-token -n {{ .Release.Namespace }} >/dev/null 2>&1; then - echo "Secret already exists. Skipping token generation." - exit 0 - fi - - echo "Secret not found. Generating Headscale API token..." + echo "Generating Headscale API token..." TOKEN=$(kubectl -n {{ .Release.Namespace }} exec -i headplane-0 -c headscale -- headscale apikeys create -e 100y) if [ -z "$TOKEN" ]; then @@ -37,5 +31,15 @@ spec: exit 1 fi - echo "Creating Kubernetes Secret..." - kubectl create secret generic headscale-api-token --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY="$TOKEN" -n {{ .Release.Namespace }} + echo "Updating headplane-config secret with generated API token..." + + # Get the current config from the secret + CURRENT_CONFIG=$(kubectl get secret headplane-config -n {{ .Release.Namespace }} -o jsonpath='{.data.config\.yaml}' | base64 -d) + + # Replace the placeholder API key with the real one + UPDATED_CONFIG=$(echo "$CURRENT_CONFIG" | sed "s/headscale_api_key: \"hcap_placeholder_key_for_validation_only_12345678901234567890123456789012\"/headscale_api_key: \"$TOKEN\"/") + + # Update the secret with the new config + echo "$UPDATED_CONFIG" | kubectl create secret generic headplane-config --from-file=config.yaml=/dev/stdin -n {{ .Release.Namespace }} --dry-run=client -o yaml | kubectl apply -f - + + echo "Successfully updated headplane-config secret with real API token" diff --git a/templates/secret-headplane.yaml b/templates/secret-headplane.yaml index db6a0f3..8a79075 100644 --- a/templates/secret-headplane.yaml +++ b/templates/secret-headplane.yaml @@ -20,4 +20,5 @@ stringData: token_endpoint_auth_method: {{ .Values.headplane.oidc.token_endpoint_auth_method | quote }} redirect_uri: {{ .Values.headplane.oidc.redirect_uri | quote }} client_id: {{ .Values.headplane.oidc.client_id | quote }} + headscale_api_key: {{ .Values.headplane.oidc.headscale_api_key | default "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" | quote }} {{- end }} \ No newline at end of file diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 9c0000c..6d4b6bf 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -39,8 +39,6 @@ spec: {{- if .Values.headplane.oidc.enabled }} - secretRef: name: {{ .Values.headplane.oidc.secret_name }} - - secretRef: - name: headscale-api-token {{- end }} {{- with .Values.headplane.envFrom }} {{- toYaml . | nindent 10 }} diff --git a/test/test-cases/01-definetelynobody-test-case.yaml b/test/test-cases/01-definetelynobody-test-case.yaml index 2f9d2c4..1bfbc69 100644 --- a/test/test-cases/01-definetelynobody-test-case.yaml +++ b/test/test-cases/01-definetelynobody-test-case.yaml @@ -2,9 +2,6 @@ headplane: config: headscale: url: "https://vpn.test.example.com" - secret: - server: - cookie_secret: "test-cookie-secret-12345678901234567890123456789012" oidc: # client_id: "test-headplane-client-id" enabled: true From c0127950aab207e1e39c61d0eb0ff4549738f88c Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:55:30 -0300 Subject: [PATCH 29/46] Update roles.yaml --- templates/roles.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/roles.yaml b/templates/roles.yaml index 39b2d72..568b83d 100644 --- a/templates/roles.yaml +++ b/templates/roles.yaml @@ -57,4 +57,4 @@ rules: verbs: ["create"] - apiGroups: [""] resources: ["secrets"] - verbs: ["create", "get"] + verbs: ["create", "get", "update", "patch"] From 12cb7e2c8aee5a845e0e3f24ef09577867bfdc90 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:59:45 -0300 Subject: [PATCH 30/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index f1aa942..960a85f 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -276,6 +276,29 @@ jobs: if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" + + # Verify API key replacement if OIDC is enabled + if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then + echo "[$test_name] Verifying API key replacement in headplane-config secret..." + CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) + + if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then + API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') + + if [[ "$API_KEY" == hcap_* ]] && [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]]; then + echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." + else + echo "[$test_name] ❌ API key replacement verification failed" + echo "[$test_name] Current API key: $API_KEY" + echo "[$test_name] Expected: Real hcap_ token (not placeholder)" + exit 1 + fi + else + echo "[$test_name] ❌ No headscale_api_key found in config" + exit 1 + fi + fi + break fi From 4c9c0cc04c1915e8c95a49c2176afb08d22ffd15 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:02:07 -0300 Subject: [PATCH 31/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 960a85f..897a34e 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -285,12 +285,12 @@ jobs: if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') - if [[ "$API_KEY" == hcap_* ]] && [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]]; then + if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." else echo "[$test_name] ❌ API key replacement verification failed" echo "[$test_name] Current API key: $API_KEY" - echo "[$test_name] Expected: Real hcap_ token (not placeholder)" + echo "[$test_name] Expected: Real API token (not placeholder)" exit 1 fi else From 823a88c1464bccc1d5736255296302b4555af468 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:07:58 -0300 Subject: [PATCH 32/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 897a34e..6a55ed9 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -279,23 +279,29 @@ jobs: # Verify API key replacement if OIDC is enabled if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then - echo "[$test_name] Verifying API key replacement in headplane-config secret..." CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) - if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then - API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') + # Only verify API key replacement if OIDC is enabled + if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then + echo "[$test_name] Verifying API key replacement in headplane-config secret..." - if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then - echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." + if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then + API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') + + if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then + echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." + else + echo "[$test_name] ❌ API key replacement verification failed" + echo "[$test_name] Current API key: $API_KEY" + echo "[$test_name] Expected: Real API token (not placeholder)" + exit 1 + fi else - echo "[$test_name] ❌ API key replacement verification failed" - echo "[$test_name] Current API key: $API_KEY" - echo "[$test_name] Expected: Real API token (not placeholder)" + echo "[$test_name] ❌ No headscale_api_key found in config" exit 1 fi else - echo "[$test_name] ❌ No headscale_api_key found in config" - exit 1 + echo "[$test_name] OIDC not enabled, skipping API key verification" fi fi From df04b2547015051429d85c7c4f18ce9032f58d8b Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:16:19 -0300 Subject: [PATCH 33/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 6a55ed9..07dcd6a 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -374,6 +374,7 @@ jobs: publish: if: startsWith(github.ref, 'refs/tags/') + needs: [setup, test] runs-on: ubuntu-latest permissions: contents: read From 6f12d290c64fe98f0b4d271ce66d3a90b974c488 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:37:51 -0300 Subject: [PATCH 34/46] move headscale_api_key to oidc-secrets --- README.md.gotmpl | 1 + templates/job.yaml | 22 +++++++++------------- templates/secret-headplane.yaml | 2 +- templates/statefulset-headplane.yaml | 3 +++ values.yaml | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md.gotmpl b/README.md.gotmpl index 943bb75..333f966 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -51,6 +51,7 @@ kubectl create secret generic oidc-secrets \ --from-literal=HEADPLANE_OIDC__CLIENT_ID=your-headplane-oidc-client-id \ --from-literal=HEADSCALE_OIDC__CLIENT_SECRET=your-headscale-oidc-client-secret \ --from-literal=HEADSCALE_OIDC__CLIENT_ID=your-headscale-oidc-client-id \ + --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY=this-is-optional-and-autogenerated-by-chart -n ``` diff --git a/templates/job.yaml b/templates/job.yaml index 04f1c82..27300b5 100644 --- a/templates/job.yaml +++ b/templates/job.yaml @@ -23,7 +23,13 @@ spec: sleep 1 done - echo "Generating Headscale API token..." + echo "Checking if Secret 'headscale-api-token' exists..." + if kubectl get secret headscale-api-token -n {{ .Release.Namespace }} >/dev/null 2>&1; then + echo "Secret already exists. Skipping token generation." + exit 0 + fi + + echo "Secret not found. Generating Headscale API token..." TOKEN=$(kubectl -n {{ .Release.Namespace }} exec -i headplane-0 -c headscale -- headscale apikeys create -e 100y) if [ -z "$TOKEN" ]; then @@ -31,15 +37,5 @@ spec: exit 1 fi - echo "Updating headplane-config secret with generated API token..." - - # Get the current config from the secret - CURRENT_CONFIG=$(kubectl get secret headplane-config -n {{ .Release.Namespace }} -o jsonpath='{.data.config\.yaml}' | base64 -d) - - # Replace the placeholder API key with the real one - UPDATED_CONFIG=$(echo "$CURRENT_CONFIG" | sed "s/headscale_api_key: \"hcap_placeholder_key_for_validation_only_12345678901234567890123456789012\"/headscale_api_key: \"$TOKEN\"/") - - # Update the secret with the new config - echo "$UPDATED_CONFIG" | kubectl create secret generic headplane-config --from-file=config.yaml=/dev/stdin -n {{ .Release.Namespace }} --dry-run=client -o yaml | kubectl apply -f - - - echo "Successfully updated headplane-config secret with real API token" + echo "Creating Kubernetes Secret..." + kubectl create secret generic headscale-api-token --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY="$TOKEN" -n {{ .Release.Namespace }} diff --git a/templates/secret-headplane.yaml b/templates/secret-headplane.yaml index 8a79075..5b80c6f 100644 --- a/templates/secret-headplane.yaml +++ b/templates/secret-headplane.yaml @@ -20,5 +20,5 @@ stringData: token_endpoint_auth_method: {{ .Values.headplane.oidc.token_endpoint_auth_method | quote }} redirect_uri: {{ .Values.headplane.oidc.redirect_uri | quote }} client_id: {{ .Values.headplane.oidc.client_id | quote }} - headscale_api_key: {{ .Values.headplane.oidc.headscale_api_key | default "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" | quote }} + # headscale_api_key: {{ .Values.headplane.oidc.headscale_api_key | default "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" | quote }} {{- end }} \ No newline at end of file diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 6d4b6bf..2b0dc07 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -39,6 +39,9 @@ spec: {{- if .Values.headplane.oidc.enabled }} - secretRef: name: {{ .Values.headplane.oidc.secret_name }} + - secretRef: + name: headscale-api-token + optional: true {{- end }} {{- with .Values.headplane.envFrom }} {{- toYaml . | nindent 10 }} diff --git a/values.yaml b/values.yaml index 3a63fb8..03f1e54 100644 --- a/values.yaml +++ b/values.yaml @@ -14,7 +14,7 @@ headplane: # OIDC redirect URI redirect_uri: "https://your-headplane-admin-domain.com/admin/oidc/callback" # OIDC client ID - client_id: "REPLACE_IT_WITH_YOUR_OIDC_CLIENT_ID_FOR_HEADPLANE" + # client_id: "REPLACE_IT_WITH_YOUR_OIDC_CLIENT_ID_FOR_HEADPLANE" #USE secret below instead # Name of the secret containing OIDC credentials secret_name: "oidc-secrets" config: From 0b21633b61d330940c3da60f8fde23dcf9f09fe1 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:43:52 -0300 Subject: [PATCH 35/46] add headplane wait check for job to finish --- templates/roles.yaml | 3 +++ templates/statefulset-headplane.yaml | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/templates/roles.yaml b/templates/roles.yaml index 568b83d..5c0e288 100644 --- a/templates/roles.yaml +++ b/templates/roles.yaml @@ -11,6 +11,9 @@ rules: - apiGroups: ['apps'] resources: ['deployments'] verbs: ['get', 'list'] +- apiGroups: ['batch'] + resources: ['jobs'] + verbs: ['get', 'list', 'watch'] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 2b0dc07..ac8ef8e 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -35,6 +35,16 @@ spec: containers: - name: headplane image: {{ .Values.headplane.image }} + command: + - /bin/sh + - -c + - | + {{- if .Values.headplane.oidc.enabled }} + echo "Waiting for headscale-generate-token job to complete..." + kubectl wait --for=condition=complete job/headscale-generate-token -n {{ .Release.Namespace }} --timeout=300s + echo "Job completed successfully, starting headplane..." + {{- end }} + exec node /app/build/server/index.js envFrom: {{- if .Values.headplane.oidc.enabled }} - secretRef: From b80d1f95c1a130f3b6ef23c00e439382c8023d67 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:46:07 -0300 Subject: [PATCH 36/46] Update secret-headplane.yaml --- templates/secret-headplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/secret-headplane.yaml b/templates/secret-headplane.yaml index 5b80c6f..8a79075 100644 --- a/templates/secret-headplane.yaml +++ b/templates/secret-headplane.yaml @@ -20,5 +20,5 @@ stringData: token_endpoint_auth_method: {{ .Values.headplane.oidc.token_endpoint_auth_method | quote }} redirect_uri: {{ .Values.headplane.oidc.redirect_uri | quote }} client_id: {{ .Values.headplane.oidc.client_id | quote }} - # headscale_api_key: {{ .Values.headplane.oidc.headscale_api_key | default "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" | quote }} + headscale_api_key: {{ .Values.headplane.oidc.headscale_api_key | default "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" | quote }} {{- end }} \ No newline at end of file From dc632cad06b76440042aa1b289f458096d56861c Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:49:06 -0300 Subject: [PATCH 37/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 56 ++++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 07dcd6a..52f94ab 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -274,39 +274,39 @@ jobs: fi done - if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then - echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" + # if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then + # echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" - # Verify API key replacement if OIDC is enabled - if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then - CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) + # # Verify API key replacement if OIDC is enabled + # if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then + # CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) - # Only verify API key replacement if OIDC is enabled - if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then - echo "[$test_name] Verifying API key replacement in headplane-config secret..." + # # Only verify API key replacement if OIDC is enabled + # if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then + # echo "[$test_name] Verifying API key replacement in headplane-config secret..." - if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then - API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') + # if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then + # API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') - if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then - echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." - else - echo "[$test_name] ❌ API key replacement verification failed" - echo "[$test_name] Current API key: $API_KEY" - echo "[$test_name] Expected: Real API token (not placeholder)" - exit 1 - fi - else - echo "[$test_name] ❌ No headscale_api_key found in config" - exit 1 - fi - else - echo "[$test_name] OIDC not enabled, skipping API key verification" - fi - fi + # if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then + # echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." + # else + # echo "[$test_name] ❌ API key replacement verification failed" + # echo "[$test_name] Current API key: $API_KEY" + # echo "[$test_name] Expected: Real API token (not placeholder)" + # exit 1 + # fi + # else + # echo "[$test_name] ❌ No headscale_api_key found in config" + # exit 1 + # fi + # else + # echo "[$test_name] OIDC not enabled, skipping API key verification" + # fi + # fi - break - fi + # break + # fi # Check for any failed pods if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then From ee675661c7ddd28ecc62d1cf4320e318c115cfcf Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:51:17 -0300 Subject: [PATCH 38/46] Update values.yaml --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 03f1e54..d01c058 100644 --- a/values.yaml +++ b/values.yaml @@ -14,7 +14,7 @@ headplane: # OIDC redirect URI redirect_uri: "https://your-headplane-admin-domain.com/admin/oidc/callback" # OIDC client ID - # client_id: "REPLACE_IT_WITH_YOUR_OIDC_CLIENT_ID_FOR_HEADPLANE" #USE secret below instead + client_id: "PLACEHOLDER_USE_SECRET_BELOW" #USE secret below instead # Name of the secret containing OIDC credentials secret_name: "oidc-secrets" config: From 866855d07f86ba155e8da70893e57d0ccf71daa8 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:54:59 -0300 Subject: [PATCH 39/46] Update publish-helm-chart.yml --- .github/workflows/publish-helm-chart.yml | 56 ++++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 52f94ab..53addd3 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -274,39 +274,39 @@ jobs: fi done - # if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then - # echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" + if [ "$deployments_ready" = true ] && [ "$jobs_completed" = true ] && [ "$containers_ready" = true ]; then + echo "[$test_name] All deployments are ready, all jobs are completed, and all containers are ready!" - # # Verify API key replacement if OIDC is enabled - # if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then - # CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) + # # Verify API key replacement if OIDC is enabled + # if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then + # CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) - # # Only verify API key replacement if OIDC is enabled - # if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then - # echo "[$test_name] Verifying API key replacement in headplane-config secret..." + # # Only verify API key replacement if OIDC is enabled + # if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then + # echo "[$test_name] Verifying API key replacement in headplane-config secret..." - # if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then - # API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') + # if echo "$CONFIG_CONTENT" | grep -q "headscale_api_key:"; then + # API_KEY=$(echo "$CONFIG_CONTENT" | grep "headscale_api_key:" | sed 's/.*headscale_api_key: "\([^"]*\)".*/\1/') - # if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then - # echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." - # else - # echo "[$test_name] ❌ API key replacement verification failed" - # echo "[$test_name] Current API key: $API_KEY" - # echo "[$test_name] Expected: Real API token (not placeholder)" - # exit 1 - # fi - # else - # echo "[$test_name] ❌ No headscale_api_key found in config" - # exit 1 - # fi - # else - # echo "[$test_name] OIDC not enabled, skipping API key verification" - # fi - # fi + # if [[ "$API_KEY" != "hcap_placeholder_key_for_validation_only_12345678901234567890123456789012" ]] && [[ -n "$API_KEY" ]]; then + # echo "[$test_name] ✅ API key successfully replaced with real token: ${API_KEY:0:20}..." + # else + # echo "[$test_name] ❌ API key replacement verification failed" + # echo "[$test_name] Current API key: $API_KEY" + # echo "[$test_name] Expected: Real API token (not placeholder)" + # exit 1 + # fi + # else + # echo "[$test_name] ❌ No headscale_api_key found in config" + # exit 1 + # fi + # else + # echo "[$test_name] OIDC not enabled, skipping API key verification" + # fi + # fi - # break - # fi + break + fi # Check for any failed pods if kubectl get pods -n "$namespace" | grep -q "Error\|CrashLoopBackOff\|ImagePullBackOff\|Terminating\|Failed"; then From 3eb4c0d7b29d4b069a71e104cd3aa1e689e04f29 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 20:59:34 -0300 Subject: [PATCH 40/46] Update README.md.gotmpl --- README.md.gotmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md.gotmpl b/README.md.gotmpl index 333f966..943bb75 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -51,7 +51,6 @@ kubectl create secret generic oidc-secrets \ --from-literal=HEADPLANE_OIDC__CLIENT_ID=your-headplane-oidc-client-id \ --from-literal=HEADSCALE_OIDC__CLIENT_SECRET=your-headscale-oidc-client-secret \ --from-literal=HEADSCALE_OIDC__CLIENT_ID=your-headscale-oidc-client-id \ - --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY=this-is-optional-and-autogenerated-by-chart -n ``` From 319360c111d7b87c259831286d6d999ba0710109 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:00:26 -0300 Subject: [PATCH 41/46] update readme values --- README.md | 4 ++-- values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc6dd60..cb0d65b 100644 --- a/README.md +++ b/README.md @@ -48,14 +48,14 @@ helm uninstall headplane | headplane.config.server.port | int | `3000` | | | headplane.envFrom | list | `[]` | | | headplane.image | string | `"ghcr.io/tale/headplane:0.6.0"` | | -| headplane.oidc.client_id | string | `"REPLACE_IT_WITH_YOUR_OIDC_CLIENT_ID_FOR_HEADPLANE"` | | +| headplane.oidc.client_id | string | `"PLACEHOLDER_USE_SECRET"` | | | headplane.oidc.disable_api_key_login | bool | `true` | | | headplane.oidc.enabled | bool | `false` | | | headplane.oidc.issuer | string | `"https://your-oidc-issuer-url.com"` | | | headplane.oidc.redirect_uri | string | `"https://your-headplane-admin-domain.com/admin/oidc/callback"` | | | headplane.oidc.secret_name | string | `"oidc-secrets"` | | | headplane.oidc.token_endpoint_auth_method | string | `"client_secret_post"` | | -| headscale.acl | string | `""` | | +| headscale.acl | string | `"{\n \"acls\": []\n}\n"` | | | headscale.config.database.debug | bool | `false` | | | headscale.config.database.sqlite.path | string | `"/etc/headscale/db.sqlite"` | | | headscale.config.database.type | string | `"sqlite"` | | diff --git a/values.yaml b/values.yaml index d01c058..17b873b 100644 --- a/values.yaml +++ b/values.yaml @@ -14,7 +14,7 @@ headplane: # OIDC redirect URI redirect_uri: "https://your-headplane-admin-domain.com/admin/oidc/callback" # OIDC client ID - client_id: "PLACEHOLDER_USE_SECRET_BELOW" #USE secret below instead + client_id: "PLACEHOLDER_USE_SECRET" #USE secret below instead # Name of the secret containing OIDC credentials secret_name: "oidc-secrets" config: From 553ba8066c1440e1b3cefb0efc0a41d5798a913f Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:10:50 -0300 Subject: [PATCH 42/46] Update statefulset-headplane.yaml --- templates/statefulset-headplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index ac8ef8e..0614361 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -51,7 +51,7 @@ spec: name: {{ .Values.headplane.oidc.secret_name }} - secretRef: name: headscale-api-token - optional: true + # optional: true {{- end }} {{- with .Values.headplane.envFrom }} {{- toYaml . | nindent 10 }} From 27069082996792744e0a1c3fdb2b1b163ab6a9bb Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:19:18 -0300 Subject: [PATCH 43/46] add placeholder headscale api token --- .github/workflows/publish-helm-chart.yml | 2 +- templates/job.yaml | 8 +++++--- templates/secret-headscale-api-token.yaml | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 templates/secret-headscale-api-token.yaml diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index 53addd3..03b386c 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -280,7 +280,7 @@ jobs: # # Verify API key replacement if OIDC is enabled # if kubectl get secret headplane-config -n "$namespace" >/dev/null 2>&1; then # CONFIG_CONTENT=$(kubectl get secret headplane-config -n "$namespace" -o jsonpath='{.data.config\.yaml}' | base64 -d) - + # # Only verify API key replacement if OIDC is enabled # if echo "$CONFIG_CONTENT" | grep -q "oidc:"; then # echo "[$test_name] Verifying API key replacement in headplane-config secret..." diff --git a/templates/job.yaml b/templates/job.yaml index 27300b5..b531003 100644 --- a/templates/job.yaml +++ b/templates/job.yaml @@ -29,7 +29,7 @@ spec: exit 0 fi - echo "Secret not found. Generating Headscale API token..." + echo "Generating Headscale API token..." TOKEN=$(kubectl -n {{ .Release.Namespace }} exec -i headplane-0 -c headscale -- headscale apikeys create -e 100y) if [ -z "$TOKEN" ]; then @@ -37,5 +37,7 @@ spec: exit 1 fi - echo "Creating Kubernetes Secret..." - kubectl create secret generic headscale-api-token --from-literal=HEADPLANE_OIDC__HEADSCALE_API_KEY="$TOKEN" -n {{ .Release.Namespace }} + echo "Updating headscale-api-token secret with generated API token..." + kubectl patch secret headscale-api-token -n {{ .Release.Namespace }} -p="{\"data\":{\"HEADPLANE_OIDC__HEADSCALE_API_KEY\":\"$(echo -n "$TOKEN" | base64)\"}}" + + echo "Successfully updated headscale-api-token secret with real API token" diff --git a/templates/secret-headscale-api-token.yaml b/templates/secret-headscale-api-token.yaml new file mode 100644 index 0000000..3cf9a2d --- /dev/null +++ b/templates/secret-headscale-api-token.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: headscale-api-token + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + HEADPLANE_OIDC__HEADSCALE_API_KEY: "placeholder-token-will-be-replaced-by-job" \ No newline at end of file From 7bbc4979c32e992beb7e73e93976b3259de23fd3 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:33:15 -0300 Subject: [PATCH 44/46] Update job.yaml --- templates/job.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/templates/job.yaml b/templates/job.yaml index b531003..b5a0a9a 100644 --- a/templates/job.yaml +++ b/templates/job.yaml @@ -23,10 +23,21 @@ spec: sleep 1 done - echo "Checking if Secret 'headscale-api-token' exists..." + echo "Checking if API token needs to be generated..." if kubectl get secret headscale-api-token -n {{ .Release.Namespace }} >/dev/null 2>&1; then - echo "Secret already exists. Skipping token generation." - exit 0 + CURRENT_TOKEN=$(kubectl get secret headscale-api-token -n {{ .Release.Namespace }} -o jsonpath='{.data.HEADPLANE_OIDC__HEADSCALE_API_KEY}' | base64 -d) + echo "Current token value: '$CURRENT_TOKEN'" + echo "Current token length: ${#CURRENT_TOKEN}" + echo "Expected placeholder: 'placeholder-token-will-be-replaced-by-job'" + echo "Expected length: 42" + if [[ "$CURRENT_TOKEN" != "placeholder-token-will-be-replaced-by-job" ]]; then + echo "Real API token already exists. Skipping token generation." + exit 0 + else + echo "Placeholder token found. Generating real API token..." + fi + else + echo "Secret not found. Generating API token..." fi echo "Generating Headscale API token..." From 2e9d46ec25104e79252d2c5acb099459b4710121 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:40:19 -0300 Subject: [PATCH 45/46] Update statefulset-headplane.yaml --- templates/statefulset-headplane.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 0614361..719dccf 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -39,11 +39,9 @@ spec: - /bin/sh - -c - | - {{- if .Values.headplane.oidc.enabled }} echo "Waiting for headscale-generate-token job to complete..." kubectl wait --for=condition=complete job/headscale-generate-token -n {{ .Release.Namespace }} --timeout=300s echo "Job completed successfully, starting headplane..." - {{- end }} exec node /app/build/server/index.js envFrom: {{- if .Values.headplane.oidc.enabled }} From b478b4e42ba06f28ba13c5ca4a37ff07320b9082 Mon Sep 17 00:00:00 2001 From: antoniolago <45375617+antoniolago@users.noreply.github.com> Date: Sun, 22 Jun 2025 21:43:47 -0300 Subject: [PATCH 46/46] Update statefulset-headplane.yaml --- templates/statefulset-headplane.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/statefulset-headplane.yaml b/templates/statefulset-headplane.yaml index 719dccf..790d3fd 100644 --- a/templates/statefulset-headplane.yaml +++ b/templates/statefulset-headplane.yaml @@ -44,11 +44,11 @@ spec: echo "Job completed successfully, starting headplane..." exec node /app/build/server/index.js envFrom: + - secretRef: + name: headscale-api-token {{- if .Values.headplane.oidc.enabled }} - secretRef: name: {{ .Values.headplane.oidc.secret_name }} - - secretRef: - name: headscale-api-token # optional: true {{- end }} {{- with .Values.headplane.envFrom }}