Fail smoke discovery on GitHub API and pipeline errors

.github/workflows/smoke.yml

@@ -39,6 +39,9 @@ jobs:
 
       - name: Generate suites to run
         id: suites
+        shell: bash
+        env:
+          SMOKE_TEST_REPO: ${{ vars.SMOKE_TEST_REPO || 'dependabot/smoke-tests' }}
         run: |
           # Write the changes.json file which is an array of paths
           echo "${{ toJson(steps.changes.outputs.changes) }}" > changes.json
@@ -48,9 +51,8 @@ jobs:
           jq -c --argjson changes "$(cat changes.json)" '[.[] | select(.core as $p | $changes | index($p))]' .github/smoke-matrix.json > filtered.json
           cat filtered.json
 
-          # Curl the smoke-test tests directory to get a list of tests to run
-          URL=https://api.github.com/repos/${{ vars.SMOKE_TEST_REPO || 'dependabot/smoke-tests' }}/contents/tests?ref=${{ env.SMOKE_TEST_BRANCH }}
-          curl $URL > tests.json
+          # Fetch the smoke-test directory using the job's read-only GitHub token.
+          gh api "repos/${SMOKE_TEST_REPO}/contents/tests?ref=${SMOKE_TEST_BRANCH}" > tests.json
 
           # Select the names that match smoke-$test*.yaml, where $test is the .text value from filtered.json
           # We end up with, for example: [{core: "bundler", ecosystem: "rubygems", name: "smoke-bundler.yaml"}]

NEW_ECOSYSTEMS.md

@@ -381,6 +381,10 @@ This will be automatically picked up by the dry-run script and give you higher r
 
 Add smoke tests to the [dependabot/smoke-tests](https://github.com/dependabot/smoke-tests) repository. See the repository documentation for detailed instructions on creating and running smoke tests for your ecosystem.
 
+The CI Smoke workflow uses its read-only `GITHUB_TOKEN` to discover tests. GitHub API or suite-generation errors fail discovery instead of reporting success with an empty test matrix.
+
+Run `ruby script/test-smoke-discovery` to verify discovery success, an empty selection, API failure, and suite-generation failure locally. It requires Bash and jq, but no network access or token.
+
 ## Phase 4: Coordination with Dependabot Team
 
 Since ecosystem support requires changes to Dependabot's API and deployment infrastructure, you'll need to coordinate with the Dependabot team:

script/test-smoke-discovery

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Run with ruby script/test-smoke-discovery; requires bash and jq, but no network or token.
+require "fileutils"
+require "json"
+require "open3"
+require "tmpdir"
+require "yaml"
+
+root = File.expand_path("..", __dir__)
+workflow = YAML.load_file(File.join(root, ".github/workflows/smoke.yml"))
+step = workflow.fetch("jobs").fetch("discover").fetch("steps").find { |entry| entry["id"] == "suites" }
+shell = ["bash", "--noprofile", "--norc", "-e"]
+shell += ["-o", "pipefail"] if step["shell"] == "bash"
+
+run = step.fetch("run")
+expression = "${{ toJson(steps.changes.outputs.changes) }}"
+
+response = [
+  { "name" => "smoke-gradle.yaml", "sha" => "gradle-sha" },
+  { "name" => "smoke-npm.yaml", "sha" => "npm-sha" },
+  { "name" => "smoke-npm-rules.yaml", "sha" => "npm-rules-sha" },
+  { "name" => "smoke-bundler.yaml", "sha" => "unselected-sha" }
+]
+expected = [
+  { "core" => "gradle", "ecosystem" => "gradle", "name" => "smoke-gradle.yaml", "sha" => "gradle-sha" },
+  { "core" => "npm_and_yarn", "ecosystem" => "npm", "name" => "smoke-npm.yaml", "sha" => "npm-sha" },
+  { "core" => "npm_and_yarn", "ecosystem" => "npm", "name" => "smoke-npm-rules.yaml", "sha" => "npm-rules-sha" }
+]
+selected = %w(npm_and_yarn gradle)
+cases = [
+  ["complete success", selected, response, 0, expected],
+  ["legitimate empty selection", [], [], 0, []],
+  ["API failure with a valid body", selected, response, 1, nil],
+  ["suite-generation failure", selected, { "unexpected" => "directory response" }, 0, nil]
+]
+
+cases.each do |name, changes, body, api_status, suites|
+  Dir.mktmpdir("smoke-discovery-") do |dir|
+    FileUtils.mkdir_p(File.join(dir, ".github"))
+    FileUtils.cp(File.join(root, ".github/smoke-matrix.json"), File.join(dir, ".github"))
+    File.write(File.join(dir, "response.json"), JSON.generate(body))
+    output = File.join(dir, "output")
+    File.write(output, "")
+    env = {
+      "GITHUB_OUTPUT" => output,
+      "GH_TOKEN" => "test-token",
+      "SMOKE_TEST_REPO" => "test/smoke-tests",
+      "SMOKE_TEST_BRANCH" => "test-branch",
+      "API_STATUS" => api_status.to_s
+    }
+    stub = <<~BASH
+      gh() {
+        cat response.json
+        return "$API_STATUS"
+      }
+    BASH
+    script = stub + run.gsub(expression) { JSON.generate(JSON.generate(changes)) }
+    stdout, stderr, status = Open3.capture3(env, *shell, "-c", script, chdir: dir)
+    published = File.read(output)
+    if suites.nil?
+      raise "#{name}: expected failure without output\n#{stdout}\n#{stderr}" if status.success? || !published.empty?
+    else
+      raise "#{name}: discovery failed\n#{stdout}\n#{stderr}" unless status.success?
+      unless published == "suites=#{JSON.generate(suites)}\n"
+        raise "#{name}: unexpected suites output #{published.inspect}"
+      end
+    end
+    puts "PASS: #{name}"
+  end
+end