diff --git a/.github/workflows/e2e_manual.yml b/.github/workflows/e2e_manual.yml index 4fdf950e644..1e6ed9701dc 100644 --- a/.github/workflows/e2e_manual.yml +++ b/.github/workflows/e2e_manual.yml @@ -19,6 +19,7 @@ on: - gpu - imagepuller-auth - imagestore + - insecure - kds-pcs-downtime - memdump - multi-runtime-class diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index b3021cda343..fa07f18a276 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -96,6 +96,7 @@ subcommands.`, cmd.Flags().Bool("inject-image-store", false, "inject an ephemeral storage device to pull images onto instead of into memory") cmd.Flags().Bool("insecure-enable-debug-shell-access", false, "enable the debug shell service in the pod CVM to get access from container to guest VM") cmd.Flags().StringP("output", "o", "", "output file for generated YAML") + cmd.Flags().Bool("INSECURE", false, "allow generation for insecure (non-CC) runtimes (also requires the CONTRAST_ALLOW_INSECURE_RUNTIMES environment variable to be set)") must(cmd.MarkFlagFilename("policy", "rego")) must(cmd.MarkFlagFilename("settings", "json")) must(cmd.MarkFlagFilename("manifest", "json")) @@ -147,6 +148,10 @@ func runGenerate(cmd *cobra.Command, args []string) error { usedPlatforms.Add(flags.referenceValuesPlatform) } + if err := validateInsecurePlatforms(usedPlatforms, flags.allowInsecureRuntimes); err != nil { + return err + } + // generate a manifest by checking if a manifest exists and using that, // or otherwise using a default. var mnf *manifest.Manifest @@ -284,17 +289,17 @@ func runGenerate(cmd *cobra.Command, args []string) error { return nil } -// mapCCWorkloads applies the given function to all workloads with the 'contrast-cc' runtime class. +// mapContrastWorkloads applies the given function to all workloads with the 'contrast-cc' or 'contrast-insecure' runtime class. // The callback receives an apply configuration together with the file path and index the unstructured object has in the file map. // Changes to the apply configuration are not applied to the original unstructured object. -func mapCCWorkloads(fileMap map[string][]*unstructured.Unstructured, f func(res any, path string, idx int) (any, error)) error { +func mapContrastWorkloads(fileMap map[string][]*unstructured.Unstructured, f func(res any, path string, idx int) (any, error)) error { for path, resources := range fileMap { for idx, r := range resources { applyConfig, err := kuberesource.UnstructuredToApplyConfiguration(r) if err != nil { continue } - if !isCCWorkload(applyConfig) { + if !isContrastWorkload(applyConfig) { continue } changed, err := f(applyConfig, path, idx) @@ -313,9 +318,9 @@ func mapCCWorkloads(fileMap map[string][]*unstructured.Unstructured, f func(res return nil } -func isCCWorkload(resource any) (ret bool) { +func isContrastWorkload(resource any) (ret bool) { kuberesource.MapPodSpec(resource, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec != nil && spec.RuntimeClassName != nil && strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if kuberesource.IsContrastPod(spec) { ret = true } return spec @@ -336,10 +341,20 @@ func isCoordinator(resource any) bool { return false } +func patchCoordinatorAllowInsecure(resource any) { + r, ok := resource.(*applyappsv1.StatefulSetApplyConfiguration) + if !ok || !isCoordinator(resource) { + return + } + if len(r.Spec.Template.Spec.Containers) > 0 { + r.Spec.Template.Spec.Containers[0].WithEnv(kuberesource.NewEnvVar("CONTRAST_ALLOW_INSECURE", "1")) + } +} + func runVerifiers(fileMap map[string][]*unstructured.Unstructured, verifiers []verifier.Verifier) error { var findings error for _, v := range verifiers { - _ = mapCCWorkloads(fileMap, func(res any, path string, idx int) (any, error) { + _ = mapContrastWorkloads(fileMap, func(res any, path string, idx int) (any, error) { if err := v.Verify(res); err != nil { findings = errors.Join(findings, fmt.Errorf("failed to verify resource %q in file %q: %w", fileMap[path][idx].GetName(), path, err)) } @@ -406,7 +421,7 @@ func extractTargets(paths []string, configFile io.Writer, logger *slog.Logger) ( applyConfig, err := kuberesource.UnstructuredToApplyConfiguration(object) if err != nil { logger.Warn("Could not convert resource into ApplyConfiguration", "path", path, "err", err) - } else if isCCWorkload(applyConfig) { + } else if isContrastWorkload(applyConfig) { containsCC = true if isCoordinator(applyConfig) { r, ok := applyConfig.(*applyappsv1.StatefulSetApplyConfiguration) @@ -421,7 +436,7 @@ func extractTargets(paths []string, configFile io.Writer, logger *slog.Logger) ( } } if len(fileMap) == 0 { - return nil, "", fmt.Errorf("no .yml/.yaml files with 'contrast-cc' runtime found") + return nil, "", fmt.Errorf("no .yml/.yaml files with 'contrast-cc' or 'contrast-insecure' runtime found") } extraData, err := kuberesource.EncodeUnstructured(extraResources) @@ -454,7 +469,7 @@ func generatePolicies(ctx context.Context, flags *generateFlags, fileMap map[str } }() - return mapCCWorkloads(fileMap, func(res any, path string, idx int) (any, error) { + return mapContrastWorkloads(fileMap, func(res any, path string, idx int) (any, error) { initdataAnno, err := runner.Run(ctx, res, extraPath, logger) if err != nil { return nil, fmt.Errorf("failed to generate policy for %q in %q: %w", fileMap[path][idx].GetName(), path, err) @@ -496,7 +511,7 @@ func patchTargets(fileMap map[string][]*unstructured.Unstructured, imageReplacem return fmt.Errorf("parsing release image definitions %s: %w", ReleaseImageReplacements, err) } } - return mapCCWorkloads(fileMap, func(res any, _ string, _ int) (any, error) { + return mapContrastWorkloads(fileMap, func(res any, _ string, _ int) (any, error) { if flags.insecureEnableDebugShell { if _, err := kuberesource.AddDebugShell(res, kuberesource.DebugShell()); err != nil { return nil, fmt.Errorf("injecting debug shell container: %w", err) @@ -515,6 +530,9 @@ func patchTargets(fileMap map[string][]*unstructured.Unstructured, imageReplacem if flags.injectImageStore { kuberesource.AddImageStore([]any{res}) } + if flags.allowInsecureRuntimes { + patchCoordinatorAllowInsecure(res) + } kuberesource.PatchImages([]any{res}, replacements) @@ -556,6 +574,19 @@ func injectServiceMesh(resource any) error { return nil } +func validateInsecurePlatforms(usedPlatforms kuberesource.PlatformCollection, allowInsecure bool) error { + if !slices.ContainsFunc(usedPlatforms.Platforms(), platforms.IsInsecure) { + return nil + } + if !allowInsecure { + return fmt.Errorf("insecure runtime platforms detected but --INSECURE flag not set") + } + if os.Getenv("CONTRAST_ALLOW_INSECURE_RUNTIMES") == "" { + return fmt.Errorf("insecure runtime platforms detected but CONTRAST_ALLOW_INSECURE_RUNTIMES environment variable not set") + } + return nil +} + func validateOutputFile(outputFile string) error { if outputFile == "" { return nil @@ -683,7 +714,17 @@ func patchRuntimeClassName(defaultRuntimeHandler string) func(*applycorev1.PodSp if spec == nil || spec.RuntimeClassName == nil { return spec, nil } - if *spec.RuntimeClassName == "kata-cc-isolation" || *spec.RuntimeClassName == "contrast-cc" { + if *spec.RuntimeClassName == "kata-cc-isolation" || *spec.RuntimeClassName == "contrast-cc" || *spec.RuntimeClassName == "contrast-insecure" { + // Only allow the bare runtime class names if the default runtime handler is compatible. + // For example, `contrast-cc` should only resolve when `--reference-values` is set to a CC-enabled platform, + // and `contrast-insecure` should only resolve when `--reference-values` is set to an insecure platform. + if *spec.RuntimeClassName == "contrast-insecure" && !strings.HasPrefix(defaultRuntimeHandler, "contrast-insecure-") { + return nil, fmt.Errorf("bare 'contrast-insecure' runtime class requires --reference-values to be set to an insecure platform") + } + if (*spec.RuntimeClassName == "contrast-cc" || *spec.RuntimeClassName == "kata-cc-isolation") && + strings.HasPrefix(defaultRuntimeHandler, "contrast-insecure-") { + return nil, fmt.Errorf("bare %q runtime class is incompatible with insecure --reference-values platform %q", *spec.RuntimeClassName, defaultRuntimeHandler) + } spec.RuntimeClassName = &defaultRuntimeHandler if kuberesource.PodSpecRequiresGPU(spec) { platform, err := platforms.FromRuntimeClassString(*spec.RuntimeClassName) @@ -698,7 +739,7 @@ func patchRuntimeClassName(defaultRuntimeHandler string) func(*applycorev1.PodSp } return spec, nil } - if !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc-") { + if !kuberesource.IsContrastPod(spec) { return spec, nil } overridePlatform, err := platforms.FromRuntimeClassString(*spec.RuntimeClassName) @@ -870,6 +911,7 @@ type generateFlags struct { skipServiceMesh bool injectImageStore bool insecureEnableDebugShell bool + allowInsecureRuntimes bool outputFile string } @@ -967,6 +1009,10 @@ func parseGenerateFlags(cmd *cobra.Command) (*generateFlags, error) { if err != nil { return nil, err } + allowInsecureRuntimes, err := cmd.Flags().GetBool("INSECURE") + if err != nil { + return nil, err + } outputFile, err := cmd.Flags().GetString("output") if err != nil { return nil, err @@ -992,6 +1038,7 @@ func parseGenerateFlags(cmd *cobra.Command) (*generateFlags, error) { skipServiceMesh: skipServiceMesh, injectImageStore: injectImageStore, insecureEnableDebugShell: insecureEnableDebugShell, + allowInsecureRuntimes: allowInsecureRuntimes, outputFile: outputFile, }, nil } diff --git a/cli/cmd/generate_test.go b/cli/cmd/generate_test.go index db05aa5cf5b..ad1d1844835 100644 --- a/cli/cmd/generate_test.go +++ b/cli/cmd/generate_test.go @@ -4,6 +4,7 @@ package cmd import ( + "os" "testing" "github.com/edgelesssys/contrast/internal/kuberesource" @@ -77,6 +78,40 @@ spec: }, want: []platforms.Platform{platforms.MetalQEMUSNP, platforms.MetalQEMUTDX}, }, + "single insecure snp": { + yaml: map[string]string{ + "file1.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: p1 +spec: + runtimeClassName: contrast-insecure-metal-qemu-snp +`, + }, + want: []platforms.Platform{platforms.MetalQEMUSNPInsecure}, + }, + "mixed cc and insecure": { + yaml: map[string]string{ + "file1.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: p1 +spec: + runtimeClassName: contrast-cc-metal-qemu-snp +`, + "file2.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: p2 +spec: + runtimeClassName: contrast-insecure-metal-qemu-tdx +`, + }, + want: []platforms.Platform{platforms.MetalQEMUSNP, platforms.MetalQEMUTDXInsecure}, + }, } for name, tc := range testCases { @@ -99,33 +134,73 @@ spec: } func TestPatchRuntimeClassName(t *testing.T) { - defaultHandler := "contrast-cc-metal-qemu-snp" + ccHandler := "contrast-cc-metal-qemu-snp" + insecureHandler := "contrast-insecure-metal-qemu-snp" testCases := map[string]struct { - initial string - want string - updateHandler bool + defaultHandler string + initial string + want string + updateHandler bool + wantErr bool }{ "no runtime class": { - initial: "", - want: "", + defaultHandler: ccHandler, + initial: "", + want: "", }, "irrelevant class": { - initial: "runc", - want: "runc", + defaultHandler: ccHandler, + initial: "runc", + want: "runc", }, "generic kata": { - initial: "kata-cc-isolation", - want: defaultHandler, + defaultHandler: ccHandler, + initial: "kata-cc-isolation", + want: ccHandler, }, "generic contrast": { - initial: "contrast-cc", - want: defaultHandler, + defaultHandler: ccHandler, + initial: "contrast-cc", + want: ccHandler, }, "specific contrast-cc-metal-qemu-tdx": { - initial: "contrast-cc-metal-qemu-tdx", - want: "contrast-cc-metal-qemu-tdx", - updateHandler: true, + defaultHandler: ccHandler, + initial: "contrast-cc-metal-qemu-tdx", + want: "contrast-cc-metal-qemu-tdx", + updateHandler: true, + }, + "generic contrast-insecure with insecure handler": { + defaultHandler: insecureHandler, + initial: "contrast-insecure", + want: insecureHandler, + }, + "generic contrast-insecure with cc handler errors": { + defaultHandler: ccHandler, + initial: "contrast-insecure", + wantErr: true, + }, + "generic contrast-cc with insecure handler errors": { + defaultHandler: insecureHandler, + initial: "contrast-cc", + wantErr: true, + }, + "generic kata with insecure handler errors": { + defaultHandler: insecureHandler, + initial: "kata-cc-isolation", + wantErr: true, + }, + "specific contrast-insecure-metal-qemu-snp": { + defaultHandler: ccHandler, + initial: "contrast-insecure-metal-qemu-snp", + want: "contrast-insecure-metal-qemu-snp", + updateHandler: true, + }, + "specific contrast-insecure-metal-qemu-tdx": { + defaultHandler: ccHandler, + initial: "contrast-insecure-metal-qemu-tdx", + want: "contrast-insecure-metal-qemu-tdx", + updateHandler: true, }, } @@ -141,12 +216,16 @@ func TestPatchRuntimeClassName(t *testing.T) { tc.want = getHandler(t, tc.want) } - patch := patchRuntimeClassName(tc.want) + patch := patchRuntimeClassName(tc.defaultHandler) spec := applycorev1.PodSpec() if tc.initial != "" { spec.WithRuntimeClassName(tc.initial) } _, err := patch(spec) + if tc.wantErr { + require.Error(t, err) + return + } require.NoError(t, err) if tc.want == "" { assert.Nil(t, spec.RuntimeClassName) @@ -158,13 +237,119 @@ func TestPatchRuntimeClassName(t *testing.T) { } t.Run("nil spec returns nil", func(t *testing.T) { - patch := patchRuntimeClassName(defaultHandler) + patch := patchRuntimeClassName(ccHandler) result, err := patch(nil) require.NoError(t, err) assert.Nil(t, result) }) } +func TestIsContrastWorkload(t *testing.T) { + testCases := map[string]struct { + runtimeClass string + want bool + }{ + "no runtime class": { + runtimeClass: "", + want: false, + }, + "non-contrast runtime class": { + runtimeClass: "foobar", + want: false, + }, + "contrast-cc": { + runtimeClass: "contrast-cc", + want: true, + }, + "contrast-cc-metal-qemu-snp": { + runtimeClass: "contrast-cc-metal-qemu-snp", + want: true, + }, + "contrast-insecure": { + runtimeClass: "contrast-insecure", + want: true, + }, + "contrast-insecure-metal-qemu-snp": { + runtimeClass: "contrast-insecure-metal-qemu-snp", + want: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + spec := applycorev1.PodSpec() + if tc.runtimeClass != "" { + spec.WithRuntimeClassName(tc.runtimeClass) + } + pod := applycorev1.Pod("test", "default").WithSpec(spec) + assert.Equal(t, tc.want, isContrastWorkload(pod)) + }) + } +} + +func TestValidateInsecurePlatforms(t *testing.T) { + testCases := map[string]struct { + platforms []platforms.Platform + allowInsecure bool + setEnv bool + wantErr bool + wantErrContain string + }{ + "no insecure platforms": { + platforms: []platforms.Platform{platforms.MetalQEMUSNP}, + wantErr: false, + }, + "insecure without flag": { + platforms: []platforms.Platform{platforms.MetalQEMUSNPInsecure}, + allowInsecure: false, + wantErr: true, + wantErrContain: "--INSECURE flag not set", + }, + "insecure with flag but no env": { + platforms: []platforms.Platform{platforms.MetalQEMUSNPInsecure}, + allowInsecure: true, + setEnv: false, + wantErr: true, + wantErrContain: "CONTRAST_ALLOW_INSECURE_RUNTIMES", + }, + "insecure with flag and env": { + platforms: []platforms.Platform{platforms.MetalQEMUSNPInsecure}, + allowInsecure: true, + setEnv: true, + wantErr: false, + }, + "mixed with flag and env": { + platforms: []platforms.Platform{platforms.MetalQEMUSNP, platforms.MetalQEMUTDXInsecure}, + allowInsecure: true, + setEnv: true, + wantErr: false, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.setEnv { + t.Setenv("CONTRAST_ALLOW_INSECURE_RUNTIMES", "true") + } else { + os.Unsetenv("CONTRAST_ALLOW_INSECURE_RUNTIMES") + } + + collection := kuberesource.PlatformCollection{} + for _, p := range tc.platforms { + collection.Add(p) + } + + err := validateInsecurePlatforms(collection, tc.allowInsecure) + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrContain) + } else { + require.NoError(t, err) + } + }) + } +} + func getHandler(t *testing.T, name string) string { t.Helper() platform, err := platforms.FromRuntimeClassString(name) diff --git a/cli/cmd/policies.go b/cli/cmd/policies.go index 2c7d03c70be..a559bb6cb53 100644 --- a/cli/cmd/policies.go +++ b/cli/cmd/policies.go @@ -18,7 +18,7 @@ import ( ) func manipulateInitdata(fileMap map[string][]*unstructured.Unstructured, manipulators ...func(*initdata.Initdata) error) error { - return mapCCWorkloads(fileMap, func(res any, path string, _ int) (resource any, retErr error) { + return mapContrastWorkloads(fileMap, func(res any, path string, _ int) (resource any, retErr error) { return kuberesource.MapPodSpecWithMeta(res, func(meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { if meta == nil { return meta, spec @@ -60,7 +60,7 @@ func manipulateInitdata(fileMap map[string][]*unstructured.Unstructured, manipul func policiesFromKubeResources(fileMap map[string][]*unstructured.Unstructured) ([]deployment, error) { var deployments []deployment - if err := mapCCWorkloads(fileMap, func(res any, path string, idx int) (any, error) { + if err := mapContrastWorkloads(fileMap, func(res any, path string, idx int) (any, error) { name := fileMap[path][idx].GetName() namespace := orDefault(fileMap[path][idx].GetNamespace(), "default") gvk := fileMap[path][idx].GetObjectKind().GroupVersionKind() diff --git a/cli/cmd/verify.go b/cli/cmd/verify.go index 5831c73cd1d..64500e0eba2 100644 --- a/cli/cmd/verify.go +++ b/cli/cmd/verify.go @@ -48,6 +48,7 @@ all policies, and the certificates of the Coordinator certificate authority.`, cmd.Flags().StringP("manifest", "m", manifestFilename, "path to manifest (.json) file") cmd.Flags().StringP("coordinator", "c", "", "endpoint the coordinator can be reached at") must(cobra.MarkFlagRequired(cmd.Flags(), "coordinator")) + cmd.Flags().Bool("INSECURE", false, "allow verification of insecure (non-CC) deployments (also requires the CONTRAST_ALLOW_INSECURE_RUNTIMES environment variable to be set)") return cmd } @@ -69,6 +70,19 @@ func runVerify(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to read manifest file: %w", err) } + var mnfst manifest.Manifest + if err := json.Unmarshal(manifestBytes, &mnfst); err != nil { + return fmt.Errorf("unmarshalling manifest: %w", err) + } + if mnfst.AllowInsecure() { + if !flags.allowInsecureRuntimes { + return fmt.Errorf("manifest contains insecure platforms but --INSECURE flag not set") + } + if os.Getenv("CONTRAST_ALLOW_INSECURE_RUNTIMES") == "" { + return fmt.Errorf("manifest contains insecure platforms but CONTRAST_ALLOW_INSECURE_RUNTIMES environment variable not set") + } + } + kdsDir, err := cachedir("kds") if err != nil { return fmt.Errorf("getting cache dir: %w", err) @@ -130,9 +144,10 @@ func runVerify(cmd *cobra.Command, _ []string) error { } type verifyFlags struct { - manifestPath string - coordinator string - workspaceDir string + manifestPath string + coordinator string + workspaceDir string + allowInsecureRuntimes bool } func parseVerifyFlags(cmd *cobra.Command) (*verifyFlags, error) { @@ -148,6 +163,10 @@ func parseVerifyFlags(cmd *cobra.Command) (*verifyFlags, error) { if err != nil { return nil, err } + allowInsecureRuntimes, err := cmd.Flags().GetBool("INSECURE") + if err != nil { + return nil, err + } if workspaceDir != "" { // Prepend default path with workspaceDir @@ -157,9 +176,10 @@ func parseVerifyFlags(cmd *cobra.Command) (*verifyFlags, error) { } return &verifyFlags{ - manifestPath: manifestPath, - coordinator: coordinator, - workspaceDir: workspaceDir, + manifestPath: manifestPath, + coordinator: coordinator, + workspaceDir: workspaceDir, + allowInsecureRuntimes: allowInsecureRuntimes, }, nil } diff --git a/cli/genpolicy/genpolicy.go b/cli/genpolicy/genpolicy.go index 258c068730b..774d2d1a13d 100644 --- a/cli/genpolicy/genpolicy.go +++ b/cli/genpolicy/genpolicy.go @@ -55,6 +55,7 @@ func New(rulesPath, settingsPath, cachePath string, bin []byte) (*Runner, error) func (r *Runner) Run(ctx context.Context, res any, extraPath string, logger *slog.Logger) (string, error) { args := []string{ "--runtime-class-names=contrast-cc", + "--runtime-class-names=contrast-insecure", "--rego-rules-path=" + r.rulesPath, "--json-settings-path=" + r.settingsPath, "--layers-cache-file-path=" + r.cachePath, diff --git a/cli/verifier/image_ref_valid.go b/cli/verifier/image_ref_valid.go index 85b54289cdc..576746961e6 100644 --- a/cli/verifier/image_ref_valid.go +++ b/cli/verifier/image_ref_valid.go @@ -25,7 +25,8 @@ func (v *ImageRefValid) Verify(toVerify any) error { kuberesource.MapPodSpec(toVerify, func( spec *applycorev1.PodSpecApplyConfiguration, ) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !kuberesource.IsContrastPod(spec) { + // Non-Contrast pods are not subject to this verification. return spec } diff --git a/cli/verifier/no_shared_fs_mount.go b/cli/verifier/no_shared_fs_mount.go index 2ffb62d6f05..478c3367566 100644 --- a/cli/verifier/no_shared_fs_mount.go +++ b/cli/verifier/no_shared_fs_mount.go @@ -6,7 +6,6 @@ package verifier import ( "errors" "fmt" - "strings" "github.com/edgelesssys/contrast/internal/kuberesource" @@ -25,8 +24,8 @@ func (v *NoSharedFSMount) Verify(toVerify any) error { // get all volume mounts that are referenced in containers isNonCC := false kuberesource.MapPodSpec(toVerify, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { - // this isn't a confidential pod so we don't need to check further + if !kuberesource.IsContrastPod(spec) { + // this isn't a Contrast pod so we don't need to check further isNonCC = true return spec } diff --git a/cli/verifier/runtimeclasses_exist.go b/cli/verifier/runtimeclasses_exist.go index c6dceee669b..f2a25732f55 100644 --- a/cli/verifier/runtimeclasses_exist.go +++ b/cli/verifier/runtimeclasses_exist.go @@ -6,7 +6,6 @@ package verifier import ( "errors" "fmt" - "strings" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/platforms" @@ -15,12 +14,12 @@ import ( applycorev1 "k8s.io/client-go/applyconfigurations/core/v1" ) -// RuntimeClassesExist verifies that all used contrast-cc -prefixed runtimeClassNames are valid. +// RuntimeClassesExist verifies that all used contrast-cc or contrast-insecure prefixed runtimeClassNames are valid. type RuntimeClassesExist struct { Command *cobra.Command } -// Verify verifies that all used contrast-cc -prefixed runtimeClassNames are valid. +// Verify verifies that all used contrast-cc or contrast-insecure prefixed runtimeClassNames are valid. func (r *RuntimeClassesExist) Verify(toVerify any) error { var collectedErrs error collectedMissingRuntimes := map[string]error{} @@ -31,14 +30,15 @@ func (r *RuntimeClassesExist) Verify(toVerify any) error { } kuberesource.MapPodSpec(toVerify, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil { + if !kuberesource.IsContrastPod(spec) { return spec } - if defaultRuntimeClass == "" && *spec.RuntimeClassName == "contrast-cc" { - collectedMissingRuntimes["contrast-cc"] = fmt.Errorf("no default platform was specified using --reference-values") - return spec - } - if !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc-") { + // Bare runtime class names (without hash suffix) are placeholders that + // get resolved during generate. They can't be parsed as platforms. + if *spec.RuntimeClassName == "contrast-cc" || *spec.RuntimeClassName == "contrast-insecure" { + if defaultRuntimeClass == "" { + collectedMissingRuntimes[*spec.RuntimeClassName] = fmt.Errorf("no default platform was specified using --reference-values") + } return spec } diff --git a/cli/verifier/versions_match.go b/cli/verifier/versions_match.go index 718b225c508..34ed57af3dc 100644 --- a/cli/verifier/versions_match.go +++ b/cli/verifier/versions_match.go @@ -34,7 +34,7 @@ func (v *VersionsMatch) Verify(toVerify any) error { meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration, ) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !kuberesource.IsContrastPod(spec) { return meta, spec } diff --git a/coordinator/internal/stateguard/credentials.go b/coordinator/internal/stateguard/credentials.go index c01abb8ee61..0a0489dcb48 100644 --- a/coordinator/internal/stateguard/credentials.go +++ b/coordinator/internal/stateguard/credentials.go @@ -15,6 +15,7 @@ import ( "github.com/edgelesssys/contrast/internal/atls" "github.com/edgelesssys/contrast/internal/attestation" "github.com/edgelesssys/contrast/internal/attestation/certcache" + "github.com/edgelesssys/contrast/internal/attestation/insecure" "github.com/edgelesssys/contrast/internal/attestation/snp" "github.com/edgelesssys/contrast/internal/attestation/tdx" "github.com/edgelesssys/contrast/internal/constants" @@ -96,6 +97,12 @@ func (c *Credentials) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.A logger.NewWithAttrs(logger.NewNamed(c.logger, "validator"), map[string]string{"reference-values": name}), &authInfo, name)) } + if state.Manifest().AllowInsecure() { + validators = append(validators, insecure.NewValidatorWithReportSetter( + logger.NewWithAttrs(logger.NewNamed(c.logger, "validator"), map[string]string{"reference-values": "insecure"}), + &authInfo, "insecure")) + } + serverCfg, err := atls.CreateAttestationServerTLSConfig(c.issuer, validators, c.attestationFailuresCounter) if err != nil { log.Error("Could not create TLS config", "error", err) diff --git a/coordinator/internal/stateguard/stateguard.go b/coordinator/internal/stateguard/stateguard.go index e2e9d31eca8..94ed3cf3125 100644 --- a/coordinator/internal/stateguard/stateguard.go +++ b/coordinator/internal/stateguard/stateguard.go @@ -52,6 +52,10 @@ var ( // ErrConcurrentUpdate is returned by state-modifying operations if the input oldState is not // the current state. This usually happens when a concurrent operation succeeded. ErrConcurrentUpdate = errors.New("coordinator state was updated concurrently") + + // ErrInsecureNotAllowed is returned when a manifest contains insecure platforms but the + // coordinator was not started with the allow-insecure flag. + ErrInsecureNotAllowed = errors.New("manifest contains insecure platforms, but the coordinator is not configured to allow them") ) // Guard manages the manifest state of Contrast. @@ -65,6 +69,9 @@ type Guard struct { logger *slog.Logger metrics metrics + // allowInsecure controls whether manifests with insecure platforms are accepted. + allowInsecure bool + clock clock.Clock } @@ -73,7 +80,10 @@ type metrics struct { } // New creates a new state Guard instance. -func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger) *Guard { +// +// If allowInsecure is true, the Guard will accept manifests that contain insecure platforms. +// Otherwise, setting such a manifest will be rejected with ErrInsecureNotAllowed. +func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger, allowInsecure bool) *Guard { manifestGeneration := promauto.With(reg).NewGauge(prometheus.GaugeOpts{ Subsystem: "contrast_coordinator", Name: "manifest_generation", @@ -82,8 +92,9 @@ func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger) *Gua manifestGeneration.Set(0) return &Guard{ - hist: hist, - logger: log.WithGroup("stateguard"), + hist: hist, + logger: log.WithGroup("stateguard"), + allowInsecure: allowInsecure, metrics: metrics{ manifestGeneration: manifestGeneration, }, @@ -271,6 +282,9 @@ func (g *Guard) UpdateState(_ context.Context, oldState *State, se *seedengine.S if err := json.Unmarshal(manifestBytes, &mnfst); err != nil { return nil, fmt.Errorf("unmarshaling manifest: %w", err) } + if !g.allowInsecure && mnfst.AllowInsecure() { + return nil, ErrInsecureNotAllowed + } policyMap := make(map[[history.HashSize]byte][]byte) for _, policy := range policies { policyHash, err := g.hist.SetPolicy(policy) diff --git a/coordinator/internal/stateguard/stateguard_test.go b/coordinator/internal/stateguard/stateguard_test.go index 475267952f3..c9843ace162 100644 --- a/coordinator/internal/stateguard/stateguard_test.go +++ b/coordinator/internal/stateguard/stateguard_test.go @@ -192,6 +192,37 @@ func TestResetState(t *testing.T) { require.ErrorIs(err, assert.AnError) } +func TestUpdateStateInsecure(t *testing.T) { + ctx := t.Context() + + _, insecureManifestBytes, policies := newInsecureManifest(t) + se := newSeedEngine(t) + + t.Run("rejected when allowInsecure is false", func(t *testing.T) { + require := require.New(t) + + store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()}) + hist := history.NewWithStore(slog.Default(), store) + g := New(hist, prometheus.NewRegistry(), slog.Default(), false) + + state, err := g.UpdateState(ctx, nil, se, insecureManifestBytes, policies) + require.ErrorIs(err, ErrInsecureNotAllowed) + require.Nil(state) + }) + + t.Run("accepted when allowInsecure is true", func(t *testing.T) { + require := require.New(t) + + store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()}) + hist := history.NewWithStore(slog.Default(), store) + g := New(hist, prometheus.NewRegistry(), slog.Default(), true) + + state, err := g.UpdateState(ctx, nil, se, insecureManifestBytes, policies) + require.NoError(err) + require.NotNil(state) + }) +} + func TestConcurrentUpdateState(t *testing.T) { ctx := t.Context() assert := assert.New(t) @@ -200,7 +231,7 @@ func TestConcurrentUpdateState(t *testing.T) { Store: aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()}), } hist := history.NewWithStore(slog.Default(), store) - guard := New(hist, prometheus.NewRegistry(), slog.Default()) + guard := New(hist, prometheus.NewRegistry(), slog.Default(), false) numWorkers := 20 @@ -303,7 +334,7 @@ func TestWatchHistory(t *testing.T) { notifications: make(chan []byte), } hist := history.NewWithStore(slog.Default(), store) - g := New(hist, prometheus.NewRegistry(), slog.Default()) + g := New(hist, prometheus.NewRegistry(), slog.Default(), false) _, manifestBytes, policies := newManifest(t) @@ -352,7 +383,7 @@ func TestWatchHistoryLateNotifications(t *testing.T) { notifications: make(chan []byte), } hist := history.NewWithStore(slog.Default(), store) - g := New(hist, prometheus.NewRegistry(), slog.Default()) + g := New(hist, prometheus.NewRegistry(), slog.Default(), false) _, manifestBytes, policies := newManifest(t) @@ -409,7 +440,7 @@ func TestBadStoreWatcherIsRestarted(t *testing.T) { store.storeUpdates.Store(&ch) hist := history.NewWithStore(slog.Default(), store) reg := prometheus.NewRegistry() - a := New(hist, reg, slog.Default()) + a := New(hist, reg, slog.Default(), false) clock := &waitingClock{ FakeClock: testingclock.NewFakeClock(time.Now()), afterCalls: make(chan struct{}, 1), @@ -502,7 +533,7 @@ func newTestGuard(t *testing.T) (*Guard, *prometheus.Registry) { store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()}) hist := history.NewWithStore(slog.Default(), store) reg := prometheus.NewRegistry() - return New(hist, reg, slog.Default()), reg + return New(hist, reg, slog.Default(), false), reg } func newManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) { @@ -542,6 +573,28 @@ func newManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) { return mnfst, mnfstBytes, [][]byte{policy} } +func newInsecureManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) { + t.Helper() + policy := []byte("=== SOME REGO HERE ===") + policyHash := sha256.Sum256(policy) + policyHashHex := manifest.NewHexString(policyHash[:]) + + mnfst := &manifest.Manifest{} + mnfst.Policies = map[manifest.HexString]manifest.PolicyEntry{ + policyHashHex: { + SANs: []string{"test"}, + WorkloadSecretID: "test2", + Role: manifest.RoleCoordinator, + }, + } + mnfst.ReferenceValues.SNP = []manifest.SNPReferenceValues{ + {Platform: "Metal-QEMU-SNP-Insecure"}, + } + mnfstBytes, err := json.Marshal(mnfst) + require.NoError(t, err) + return mnfst, mnfstBytes, [][]byte{policy} +} + func newSeedEngine(t *testing.T) *seedengine.SeedEngine { t.Helper() data := make([]byte, 32) diff --git a/coordinator/internal/userapi/userapi.go b/coordinator/internal/userapi/userapi.go index 50b2e199356..5c30dfd8052 100644 --- a/coordinator/internal/userapi/userapi.go +++ b/coordinator/internal/userapi/userapi.go @@ -139,8 +139,11 @@ func (s *Server) SetManifest(ctx context.Context, req *userapi.SetManifestReques state, err := s.guard.UpdateState(ctx, oldState, se, req.GetManifest(), req.GetPolicies()) if err != nil { code := codes.Internal - if errors.Is(err, stateguard.ErrConcurrentUpdate) { + switch { + case errors.Is(err, stateguard.ErrConcurrentUpdate): code = codes.FailedPrecondition + case errors.Is(err, stateguard.ErrInsecureNotAllowed): + code = codes.InvalidArgument } return nil, status.Errorf(code, "updating Coordinator state: %v", err) } diff --git a/coordinator/internal/userapi/userapi_test.go b/coordinator/internal/userapi/userapi_test.go index fd113b425fd..08d67be16b6 100644 --- a/coordinator/internal/userapi/userapi_test.go +++ b/coordinator/internal/userapi/userapi_test.go @@ -230,6 +230,34 @@ func TestSetManifest(t *testing.T) { require.Equal(codes.InvalidArgument, status.Code(err)) }) + t.Run("insecure manifest rejected", func(t *testing.T) { + require := require.New(t) + + // Default coordinator does not allow insecure manifests. + coordinator := newCoordinator() + m := newInsecureManifest(t) + manifestBytes, err := json.Marshal(m) + require.NoError(err) + req := &userapi.SetManifestRequest{Manifest: manifestBytes} + _, err = coordinator.SetManifest(t.Context(), req) + require.Error(err) + require.Equal(codes.InvalidArgument, status.Code(err)) + require.ErrorContains(err, "insecure") + }) + + t.Run("insecure manifest accepted when allowed", func(t *testing.T) { + require := require.New(t) + + coordinator := newCoordinatorAllowInsecure() + m := newInsecureManifest(t) + manifestBytes, err := json.Marshal(m) + require.NoError(err) + req := &userapi.SetManifestRequest{Manifest: manifestBytes} + resp, err := coordinator.SetManifest(t.Context(), req) + require.NoError(err) + require.NotNil(resp) + }) + t.Run("atomic manifest update", func(t *testing.T) { require := require.New(t) @@ -366,7 +394,7 @@ func TestRecovery(t *testing.T) { fs := afero.NewMemMapFs() store := aferostore.New(&afero.Afero{Fs: fs}) hist := history.NewWithStore(slog.Default(), store) - auth := stateguard.New(hist, prometheus.NewRegistry(), logger) + auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false) discovery := &stubDiscovery{ peers: tc.peers, err: tc.peersErr, @@ -400,7 +428,7 @@ func TestRecovery(t *testing.T) { } // Simulate a restarted Coordinator. - a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default()) + a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default(), false) _, err = a.GetManifests(t.Context(), nil) require.ErrorContains(err, ErrNeedsRecovery.Error()) _, err = a.Recover(rpcContext(t.Context(), seedShareOwnerKey), recoverReq) @@ -422,7 +450,7 @@ func TestRecoveryFlow(t *testing.T) { fs := afero.NewMemMapFs() store := aferostore.New(&afero.Afero{Fs: fs}) hist := history.NewWithStore(slog.Default(), store) - auth := stateguard.New(hist, prometheus.NewRegistry(), logger) + auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false) a := New(logger, auth, &stubDiscovery{}) // 2. A manifest is set and the returned seed is recorded. @@ -458,7 +486,7 @@ func TestRecoveryFlow(t *testing.T) { // 3. A new Coordinator is created with the existing history. // GetManifests and SetManifest are expected to fail. - a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default()) + a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default(), false) _, err = a.SetManifest(t.Context(), req) require.ErrorContains(err, ErrNeedsRecovery.Error()) @@ -501,7 +529,7 @@ func TestUserAPIConcurrent(t *testing.T) { fs := afero.NewBasePathFs(afero.NewOsFs(), t.TempDir()) store := aferostore.New(&afero.Afero{Fs: fs}) hist := history.NewWithStore(slog.Default(), store) - auth := stateguard.New(hist, prometheus.NewRegistry(), logger) + auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false) coordinator := New(logger, auth, &stubDiscovery{}) setReq := &userapi.SetManifestRequest{ @@ -815,14 +843,32 @@ func newCoordinatorWithRegistry(reg *prometheus.Registry) *Server { fs := afero.NewMemMapFs() store := aferostore.New(&afero.Afero{Fs: fs}) hist := history.NewWithStore(slog.Default(), store) - auth := stateguard.New(hist, reg, logger) + auth := stateguard.New(hist, reg, logger, false) return New(logger, auth, &stubDiscovery{}) } +func newCoordinatorAllowInsecure() *Server { + logger := slog.Default() + fs := afero.NewMemMapFs() + store := aferostore.New(&afero.Afero{Fs: fs}) + hist := history.NewWithStore(slog.Default(), store) + auth := stateguard.New(hist, prometheus.NewRegistry(), logger, true) + return New(logger, auth, &stubDiscovery{}) +} + +func newInsecureManifest(t *testing.T) *manifest.Manifest { + t.Helper() + mnfst := &manifest.Manifest{} + mnfst.ReferenceValues.SNP = []manifest.SNPReferenceValues{ + {Platform: "Metal-QEMU-SNP-Insecure"}, + } + return mnfst +} + func newCoordinatorWithWatcher(t *testing.T, hist *history.History) *Server { t.Helper() logger := slog.Default() - auth := stateguard.New(hist, prometheus.NewRegistry(), logger) + auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false) coordinator := New(logger, auth, &stubDiscovery{}) ctx, cancel := context.WithCancel(t.Context()) diff --git a/coordinator/main.go b/coordinator/main.go index e88ed24ab94..cdb6d77a13a 100644 --- a/coordinator/main.go +++ b/coordinator/main.go @@ -52,6 +52,7 @@ import ( const ( metricsEnvVar = "CONTRAST_METRICS" + allowInsecureEnvVar = "CONTRAST_ALLOW_INSECURE" probeAndMetricsPort = 9102 // transitEngineAPIPort specifies the default port to expose the transit engine API. transitEngineAPIPort = "8200" @@ -115,7 +116,12 @@ func run() (retErr error) { hist := history.NewWithStore(logger.WithGroup("history"), store) - meshAuth := stateguard.New(hist, promRegistry, logger) + _, allowInsecure := os.LookupEnv(allowInsecureEnvVar) + if allowInsecure { + logger.Warn("Coordinator is configured to allow insecure manifests") + } + + meshAuth := stateguard.New(hist, promRegistry, logger, allowInsecure) issuer, err := issuer.New(logger) if err != nil { diff --git a/e2e/insecure/insecure_test.go b/e2e/insecure/insecure_test.go new file mode 100644 index 00000000000..eee0639d617 --- /dev/null +++ b/e2e/insecure/insecure_test.go @@ -0,0 +1,120 @@ +// Copyright 2026 Edgeless Systems GmbH +// SPDX-License-Identifier: BUSL-1.1 + +//go:build e2e + +package insecure + +import ( + "context" + "flag" + "os" + "strings" + "testing" + "time" + + "github.com/edgelesssys/contrast/e2e/internal/contrasttest" + "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/internal/platforms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + secureDeployment = "secure-pod" + insecureDeployment = "insecure-pod" +) + +// TestInsecure deploys a secure and an insecure pod side by side and verifies +// that only the secure pod runs inside a TEE. +func TestInsecure(t *testing.T) { + platform, err := platforms.FromString(contrasttest.Flags.PlatformStr) + require.NoError(t, err) + + insecurePlatform := platform.InsecureVariant() + if insecurePlatform == platforms.Unknown { + t.Skip("no insecure variant for platform", platform) + } + + // The generate and verify commands require this env var for insecure platforms. + t.Setenv("CONTRAST_ALLOW_INSECURE_RUNTIMES", "1") + + ct := contrasttest.New(t) + ct.Platform = insecurePlatform // Required so RunGenerate/RunVerify pass --INSECURE. + + secureHandler, err := manifest.RuntimeHandler(platform) + require.NoError(t, err) + insecureHandler, err := manifest.RuntimeHandler(insecurePlatform) + require.NoError(t, err) + + resources := kuberesource.CoordinatorBundle() + // Patch the coordinator with the insecure runtime handler. + resources = kuberesource.PatchRuntimeHandlers(resources, insecureHandler) + resources = kuberesource.AddPortForwarders(resources) + + // Add deployments *after* PatchRuntimeHandlers to retain control over the RuntimeClassNames. + resources = append(resources, kuberesource.DeploymentWithRuntimeClass(secureDeployment, secureHandler)) + resources = append(resources, kuberesource.DeploymentWithRuntimeClass(insecureDeployment, insecureHandler)) + + ct.Init(t, resources) + require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") + require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests") + require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests") + require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests") + + t.Run("pods use correct runtime classes", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(2*time.Minute)) + defer cancel() + require := require.New(t) + + securePods, err := ct.Kubeclient.PodsFromDeployment(ctx, ct.Namespace, secureDeployment) + require.NoError(err) + require.Len(securePods, 1) + assert.True(t, strings.HasPrefix(*securePods[0].Spec.RuntimeClassName, secureHandler)) + + insecurePods, err := ct.Kubeclient.PodsFromDeployment(ctx, ct.Namespace, insecureDeployment) + require.NoError(err) + require.Len(insecurePods, 1) + assert.True(t, strings.HasPrefix(*insecurePods[0].Spec.RuntimeClassName, insecureHandler)) + }) + + t.Run("pods start", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(2*time.Minute)) + defer cancel() + require := require.New(t) + + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, secureDeployment)) + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, insecureDeployment)) + }) + + t.Run("secure pod runs in TEE", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(1*time.Minute)) + defer cancel() + require := require.New(t) + + stdout, stderr, err := ct.Kubeclient.ExecDeployment(ctx, ct.Namespace, secureDeployment, []string{ + "/usr/local/bin/bash", "-c", "dmesg | grep -i -E 'tdx|sev|snp'", + }) + require.NoError(err, "stderr: %q", stderr) + require.NotEmpty(strings.TrimSpace(stdout), "expected TEE-related dmesg output in secure pod") + }) + + t.Run("insecure pod does not run in TEE", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), ct.FactorPlatformTimeout(1*time.Minute)) + defer cancel() + + // grep exits with 1 when no lines match, so we expect an error here. + stdout, _, _ := ct.Kubeclient.ExecDeployment(ctx, ct.Namespace, insecureDeployment, []string{ + "/usr/local/bin/bash", "-c", "dmesg | grep -i -E 'tdx|sev|snp'", + }) + assert.Empty(t, strings.TrimSpace(stdout), "expected no TEE-related dmesg output in insecure pod") + }) +} + +func TestMain(m *testing.M) { + contrasttest.RegisterFlags() + flag.Parse() + + os.Exit(m.Run()) +} diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index 10b5d9bb102..72d397d9b6e 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -231,6 +231,9 @@ func (ct *ContrastTest) RunGenerate(ctx context.Context) error { if Flags.GenpolicyCachePath != "" { args = append(args, "--genpolicy-cache-path", Flags.GenpolicyCachePath) } + if platforms.IsInsecure(ct.Platform) { + args = append(args, "--INSECURE") + } args = append(args, ct.WorkDir) generate := cmd.NewGenerateCmd() @@ -365,7 +368,11 @@ func (ct *ContrastTest) RunVerify(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() - if err := ct.runAgainstCoordinator(ctx, cmd.NewVerifyCmd()); err != nil { + var verifyArgs []string + if platforms.IsInsecure(ct.Platform) { + verifyArgs = append(verifyArgs, "--INSECURE") + } + if err := ct.runAgainstCoordinator(ctx, cmd.NewVerifyCmd(), verifyArgs...); err != nil { return err } @@ -569,7 +576,8 @@ func (ct *ContrastTest) runAgainstCoordinator(ctx context.Context, cmd *cobra.Co // Baseline is AKS. func (ct *ContrastTest) FactorPlatformTimeout(timeout time.Duration) time.Duration { switch ct.Platform { - case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUTDXGPU: + case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUTDXGPU, + platforms.MetalQEMUSNPInsecure, platforms.MetalQEMUTDXInsecure, platforms.MetalQEMUSNPGPUInsecure, platforms.MetalQEMUTDXGPUInsecure: return 2 * timeout default: panic(fmt.Sprintf("FactorPlatformTimeout not configured for platform %q", ct.Platform)) diff --git a/initdata-processor/main.go b/initdata-processor/main.go index 7ffeafb44c6..2f3090229d0 100644 --- a/initdata-processor/main.go +++ b/initdata-processor/main.go @@ -5,15 +5,20 @@ package main import ( "bytes" + "context" + "errors" "fmt" "io" "io/fs" "log" + "net" + "net/http" "os" "path/filepath" "github.com/edgelesssys/contrast/initdata-processor/policy" "github.com/edgelesssys/contrast/initdata-processor/validator" + "github.com/edgelesssys/contrast/internal/attestation/insecure" "github.com/edgelesssys/contrast/internal/initdata" ) @@ -30,6 +35,9 @@ func main() { log.Printf("Contrast initdata-processor %s", version) log.Print("Report issues at https://github.com/edgelesssys/contrast/issues") + var hostdata []byte + var insecurePlatform bool + // Handle initdata. if err := os.MkdirAll(measuredConfigPath, 0o755); err != nil { failf("Could not create directory %q: %v", measuredConfigPath, err) @@ -45,7 +53,8 @@ func main() { failf("%s is not an initdata device: %v", device, err) return } - if err := handleInitdata(doc); err != nil { + hostdata, insecurePlatform, err = handleInitdata(doc) + if err != nil { failf("handling initdata: %v", err) return } @@ -59,44 +68,89 @@ func main() { device, err = checkDeviceAvailability("imagepuller") if err != nil { log.Println("No imagepuller auth config found, only unauthenticated pulls will be available") - return - } - doc, err = initdata.FromDevice(device, "imgpullr") - if err != nil { - failf("%s is not an imagepuller config device: %v", device, err) - return + } else { + doc, err = initdata.FromDevice(device, "imgpullr") + if err != nil { + failf("%s is not an imagepuller config device: %v", device, err) + return + } + if err := handleImagepullerAuthConfig(doc); err != nil { + failf("handling imagepuller auth config: %v", err) + return + } + log.Printf("Processed imagepuller auth config from %q ", device) } - if err := handleImagepullerAuthConfig(doc); err != nil { - failf("handling imagepuller auth config: %v", err) - return + + // Signal systemd that initdata processing is complete. + sdNotifyReady() + + // On insecure platforms, serve the hostdata digest via HTTP so that + // the insecure aTLS issuer (running inside containers) can fetch it. + if insecurePlatform { + log.Printf("Starting insecure hostdata server on %s", insecure.HostdataAddr) + if err := serveHostdata(hostdata); err != nil { + log.Printf("Hostdata server error: %v", err) + } } - log.Printf("Processed imagepuller auth config from %q ", device) } -func handleInitdata(doc initdata.Raw) error { +func handleInitdata(doc initdata.Raw) (hostdata []byte, insecurePlatform bool, retErr error) { digest, err := doc.Digest() if err != nil { - return fmt.Errorf("initdata validation failed: %w", err) - } - validator, err := validator.New() - if err != nil { - return fmt.Errorf("creating validator: %w", err) + return nil, false, fmt.Errorf("computing initdata digest: %w", err) } - if err := validator.ValidateDigest(digest); err != nil { - return fmt.Errorf("validating initdata digest: %w", err) + + v, verr := validator.New() + if errors.Is(verr, validator.ErrNoPlatform) { + log.Print("WARNING: No TEE platform detected, skipping initdata digest validation. This is expected on insecure platforms.") + insecurePlatform = true + } else if verr != nil { + return nil, false, fmt.Errorf("creating validator: %w", verr) + } else if err := v.ValidateDigest(digest); err != nil { + return nil, false, fmt.Errorf("validating initdata digest: %w", err) } + data, err := doc.Parse() if err != nil { - return fmt.Errorf("parsing initdata: %w", err) + return nil, false, fmt.Errorf("parsing initdata: %w", err) } for name, content := range data.Data { name = filepath.Clean(name) path := filepath.Join(measuredConfigPath, name) if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - return fmt.Errorf("writing file %q: %w", path, err) + return nil, false, fmt.Errorf("writing file %q: %w", path, err) } } - return nil + return digest, insecurePlatform, nil +} + +// serveHostdata starts an HTTP server that serves the hostdata digest. +func serveHostdata(hostdata []byte) error { + mux := http.NewServeMux() + mux.HandleFunc("GET /hostdata", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + if _, err := w.Write(hostdata); err != nil { + log.Printf("hostdata write error: %v", err) + } + }) + return http.ListenAndServe(insecure.HostdataAddr, mux) +} + +// sdNotifyReady signals systemd that the service is ready. +func sdNotifyReady() { + addr := os.Getenv("NOTIFY_SOCKET") + if addr == "" { + return + } + conn, err := (&net.Dialer{}).DialContext(context.Background(), "unixgram", addr) + if err != nil { + log.Printf("sd_notify: dial: %v", err) + return + } + defer conn.Close() + if _, err := conn.Write([]byte("READY=1")); err != nil { + log.Printf("sd_notify: write: %v", err) + } } func handleImagepullerAuthConfig(doc initdata.Raw) error { diff --git a/initdata-processor/validator/validator.go b/initdata-processor/validator/validator.go index c78ec25c4ac..b09c46d5bb1 100644 --- a/initdata-processor/validator/validator.go +++ b/initdata-processor/validator/validator.go @@ -29,7 +29,7 @@ func New() (*Validator, error) { if terr == nil { return &Validator{&tdxDigestGetter{tqp}}, nil } - return nil, fmt.Errorf("%w:\nTDX:%w\nSNP:%w", errBadPlatform, terr, serr) + return nil, fmt.Errorf("%w:\nTDX:%w\nSNP:%w", ErrNoPlatform, terr, serr) } // ValidateDigest compares the given digest with either MRCONFIGID or HOSTDATA, and returns an error if they don't match. @@ -85,7 +85,9 @@ func getTDXQuoteProvider() (tdxclient.QuoteProvider, error) { } var ( - errBadPlatform = errors.New("no digest getter available for current platform") + // ErrNoPlatform is returned by New when no TEE platform is available for digest validation. + ErrNoPlatform = errors.New("no digest getter available for current platform") + errUnexpectedDigestSize = errors.New("unexpected digest size") errDigestMismatch = errors.New("digests don't match") ) diff --git a/internal/atls/issuer/issuer_linux.go b/internal/atls/issuer/issuer_linux.go index 6f29ee179cc..4e9e47967cb 100644 --- a/internal/atls/issuer/issuer_linux.go +++ b/internal/atls/issuer/issuer_linux.go @@ -6,10 +6,10 @@ package issuer import ( - "fmt" "log/slog" "github.com/edgelesssys/contrast/internal/atls" + "github.com/edgelesssys/contrast/internal/attestation/insecure" snpissuer "github.com/edgelesssys/contrast/internal/attestation/snp/issuer" tdxissuer "github.com/edgelesssys/contrast/internal/attestation/tdx/issuer" "github.com/edgelesssys/contrast/internal/logger" @@ -29,6 +29,7 @@ func New(log *slog.Logger) (atls.Issuer, error) { logger.NewWithAttrs(logger.NewNamed(log, "issuer"), map[string]string{"tee-type": "tdx"}), ), nil default: - return nil, fmt.Errorf("unsupported platform: %T", cpuid.CPU) + log.Warn("No TEE platform detected, using insecure attestation issuer") + return insecure.NewIssuer(), nil } } diff --git a/internal/attestation/insecure/issuer.go b/internal/attestation/insecure/issuer.go new file mode 100644 index 00000000000..98e3aa2c5e3 --- /dev/null +++ b/internal/attestation/insecure/issuer.go @@ -0,0 +1,71 @@ +// Copyright 2025 Edgeless Systems GmbH +// SPDX-License-Identifier: BUSL-1.1 + +// Package insecure provides a fake aTLS issuer and validator for development +// platforms without confidential computing hardware. +package insecure + +import ( + "context" + "encoding/asn1" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/edgelesssys/contrast/internal/oid" +) + +// HostdataAddr is the address where the initdata-processor serves the +// hostdata digest on insecure platforms. +const HostdataAddr = "127.0.0.1:19629" + +// HostdataURL is the full URL for fetching the hostdata digest. +const HostdataURL = "http://" + HostdataAddr + "/hostdata" + +// Issuer issues fake attestation documents for insecure (non-CC) platforms. +// +// It fetches the initdata digest from the local initdata-processor HTTP server +// and packages it with the report data into a JSON attestation document. +type Issuer struct{} + +// NewIssuer creates a new insecure issuer. +func NewIssuer() *Issuer { + return &Issuer{} +} + +// OID returns the OID for the insecure attestation. +func (i *Issuer) OID() asn1.ObjectIdentifier { + return oid.RawInsecureReport +} + +// Issue creates a fake attestation document containing the report data and +// the initdata digest fetched from the local hostdata server. +func (i *Issuer) Issue(ctx context.Context, reportData [64]byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, HostdataURL, nil) + if err != nil { + return nil, fmt.Errorf("creating hostdata request: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching hostdata from %q: %w", HostdataURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching hostdata: status %s", resp.Status) + } + hostData, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading hostdata response: %w", err) + } + return json.Marshal(attestationDoc{ + ReportData: reportData[:], + HostData: hostData, + }) +} + +// attestationDoc is the fake attestation document exchanged between issuer and validator. +type attestationDoc struct { + ReportData []byte `json:"reportData"` + HostData []byte `json:"hostData"` +} diff --git a/internal/attestation/insecure/validator.go b/internal/attestation/insecure/validator.go new file mode 100644 index 00000000000..2d06a203953 --- /dev/null +++ b/internal/attestation/insecure/validator.go @@ -0,0 +1,74 @@ +// Copyright 2025 Edgeless Systems GmbH +// SPDX-License-Identifier: BUSL-1.1 + +package insecure + +import ( + "bytes" + "context" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "fmt" + "log/slog" + + "github.com/edgelesssys/contrast/internal/attestation" + "github.com/edgelesssys/contrast/internal/oid" +) + +// Validator validates fake attestation documents from insecure (non-CC) platforms. +type Validator struct { + reportSetter attestation.ReportSetter + logger *slog.Logger + name string +} + +// NewValidator creates a new insecure validator. +func NewValidator(log *slog.Logger, name string) *Validator { + return &Validator{logger: log, name: name} +} + +// NewValidatorWithReportSetter creates a new insecure validator with a report setter callback. +func NewValidatorWithReportSetter(log *slog.Logger, reportSetter attestation.ReportSetter, name string) *Validator { + return &Validator{reportSetter: reportSetter, logger: log, name: name} +} + +// OID returns the OID for the insecure attestation. +func (v *Validator) OID() asn1.ObjectIdentifier { + return oid.RawInsecureReport +} + +// Validate verifies the fake attestation document and extracts the host data. +func (v *Validator) Validate(_ context.Context, attDocRaw []byte, reportData []byte) error { + var doc attestationDoc + if err := json.Unmarshal(attDocRaw, &doc); err != nil { + return fmt.Errorf("unmarshaling insecure attestation: %w", err) + } + if !bytes.Equal(doc.ReportData, reportData) { + return fmt.Errorf("reportData mismatch: expected %x, got %x", reportData, doc.ReportData) + } + if v.reportSetter != nil { + v.reportSetter.SetReport(report{hostData: doc.HostData}) + } + return nil +} + +// String returns the validator's name. +func (v *Validator) String() string { + return v.name +} + +// report implements the [attestation.Report] interface for insecure platforms. +type report struct { + hostData []byte +} + +// HostData returns the initdata digest. +func (r report) HostData() []byte { + return r.hostData +} + +// ClaimsToCertExtension returns no extensions for insecure platforms. +func (r report) ClaimsToCertExtension() ([]pkix.Extension, error) { + return nil, nil +} diff --git a/internal/attestation/oid.go b/internal/attestation/oid.go index 76151f88e5f..2e58fa830c0 100644 --- a/internal/attestation/oid.go +++ b/internal/attestation/oid.go @@ -10,7 +10,7 @@ import ( ) // IsAttestationDocumentExtension checks whether the given OID corresponds to an attestation document extension -// supported by Contrast (i.e. TDX or SNP). +// supported by Contrast (i.e. TDX, SNP, or insecure). func IsAttestationDocumentExtension(oid asn1.ObjectIdentifier) bool { - return oid.Equal(oids.RawTDXReport) || oid.Equal(oids.RawSNPReport) + return oid.Equal(oids.RawTDXReport) || oid.Equal(oids.RawSNPReport) || oid.Equal(oids.RawInsecureReport) } diff --git a/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go index a1e54450e91..70493c044ae 100644 --- a/internal/kuberesource/mutators.go +++ b/internal/kuberesource/mutators.go @@ -34,6 +34,19 @@ const ( imageStoreSizeAnnotationKey = "contrast.edgeless.systems/image-store-size" ) +// contrastRuntimeClassPrefixes lists runtime class prefixes that identify Contrast pods. +var contrastRuntimeClassPrefixes = []string{"contrast-cc", "contrast-insecure"} + +// IsContrastPod reports whether a pod uses a Contrast runtime. +func IsContrastPod(spec *applycorev1.PodSpecApplyConfiguration) bool { + if spec == nil || spec.RuntimeClassName == nil { + return false + } + return slices.ContainsFunc(contrastRuntimeClassPrefixes, func(p string) bool { + return strings.HasPrefix(*spec.RuntimeClassName, p) + }) +} + // AddInitializer adds an initializer and its shared volume to the resource. // // If the resource does not contain a PodSpec, this function does nothing. @@ -46,7 +59,7 @@ func AddInitializer( if meta != nil && meta.Annotations[skipInitializerAnnotationKey] == "true" { return meta, spec } - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return meta, spec } if meta != nil && meta.Annotations[securePVAnnotationKey] != "" { @@ -173,7 +186,7 @@ func AddServiceMesh( serviceMeshProxy *applycorev1.ContainerApplyConfiguration, ) (res any, retErr error) { res = MapPodSpecWithMeta(resource, func(meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return meta, spec } @@ -230,7 +243,7 @@ func AddDebugShell( debugShell *applycorev1.ContainerApplyConfiguration, ) (any, error) { return MapPodSpec(resource, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return spec } @@ -319,7 +332,7 @@ func AddDmesg(resources []any) []any { WithPrivileged(true).SecurityContextApplyConfiguration) addDmesg := func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return spec } spec.Containers = append(spec.Containers, *dmesgContainer) @@ -380,7 +393,7 @@ func AddImageStore(resources []any) []any { addPvc := func(meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration, ) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return meta, spec } @@ -733,7 +746,7 @@ func PatchNodeSelector(resources []any) []any { var out []any for _, resource := range resources { out = append(out, MapPodSpec(resource, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc") { + if !IsContrastPod(spec) { return spec } spec = spec.WithNodeSelector(map[string]string{ diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index a91dc3ac428..c60231185d7 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -28,6 +28,15 @@ func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, err // Consists of the default VM memory, 70MiB for the Kata shim and 100MiB for qemu overhead. memoryOverhead := platforms.DefaultMemoryInMebiBytes(platform) + 170 + if platforms.IsInsecure(platform) && platforms.IsGPU(platform) { + // On insecure (non-CC) GPU platforms, iommufd VFIO passthrough pins guest + // memory and allocates IOMMU page tables that are charged to the pod's + // cgroup. On CC platforms, TDX manages this memory outside the cgroup. + // (in the kernel / TDX module memory) + // Add extra headroom to avoid OOM kills. 512MB should suffice for VMs up + // to ~256GB of memory, which is our current limit on TDX as well. + memoryOverhead += 512 + } r := RuntimeClass(runtimeHandler). WithHandler(runtimeHandler). diff --git a/internal/kuberesource/runtimeclasses.go b/internal/kuberesource/runtimeclasses.go index 9020ad9ccd0..79047d79769 100644 --- a/internal/kuberesource/runtimeclasses.go +++ b/internal/kuberesource/runtimeclasses.go @@ -90,7 +90,12 @@ func (p PlatformCollection) AddFromResources(resources []any) error { for _, resource := range resources { _ = MapPodSpecWithMeta(resource, func(meta *applymetav1.ObjectMetaApplyConfiguration, spec *applycorev1.PodSpecApplyConfiguration, ) (*applymetav1.ObjectMetaApplyConfiguration, *applycorev1.PodSpecApplyConfiguration) { - if spec == nil || spec.RuntimeClassName == nil || !strings.HasPrefix(*spec.RuntimeClassName, "contrast-cc-") { + if !IsContrastPod(spec) { + return meta, spec + } + // Bare runtime class names (e.g. "contrast-cc") are placeholders + // that get resolved during generate. Skip them here. + if *spec.RuntimeClassName == "contrast-cc" || *spec.RuntimeClassName == "contrast-insecure" { return meta, spec } platform, err := platforms.FromRuntimeClassString(*spec.RuntimeClassName) diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 82ab9e2752a..d3dad358876 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -119,6 +119,21 @@ func (m *Manifest) CoordinatorPolicyHash() (HexString, error) { return "", errors.New("no coordinator found in manifest") } +// AllowInsecure returns true if the manifest contains reference values for insecure platforms. +func (m *Manifest) AllowInsecure() bool { + for _, v := range m.ReferenceValues.SNP { + if p, err := platforms.FromString(v.Platform); err == nil && platforms.IsInsecure(p) { + return true + } + } + for _, v := range m.ReferenceValues.TDX { + if p, err := platforms.FromString(v.Platform); err == nil && platforms.IsInsecure(p) { + return true + } + } + return false +} + // SNPValidateOpts returns validate options generators populated with the manifest's // SNP reference values and trusted measurement for the given runtime. func (m *Manifest) SNPValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]SNPValidatorOptions, error) { @@ -128,6 +143,9 @@ func (m *Manifest) SNPValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]SN var out []SNPValidatorOptions for _, refVal := range m.ReferenceValues.SNP { + if p, err := platforms.FromString(refVal.Platform); err == nil && platforms.IsInsecure(p) { + continue + } if len(refVal.TrustedMeasurement) == 0 { return nil, errors.New("trusted measurement cannot be empty") } @@ -213,6 +231,9 @@ func (m *Manifest) TDXValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]TD var out []TDXValidatorOptions for _, refVal := range m.ReferenceValues.TDX { + if p, err := platforms.FromString(refVal.Platform); err == nil && platforms.IsInsecure(p) { + continue + } verifyOpts := tdxverify.DefaultOptions() var err error diff --git a/internal/manifest/referencevalues.go b/internal/manifest/referencevalues.go index 2477f60dcb6..de2e83062e8 100644 --- a/internal/manifest/referencevalues.go +++ b/internal/manifest/referencevalues.go @@ -208,6 +208,9 @@ type SNPReferenceValues struct { // Validate checks the validity of all fields in the AKS reference values. func (r SNPReferenceValues) Validate() error { + if p, err := platforms.FromString(r.Platform); err == nil && platforms.IsInsecure(p) { + return nil + } var minTCBErrs []error if r.MinimumTCB.BootloaderVersion == nil { minTCBErrs = append(minTCBErrs, newValidationError("BootloaderVersion", ExpectedMissingReferenceValueError{Err: errors.New("field cannot be empty")})) @@ -312,6 +315,9 @@ type TDXReferenceValues struct { // Validate checks the validity of all fields in the bare metal TDX reference values. func (r TDXReferenceValues) Validate() error { + if p, err := platforms.FromString(r.Platform); err == nil && platforms.IsInsecure(p) { + return nil + } var errs []error if err := validateHexString(r.MrTd, 48); err != nil { errs = append(errs, newValidationError("MrTd", err)) diff --git a/internal/manifest/runtimehandler.go b/internal/manifest/runtimehandler.go index 03433729f56..9a2b3b1816b 100644 --- a/internal/manifest/runtimehandler.go +++ b/internal/manifest/runtimehandler.go @@ -33,6 +33,11 @@ func RuntimeHandler(platform platforms.Platform) (string, error) { // PlatformFromHandler extracts the platform from the runtime handler name. func PlatformFromHandler(handler string) (platforms.Platform, error) { rest, found := strings.CutPrefix(handler, "contrast-cc-") + isInsecure := false + if !found { + rest, found = strings.CutPrefix(handler, "contrast-insecure-") + isInsecure = true + } if !found { return platforms.Unknown, fmt.Errorf("invalid handler name: %s", handler) } @@ -43,6 +48,9 @@ func PlatformFromHandler(handler string) (platforms.Platform, error) { } rawPlatform := strings.Join(parts[:len(parts)-1], "-") + if isInsecure { + rawPlatform += "-insecure" + } platform, err := platforms.FromString(rawPlatform) if err != nil { diff --git a/internal/oid/oid.go b/internal/oid/oid.go index 60598d2f15b..8fa9a7bc105 100644 --- a/internal/oid/oid.go +++ b/internal/oid/oid.go @@ -13,6 +13,10 @@ var RawSNPReport = asn1.ObjectIdentifier{1, 3, 9901, 2, 1} // used by the aTLS issuer and validator. var RawTDXReport = asn1.ObjectIdentifier{1, 3, 9901, 2, 2} +// RawInsecureReport is the OID for the insecure (non-CC) attestation, +// used on development platforms without CC hardware. +var RawInsecureReport = asn1.ObjectIdentifier{1, 3, 9901, 2, 3} + // WorkloadSecretOID is the root OID for the workloadSecretID report // extension, added to the mesh certificates to allow verification // and authorization based on the workloadSecretID. diff --git a/internal/platforms/platforms.go b/internal/platforms/platforms.go index ecad3d19207..b3e4568410b 100644 --- a/internal/platforms/platforms.go +++ b/internal/platforms/platforms.go @@ -24,11 +24,22 @@ const ( MetalQEMUSNPGPU // MetalQEMUTDXGPU is the generic platform for bare-metal TDX deployments with GPU passthrough. MetalQEMUTDXGPU + // MetalQEMUSNPInsecure is the platform for bare-metal SNP deployments with a non-CC runtime class. + MetalQEMUSNPInsecure + // MetalQEMUTDXInsecure is the platform for bare-metal TDX deployments with a non-CC runtime class. + MetalQEMUTDXInsecure + // MetalQEMUSNPGPUInsecure is the platform for bare-metal SNP deployments with GPU passthrough and a non-CC runtime class. + MetalQEMUSNPGPUInsecure + // MetalQEMUTDXGPUInsecure is the platform for bare-metal TDX deployments with GPU passthrough and a non-CC runtime class. + MetalQEMUTDXGPUInsecure ) // All returns a list of all available platforms. func All() []Platform { - return []Platform{MetalQEMUSNP, MetalQEMUTDX, MetalQEMUSNPGPU, MetalQEMUTDXGPU} + return []Platform{ + MetalQEMUSNP, MetalQEMUTDX, MetalQEMUSNPGPU, MetalQEMUTDXGPU, + MetalQEMUSNPInsecure, MetalQEMUTDXInsecure, MetalQEMUSNPGPUInsecure, MetalQEMUTDXGPUInsecure, + } } // AllStrings returns a list of all available platforms as strings. @@ -51,11 +62,36 @@ func (p Platform) String() string { return "Metal-QEMU-TDX" case MetalQEMUTDXGPU: return "Metal-QEMU-TDX-GPU" + case MetalQEMUSNPInsecure: + return "Metal-QEMU-SNP-Insecure" + case MetalQEMUTDXInsecure: + return "Metal-QEMU-TDX-Insecure" + case MetalQEMUSNPGPUInsecure: + return "Metal-QEMU-SNP-GPU-Insecure" + case MetalQEMUTDXGPUInsecure: + return "Metal-QEMU-TDX-GPU-Insecure" default: return "Unknown" } } +// InsecureVariant returns the insecure (non-CC) variant of the +// platform, or Unknown if there is no such variant. +func (p Platform) InsecureVariant() Platform { + switch p { + case MetalQEMUSNP: + return MetalQEMUSNPInsecure + case MetalQEMUTDX: + return MetalQEMUTDXInsecure + case MetalQEMUSNPGPU: + return MetalQEMUSNPGPUInsecure + case MetalQEMUTDXGPU: + return MetalQEMUTDXGPUInsecure + default: + return Unknown + } +} + // MarshalJSON marshals a Platform type to a JSON string. func (p Platform) MarshalJSON() ([]byte, error) { return fmt.Appendf(nil, `"%s"`, p.String()), nil @@ -99,6 +135,14 @@ func FromString(s string) (Platform, error) { return MetalQEMUTDX, nil case "metal-qemu-tdx-gpu": return MetalQEMUTDXGPU, nil + case "metal-qemu-snp-insecure": + return MetalQEMUSNPInsecure, nil + case "metal-qemu-tdx-insecure": + return MetalQEMUTDXInsecure, nil + case "metal-qemu-snp-gpu-insecure": + return MetalQEMUSNPGPUInsecure, nil + case "metal-qemu-tdx-gpu-insecure": + return MetalQEMUTDXGPUInsecure, nil default: return Unknown, fmt.Errorf("unknown platform: %s", s) } @@ -121,10 +165,18 @@ func FromRuntimeClassString(s string) (Platform, error) { return MetalQEMUSNPGPU, nil case strings.HasPrefix(s, "contrast-cc-metal-qemu-snp"): return MetalQEMUSNP, nil + case strings.HasPrefix(s, "contrast-insecure-metal-qemu-snp-gpu"): + return MetalQEMUSNPGPUInsecure, nil + case strings.HasPrefix(s, "contrast-insecure-metal-qemu-snp"): + return MetalQEMUSNPInsecure, nil case strings.HasPrefix(s, "contrast-cc-metal-qemu-tdx-gpu"): return MetalQEMUTDXGPU, nil case strings.HasPrefix(s, "contrast-cc-metal-qemu-tdx"): return MetalQEMUTDX, nil + case strings.HasPrefix(s, "contrast-insecure-metal-qemu-tdx-gpu"): + return MetalQEMUTDXGPUInsecure, nil + case strings.HasPrefix(s, "contrast-insecure-metal-qemu-tdx"): + return MetalQEMUTDXInsecure, nil default: return Unknown, fmt.Errorf("unknown platform: %s", s) } @@ -133,7 +185,7 @@ func FromRuntimeClassString(s string) (Platform, error) { // DefaultMemoryInMebiBytes returns the desired VM overhead for the given platform. func DefaultMemoryInMebiBytes(p Platform) int { switch p { - case MetalQEMUSNPGPU, MetalQEMUTDXGPU: + case MetalQEMUSNPGPU, MetalQEMUTDXGPU, MetalQEMUSNPGPUInsecure, MetalQEMUTDXGPUInsecure: // Guest components contribute around 600MiB with GPU enabled. return 1024 default: @@ -144,6 +196,16 @@ func DefaultMemoryInMebiBytes(p Platform) int { } } +// IsInsecure returns true if the platform is an insecure (non-CC) platform. +func IsInsecure(p Platform) bool { + switch p { + case MetalQEMUSNPInsecure, MetalQEMUTDXInsecure, MetalQEMUSNPGPUInsecure, MetalQEMUTDXGPUInsecure: + return true + default: + return false + } +} + // IsSNP returns true if the platform is a SEV-SNP platform. func IsSNP(p Platform) bool { switch p { @@ -167,7 +229,7 @@ func IsTDX(p Platform) bool { // IsGPU returns true if the platform supports GPUs. func IsGPU(p Platform) bool { switch p { - case MetalQEMUSNPGPU, MetalQEMUTDXGPU: + case MetalQEMUSNPGPU, MetalQEMUTDXGPU, MetalQEMUSNPGPUInsecure, MetalQEMUTDXGPUInsecure: return true default: return false @@ -177,7 +239,8 @@ func IsGPU(p Platform) bool { // IsQEMU returns true if the platform uses QEMU as the hypervisor. func IsQEMU(p Platform) bool { switch p { - case MetalQEMUSNP, MetalQEMUSNPGPU, MetalQEMUTDX, MetalQEMUTDXGPU: + case MetalQEMUSNP, MetalQEMUSNPGPU, MetalQEMUTDX, MetalQEMUTDXGPU, + MetalQEMUSNPInsecure, MetalQEMUTDXInsecure, MetalQEMUSNPGPUInsecure, MetalQEMUTDXGPUInsecure: return true default: return false @@ -191,6 +254,10 @@ func (p Platform) WithGPU() Platform { return MetalQEMUSNPGPU case MetalQEMUTDX, MetalQEMUTDXGPU: return MetalQEMUTDXGPU + case MetalQEMUSNPInsecure, MetalQEMUSNPGPUInsecure: + return MetalQEMUSNPGPUInsecure + case MetalQEMUTDXInsecure, MetalQEMUTDXGPUInsecure: + return MetalQEMUTDXGPUInsecure default: return Unknown } diff --git a/justfile b/justfile index 58e4d1fc46a..eec47b1797e 100644 --- a/justfile +++ b/justfile @@ -58,10 +58,10 @@ node-installer platform=default_platform: #!/usr/bin/env bash set -euo pipefail case {{ platform }} in - "Metal-QEMU-SNP"|"Metal-QEMU-TDX") + "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-SNP-Insecure"|"Metal-QEMU-TDX-Insecure") just push "node-installer-kata" ;; - "Metal-QEMU-SNP-GPU"|"Metal-QEMU-TDX-GPU") + "Metal-QEMU-SNP-GPU"|"Metal-QEMU-TDX-GPU"|"Metal-QEMU-SNP-GPU-Insecure"|"Metal-QEMU-TDX-GPU-Insecure") just push "node-installer-kata-gpu" ;; *) @@ -169,7 +169,7 @@ runtime target=default_deploy_target platform=default_platform set=default_set: --namespace {{ target }}${namespace_suffix-} \ --node-installer-target-conf-type ${node_installer_target_conf_type} \ --platform "$platforms" \ - runtime >> "./{{ workspace_dir }}/runtime/runtime.yml" + runtime > "./{{ workspace_dir }}/runtime/runtime.yml" # Populate the workspace with a Kubernetes deployment populate target=default_deploy_target platform=default_platform set=default_set: diff --git a/nodeinstaller/internal/containerdconfig/config.go b/nodeinstaller/internal/containerdconfig/config.go index 2ce2f5deb98..3809aede280 100644 --- a/nodeinstaller/internal/containerdconfig/config.go +++ b/nodeinstaller/internal/containerdconfig/config.go @@ -144,12 +144,12 @@ func ContrastRuntime(baseDir string, platform platforms.Platform) (Runtime, erro PrivilegedWithoutHostDevices: true, } - switch { - case platforms.IsTDX(platform): + switch platform { + case platforms.MetalQEMUTDX, platforms.MetalQEMUTDXGPU, platforms.MetalQEMUTDXInsecure, platforms.MetalQEMUTDXGPUInsecure: cfg.Options = map[string]any{ "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-tdx.toml"), } - case platforms.IsSNP(platform): + case platforms.MetalQEMUSNP, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUSNPInsecure, platforms.MetalQEMUSNPGPUInsecure: cfg.Options = map[string]any{ "ConfigPath": filepath.Join(baseDir, "etc", "configuration-qemu-snp.toml"), } diff --git a/nodeinstaller/internal/kataconfig/config.go b/nodeinstaller/internal/kataconfig/config.go index 193aebb5434..3f9c01bca70 100644 --- a/nodeinstaller/internal/kataconfig/config.go +++ b/nodeinstaller/internal/kataconfig/config.go @@ -39,8 +39,8 @@ func KataRuntimeConfig( ) (*Config, error) { var customContrastAnnotations []string var config Config - switch { - case platforms.IsTDX(platform): + switch platform { + case platforms.MetalQEMUTDX, platforms.MetalQEMUTDXGPU, platforms.MetalQEMUTDXInsecure, platforms.MetalQEMUTDXGPUInsecure: if err := toml.Unmarshal([]byte(kataBareMetalQEMUTDXBaseConfig), &config); err != nil { return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) } @@ -48,14 +48,16 @@ func KataRuntimeConfig( // We set up dm_verity in the system NixOS config. // Doing so again here prevents VM boots. config.Hypervisor["qemu"]["kernel_verity_params"] = "" - case platforms.IsSNP(platform): + case platforms.MetalQEMUSNP, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUSNPInsecure, platforms.MetalQEMUSNPGPUInsecure: if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil { return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err) } - for _, productLine := range []string{"_Milan", "_Genoa"} { - for _, annotationType := range []string{"snp_id_block", "snp_id_auth", "snp_guest_policy"} { - customContrastAnnotations = append(customContrastAnnotations, annotationType+productLine) + if !platforms.IsInsecure(platform) { + for _, productLine := range []string{"_Milan", "_Genoa"} { + for _, annotationType := range []string{"snp_id_block", "snp_id_auth", "snp_guest_policy"} { + customContrastAnnotations = append(customContrastAnnotations, annotationType+productLine) + } } } @@ -63,6 +65,13 @@ func KataRuntimeConfig( default: return nil, fmt.Errorf("unsupported platform: %s", platform) } + // Disable confidential computing features for insecure platforms. + if platforms.IsInsecure(platform) { + config.Hypervisor["qemu"]["confidential_guest"] = false + if platforms.IsSNP(platform) || platform == platforms.MetalQEMUSNPInsecure || platform == platforms.MetalQEMUSNPGPUInsecure { + config.Hypervisor["qemu"]["sev_snp_guest"] = false + } + } if debug { config.Agent["kata"]["enable_debug"] = true config.Agent["kata"]["debug_console_enabled"] = true diff --git a/nodeinstaller/internal/kataconfig/config_test.go b/nodeinstaller/internal/kataconfig/config_test.go index 5a2a19d10dd..fba35bf04d2 100644 --- a/nodeinstaller/internal/kataconfig/config_test.go +++ b/nodeinstaller/internal/kataconfig/config_test.go @@ -22,6 +22,14 @@ var ( expectedConfMetalQEMUSNPGPU []byte //go:embed testdata/expected-configuration-qemu-tdx-gpu.toml expectedConfMetalQEMUTDXGPU []byte + //go:embed testdata/expected-configuration-qemu-snp-insecure.toml + expectedConfMetalQEMUSNPInsecure []byte + //go:embed testdata/expected-configuration-qemu-tdx-insecure.toml + expectedConfMetalQEMUTDXInsecure []byte + //go:embed testdata/expected-configuration-qemu-snp-gpu-insecure.toml + expectedConfMetalQEMUSNPGPUInsecure []byte + //go:embed testdata/expected-configuration-qemu-tdx-gpu-insecure.toml + expectedConfMetalQEMUTDXGPUInsecure []byte ) func TestKataRuntimeConfig(t *testing.T) { @@ -45,6 +53,22 @@ func TestKataRuntimeConfig(t *testing.T) { changeSnpFields: false, want: string(expectedConfMetalQEMUTDXGPU), }, + platforms.MetalQEMUSNPInsecure: { + changeSnpFields: true, + want: string(expectedConfMetalQEMUSNPInsecure), + }, + platforms.MetalQEMUTDXInsecure: { + changeSnpFields: false, + want: string(expectedConfMetalQEMUTDXInsecure), + }, + platforms.MetalQEMUSNPGPUInsecure: { + changeSnpFields: true, + want: string(expectedConfMetalQEMUSNPGPUInsecure), + }, + platforms.MetalQEMUTDXGPUInsecure: { + changeSnpFields: false, + want: string(expectedConfMetalQEMUTDXGPUInsecure), + }, } for platform, tc := range testCases { t.Run(platform.String(), func(t *testing.T) { diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu-insecure.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu-insecure.toml new file mode 100644 index 00000000000..dab760540be --- /dev/null +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu-insecure.toml @@ -0,0 +1,117 @@ +[Hypervisor] +[Hypervisor.qemu] +block_device_aio = 'threads' +block_device_cache_direct = false +block_device_cache_noflush = false +block_device_cache_set = false +block_device_driver = 'virtio-scsi' +cold_plug_vfio = 'root-port' +confidential_guest = false +contrast_imagepuller_config = '' +cpu_features = 'pmu=off' +default_bridges = 1 +default_maxmemory = 0 +default_maxvcpus = 0 +default_memory = 1024 +default_vcpus = 1 +disable_block_device_use = true +disable_guest_selinux = true +disable_image_nvdimm = true +disable_nesting_checks = true +disable_selinux = false +disable_vhost_net = false +enable_annotations = ['cc_init_data'] +enable_debug = false +enable_guest_swap = false +enable_hugepages = false +enable_iommu = false +enable_iommu_platform = false +enable_iothreads = false +enable_mem_prealloc = false +enable_numa = false +enable_vhost_user_store = false +enable_virtio_mem = false +entropy_source = '/dev/urandom' +file_mem_backend = '' +firmware = '/snp/share/OVMF.fd' +firmware_volume = '' +guest_hook_path = '' +guest_memory_dump_paging = false +guest_memory_dump_path = '' +image = '/share/kata-containers.img' +indep_iothreads = 0 +initrd = '/share/kata-initrd.zst' +kernel = '/share/kata-kernel' +kernel_params = '' +machine_accelerators = '' +machine_type = 'q35' +memory_offset = 0 +memory_slots = 10 +msize_9p = 8192 +numa_mapping = [] +path = '/bin/qemu-system-x86_64' +pcie_root_port = 0 +pflashes = [] +reclaim_guest_freed_memory = false +rootfs_type = 'erofs' +rootless = false +rx_rate_limiter_max_rate = 0 +seccompsandbox = '' +sev_snp_guest = false +shared_fs = 'none' +snp_guest_policy = 196608 +snp_id_auth = '' +snp_id_block = '' +tx_rate_limiter_max_rate = 0 +use_legacy_serial = false +valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] +valid_file_mem_backends = [''] +valid_hypervisor_paths = ['/bin/qemu-system-x86_64'] +valid_vhost_user_store_paths = ['/var/run/kata-containers/vhost-user'] +valid_virtio_fs_daemon_paths = ['/opt/kata/libexec/virtiofsd'] +vhost_user_reconnect_timeout_sec = 0 +vhost_user_store_path = '/var/run/kata-containers/vhost-user' +virtio_fs_cache = 'auto' +virtio_fs_cache_size = 0 +virtio_fs_daemon = '/opt/kata/libexec/virtiofsd' +virtio_fs_extra_args = ['--thread-pool-size=1', '--announce-submounts'] +virtio_fs_queue_size = 1024 + +[Agent] +[Agent.kata] +debug_console_enabled = false +dial_timeout = 600 +enable_debug = false +enable_tracing = false +kernel_modules = [] + +[Factory] +enable_template = false +template_path = '/run/vc/vm/template' +vm_cache_endpoint = '/var/run/kata-containers/cache.sock' +vm_cache_number = 0 + +[Runtime] +create_container_timeout = 600 +dan_conf = '/run/kata-containers/dans' +disable_guest_empty_dir = false +disable_guest_seccomp = true +disable_new_netns = false +emptydir_mode = 'shared-fs' +enable_debug = false +enable_pprof = false +enable_tracing = false +enable_vcpus_pinning = false +experimental = [] +experimental_force_guest_pull = true +guest_selinux_label = '' +internetworking_model = 'tcfilter' +jaeger_endpoint = '' +jaeger_password = '' +jaeger_user = '' +kubelet_root_dir = '/var/lib/kubelet' +pod_resource_api_sock = '/var/lib/kubelet/pod-resources/kubelet.sock' +sandbox_bind_mounts = [] +sandbox_cgroup_only = true +static_sandbox_resource_mgmt = true +vfio_mode = 'guest-kernel' diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-insecure.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-insecure.toml new file mode 100644 index 00000000000..2776f829ad2 --- /dev/null +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-insecure.toml @@ -0,0 +1,116 @@ +[Hypervisor] +[Hypervisor.qemu] +block_device_aio = 'threads' +block_device_cache_direct = false +block_device_cache_noflush = false +block_device_cache_set = false +block_device_driver = 'virtio-scsi' +confidential_guest = false +contrast_imagepuller_config = '' +cpu_features = 'pmu=off' +default_bridges = 1 +default_maxmemory = 0 +default_maxvcpus = 0 +default_memory = 512 +default_vcpus = 1 +disable_block_device_use = true +disable_guest_selinux = true +disable_image_nvdimm = true +disable_nesting_checks = true +disable_selinux = false +disable_vhost_net = false +enable_annotations = ['cc_init_data'] +enable_debug = false +enable_guest_swap = false +enable_hugepages = false +enable_iommu = false +enable_iommu_platform = false +enable_iothreads = false +enable_mem_prealloc = false +enable_numa = false +enable_vhost_user_store = false +enable_virtio_mem = false +entropy_source = '/dev/urandom' +file_mem_backend = '' +firmware = '/snp/share/OVMF.fd' +firmware_volume = '' +guest_hook_path = '' +guest_memory_dump_paging = false +guest_memory_dump_path = '' +image = '/share/kata-containers.img' +indep_iothreads = 0 +initrd = '/share/kata-initrd.zst' +kernel = '/share/kata-kernel' +kernel_params = '' +machine_accelerators = '' +machine_type = 'q35' +memory_offset = 0 +memory_slots = 10 +msize_9p = 8192 +numa_mapping = [] +path = '/bin/qemu-system-x86_64' +pcie_root_port = 0 +pflashes = [] +reclaim_guest_freed_memory = false +rootfs_type = 'erofs' +rootless = false +rx_rate_limiter_max_rate = 0 +seccompsandbox = '' +sev_snp_guest = false +shared_fs = 'none' +snp_guest_policy = 196608 +snp_id_auth = '' +snp_id_block = '' +tx_rate_limiter_max_rate = 0 +use_legacy_serial = false +valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] +valid_file_mem_backends = [''] +valid_hypervisor_paths = ['/bin/qemu-system-x86_64'] +valid_vhost_user_store_paths = ['/var/run/kata-containers/vhost-user'] +valid_virtio_fs_daemon_paths = ['/opt/kata/libexec/virtiofsd'] +vhost_user_reconnect_timeout_sec = 0 +vhost_user_store_path = '/var/run/kata-containers/vhost-user' +virtio_fs_cache = 'auto' +virtio_fs_cache_size = 0 +virtio_fs_daemon = '/opt/kata/libexec/virtiofsd' +virtio_fs_extra_args = ['--thread-pool-size=1', '--announce-submounts'] +virtio_fs_queue_size = 1024 + +[Agent] +[Agent.kata] +debug_console_enabled = false +dial_timeout = 120 +enable_debug = false +enable_tracing = false +kernel_modules = [] + +[Factory] +enable_template = false +template_path = '/run/vc/vm/template' +vm_cache_endpoint = '/var/run/kata-containers/cache.sock' +vm_cache_number = 0 + +[Runtime] +create_container_timeout = 120 +dan_conf = '/run/kata-containers/dans' +disable_guest_empty_dir = false +disable_guest_seccomp = true +disable_new_netns = false +emptydir_mode = 'shared-fs' +enable_debug = false +enable_pprof = false +enable_tracing = false +enable_vcpus_pinning = false +experimental = [] +experimental_force_guest_pull = true +guest_selinux_label = '' +internetworking_model = 'tcfilter' +jaeger_endpoint = '' +jaeger_password = '' +jaeger_user = '' +kubelet_root_dir = '/var/lib/kubelet' +pod_resource_api_sock = '' +sandbox_bind_mounts = [] +sandbox_cgroup_only = true +static_sandbox_resource_mgmt = true +vfio_mode = 'guest-kernel' diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-gpu-insecure.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-gpu-insecure.toml new file mode 100644 index 00000000000..5e91f27cc4a --- /dev/null +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-gpu-insecure.toml @@ -0,0 +1,115 @@ +[Hypervisor] +[Hypervisor.qemu] +block_device_aio = 'threads' +block_device_cache_direct = false +block_device_cache_noflush = false +block_device_cache_set = false +block_device_driver = 'virtio-scsi' +cold_plug_vfio = 'root-port' +confidential_guest = false +contrast_imagepuller_config = '' +cpu_features = 'pmu=off' +default_bridges = 1 +default_maxmemory = 0 +default_maxvcpus = 0 +default_memory = 1024 +default_vcpus = 1 +disable_block_device_use = true +disable_guest_selinux = true +disable_image_nvdimm = true +disable_nesting_checks = false +disable_selinux = false +disable_vhost_net = false +enable_annotations = ['cc_init_data'] +enable_debug = false +enable_guest_swap = false +enable_hugepages = false +enable_iommu = false +enable_iommu_platform = false +enable_iothreads = false +enable_mem_prealloc = false +enable_numa = false +enable_vhost_user_store = false +enable_virtio_mem = false +entropy_source = '/dev/urandom' +file_mem_backend = '' +firmware = '/tdx/share/OVMF.fd' +firmware_volume = '' +guest_hook_path = '' +guest_memory_dump_paging = false +guest_memory_dump_path = '' +image = '/share/kata-containers.img' +indep_iothreads = 0 +initrd = '/share/kata-initrd.zst' +kernel = '/share/kata-kernel' +kernel_params = '' +kernel_verity_params = '' +machine_accelerators = '' +machine_type = 'q35' +memory_offset = 0 +memory_slots = 10 +msize_9p = 8192 +numa_mapping = [] +path = '/bin/qemu-system-x86_64' +pcie_root_port = 0 +pflashes = [] +reclaim_guest_freed_memory = false +rootfs_type = 'erofs' +rootless = false +rx_rate_limiter_max_rate = 0 +seccompsandbox = '' +shared_fs = 'none' +tdx_quote_generation_service_socket_port = 4050 +tx_rate_limiter_max_rate = 0 +use_legacy_serial = false +valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] +valid_file_mem_backends = [''] +valid_hypervisor_paths = ['/bin/qemu-system-x86_64'] +valid_vhost_user_store_paths = ['/var/run/kata-containers/vhost-user'] +valid_virtio_fs_daemon_paths = ['/opt/kata/libexec/virtiofsd'] +vhost_user_reconnect_timeout_sec = 0 +vhost_user_store_path = '/var/run/kata-containers/vhost-user' +virtio_fs_cache = 'auto' +virtio_fs_cache_size = 0 +virtio_fs_daemon = '/opt/kata/libexec/virtiofsd' +virtio_fs_extra_args = ['--thread-pool-size=1', '--announce-submounts'] +virtio_fs_queue_size = 1024 + +[Agent] +[Agent.kata] +debug_console_enabled = false +dial_timeout = 600 +enable_debug = false +enable_tracing = false +kernel_modules = [] + +[Factory] +enable_template = false +template_path = '/run/vc/vm/template' +vm_cache_endpoint = '/var/run/kata-containers/cache.sock' +vm_cache_number = 0 + +[Runtime] +create_container_timeout = 600 +dan_conf = '/run/kata-containers/dans' +disable_guest_empty_dir = false +disable_guest_seccomp = true +disable_new_netns = false +emptydir_mode = 'shared-fs' +enable_debug = false +enable_pprof = false +enable_tracing = false +enable_vcpus_pinning = false +experimental = [] +experimental_force_guest_pull = true +guest_selinux_label = '' +internetworking_model = 'tcfilter' +jaeger_endpoint = '' +jaeger_password = '' +jaeger_user = '' +kubelet_root_dir = '/var/lib/kubelet' +pod_resource_api_sock = '/var/lib/kubelet/pod-resources/kubelet.sock' +sandbox_bind_mounts = [] +sandbox_cgroup_only = true +static_sandbox_resource_mgmt = true +vfio_mode = 'guest-kernel' diff --git a/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-insecure.toml b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-insecure.toml new file mode 100644 index 00000000000..bf5205cc826 --- /dev/null +++ b/nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-insecure.toml @@ -0,0 +1,114 @@ +[Hypervisor] +[Hypervisor.qemu] +block_device_aio = 'threads' +block_device_cache_direct = false +block_device_cache_noflush = false +block_device_cache_set = false +block_device_driver = 'virtio-scsi' +confidential_guest = false +contrast_imagepuller_config = '' +cpu_features = 'pmu=off' +default_bridges = 1 +default_maxmemory = 0 +default_maxvcpus = 0 +default_memory = 512 +default_vcpus = 1 +disable_block_device_use = true +disable_guest_selinux = true +disable_image_nvdimm = true +disable_nesting_checks = false +disable_selinux = false +disable_vhost_net = false +enable_annotations = ['cc_init_data'] +enable_debug = false +enable_guest_swap = false +enable_hugepages = false +enable_iommu = false +enable_iommu_platform = false +enable_iothreads = false +enable_mem_prealloc = false +enable_numa = false +enable_vhost_user_store = false +enable_virtio_mem = false +entropy_source = '/dev/urandom' +file_mem_backend = '' +firmware = '/tdx/share/OVMF.fd' +firmware_volume = '' +guest_hook_path = '' +guest_memory_dump_paging = false +guest_memory_dump_path = '' +image = '/share/kata-containers.img' +indep_iothreads = 0 +initrd = '/share/kata-initrd.zst' +kernel = '/share/kata-kernel' +kernel_params = '' +kernel_verity_params = '' +machine_accelerators = '' +machine_type = 'q35' +memory_offset = 0 +memory_slots = 10 +msize_9p = 8192 +numa_mapping = [] +path = '/bin/qemu-system-x86_64' +pcie_root_port = 0 +pflashes = [] +reclaim_guest_freed_memory = false +rootfs_type = 'erofs' +rootless = false +rx_rate_limiter_max_rate = 0 +seccompsandbox = '' +shared_fs = 'none' +tdx_quote_generation_service_socket_port = 4050 +tx_rate_limiter_max_rate = 0 +use_legacy_serial = false +valid_entropy_sources = ['/dev/urandom', '/dev/random', ''] +valid_file_mem_backends = [''] +valid_hypervisor_paths = ['/bin/qemu-system-x86_64'] +valid_vhost_user_store_paths = ['/var/run/kata-containers/vhost-user'] +valid_virtio_fs_daemon_paths = ['/opt/kata/libexec/virtiofsd'] +vhost_user_reconnect_timeout_sec = 0 +vhost_user_store_path = '/var/run/kata-containers/vhost-user' +virtio_fs_cache = 'auto' +virtio_fs_cache_size = 0 +virtio_fs_daemon = '/opt/kata/libexec/virtiofsd' +virtio_fs_extra_args = ['--thread-pool-size=1', '--announce-submounts'] +virtio_fs_queue_size = 1024 + +[Agent] +[Agent.kata] +debug_console_enabled = false +dial_timeout = 120 +enable_debug = false +enable_tracing = false +kernel_modules = [] + +[Factory] +enable_template = false +template_path = '/run/vc/vm/template' +vm_cache_endpoint = '/var/run/kata-containers/cache.sock' +vm_cache_number = 0 + +[Runtime] +create_container_timeout = 120 +dan_conf = '/run/kata-containers/dans' +disable_guest_empty_dir = false +disable_guest_seccomp = true +disable_new_netns = false +emptydir_mode = 'shared-fs' +enable_debug = false +enable_pprof = false +enable_tracing = false +enable_vcpus_pinning = false +experimental = [] +experimental_force_guest_pull = true +guest_selinux_label = '' +internetworking_model = 'tcfilter' +jaeger_endpoint = '' +jaeger_password = '' +jaeger_user = '' +kubelet_root_dir = '/var/lib/kubelet' +pod_resource_api_sock = '' +sandbox_bind_mounts = [] +sandbox_cgroup_only = true +static_sandbox_resource_mgmt = true +vfio_mode = 'guest-kernel' diff --git a/nodeinstaller/internal/kataconfig/update-testdata/main.go b/nodeinstaller/internal/kataconfig/update-testdata/main.go index 5745844b732..5e8d6f66be1 100644 --- a/nodeinstaller/internal/kataconfig/update-testdata/main.go +++ b/nodeinstaller/internal/kataconfig/update-testdata/main.go @@ -46,6 +46,30 @@ func main() { config: "qemu-tdx", testdata: "qemu-tdx-gpu", }, + // We intentionally take the CC upstream configs for the + // insecure platforms and then drop the CC-specific parameters + // ourselves in the `kataconfig` package to keep the CC and + // non-CC configurations as close as possible. + platforms.MetalQEMUSNPInsecure: { + upstream: "qemu-snp", + config: "qemu-snp", + testdata: "qemu-snp-insecure", + }, + platforms.MetalQEMUTDXInsecure: { + upstream: "qemu-tdx", + config: "qemu-tdx", + testdata: "qemu-tdx-insecure", + }, + platforms.MetalQEMUSNPGPUInsecure: { + upstream: "qemu-snp", + config: "qemu-snp", + testdata: "qemu-snp-gpu-insecure", + }, + platforms.MetalQEMUTDXGPUInsecure: { + upstream: "qemu-tdx", + config: "qemu-tdx", + testdata: "qemu-tdx-gpu-insecure", + }, } for platform, platformConfig := range platforms { diff --git a/nodeinstaller/internal/targetconfig/targetconfig.go b/nodeinstaller/internal/targetconfig/targetconfig.go index 88c3c5abf6f..f7bf7933358 100644 --- a/nodeinstaller/internal/targetconfig/targetconfig.go +++ b/nodeinstaller/internal/targetconfig/targetconfig.go @@ -35,10 +35,10 @@ func NewTargetConfig(hostMount, runtimeBase string, pl platforms.Platform) (*Tar hostMount: hostMount, fs: &afero.Afero{Fs: afero.NewOsFs()}, } - switch { - case platforms.IsQEMU(pl) && platforms.IsSNP(pl): + switch pl { + case platforms.MetalQEMUSNP, platforms.MetalQEMUSNPGPU, platforms.MetalQEMUSNPInsecure, platforms.MetalQEMUSNPGPUInsecure: conf.kataConfigPath = filepath.Join(runtimeBase, "etc", "configuration-qemu-snp.toml") - case platforms.IsQEMU(pl) && platforms.IsTDX(pl): + case platforms.MetalQEMUTDX, platforms.MetalQEMUTDXGPU, platforms.MetalQEMUTDXInsecure, platforms.MetalQEMUTDXGPUInsecure: conf.kataConfigPath = filepath.Join(runtimeBase, "etc", "configuration-qemu-tdx.toml") default: return nil, fmt.Errorf("unsupported platform %q", pl) diff --git a/nodeinstaller/internal/targetconfig/targetconfig_test.go b/nodeinstaller/internal/targetconfig/targetconfig_test.go index 4149ef5c877..2dbcbfe16a6 100644 --- a/nodeinstaller/internal/targetconfig/targetconfig_test.go +++ b/nodeinstaller/internal/targetconfig/targetconfig_test.go @@ -44,6 +44,30 @@ func TestNewTargetConfig(t *testing.T) { hostMount: "/host", }, }, + "valid config for metal qemu snp insecure": { + hostMount: "/host", + runtimeBase: "/opt/edgeless/qemu", + platform: platforms.MetalQEMUSNPInsecure, + wantErr: false, + wantConfig: &TargetConfig{ + containerdConfigPath: "etc/containerd/config.toml", + systemdUnitNames: []string{"containerd.service"}, + kataConfigPath: "/opt/edgeless/qemu/etc/configuration-qemu-snp.toml", + hostMount: "/host", + }, + }, + "valid config for metal qemu tdx insecure": { + hostMount: "/host", + runtimeBase: "/opt/edgeless/qemu", + platform: platforms.MetalQEMUTDXInsecure, + wantErr: false, + wantConfig: &TargetConfig{ + containerdConfigPath: "etc/containerd/config.toml", + systemdUnitNames: []string{"containerd.service"}, + kataConfigPath: "/opt/edgeless/qemu/etc/configuration-qemu-tdx.toml", + hostMount: "/host", + }, + }, "invalid platform": { hostMount: "/host", runtimeBase: "/opt/edgeless/unknown", diff --git a/packages/by-name/contrast/e2e/package.nix b/packages/by-name/contrast/e2e/package.nix index 96c64502370..ba2da89db8a 100644 --- a/packages/by-name/contrast/e2e/package.nix +++ b/packages/by-name/contrast/e2e/package.nix @@ -71,6 +71,7 @@ buildGoModule { "e2e/gpu" "e2e/imagepuller-auth" "e2e/imagestore" + "e2e/insecure" "e2e/kds-pcs-downtime" "e2e/memdump" "e2e/multi-runtime-class" diff --git a/packages/by-name/contrast/reference-values/package.nix b/packages/by-name/contrast/reference-values/package.nix index c071d2119eb..60ce7d2414b 100644 --- a/packages/by-name/contrast/reference-values/package.nix +++ b/packages/by-name/contrast/reference-values/package.nix @@ -10,13 +10,16 @@ let runtimeHandler = - platform: hashFile: - "contrast-cc-${platform}-${builtins.substring 0 8 (builtins.readFile hashFile)}"; + platform: hashFile: "contrast-${platform}-${builtins.substring 0 8 (builtins.readFile hashFile)}"; - metal-qemu-tdx-handler = runtimeHandler "metal-qemu-tdx" node-installer-image.runtimeHash; - metal-qemu-snp-handler = runtimeHandler "metal-qemu-snp" node-installer-image.runtimeHash; - metal-qemu-snp-gpu-handler = runtimeHandler "metal-qemu-snp-gpu" node-installer-image.runtimeHash; - metal-qemu-tdx-gpu-handler = runtimeHandler "metal-qemu-tdx-gpu" node-installer-image.runtimeHash; + cc-metal-qemu-tdx-handler = runtimeHandler "cc-metal-qemu-tdx" node-installer-image.runtimeHash; + cc-metal-qemu-snp-handler = runtimeHandler "cc-metal-qemu-snp" node-installer-image.runtimeHash; + cc-metal-qemu-snp-gpu-handler = runtimeHandler "cc-metal-qemu-snp-gpu" node-installer-image.runtimeHash; + cc-metal-qemu-tdx-gpu-handler = runtimeHandler "cc-metal-qemu-tdx-gpu" node-installer-image.runtimeHash; + insecure-metal-qemu-snp-handler = runtimeHandler "insecure-metal-qemu-snp" node-installer-image.runtimeHash; + insecure-metal-qemu-snp-gpu-handler = runtimeHandler "insecure-metal-qemu-snp-gpu" node-installer-image.runtimeHash; + insecure-metal-qemu-tdx-handler = runtimeHandler "insecure-metal-qemu-tdx" node-installer-image.runtimeHash; + insecure-metal-qemu-tdx-gpu-handler = runtimeHandler "insecure-metal-qemu-tdx-gpu" node-installer-image.runtimeHash; snpRefValsWith = os-image: { snp = @@ -96,13 +99,23 @@ let }; withGPU = true; }; + insecureSnpRefVals = { + snp = [ { } ]; + }; + insecureTdxRefVals = { + tdx = [ { } ]; + }; in builtins.toFile "reference-values.json" ( builtins.toJSON { - "${metal-qemu-tdx-handler}" = tdxRefVals; - "${metal-qemu-snp-handler}" = snpRefVals; - "${metal-qemu-snp-gpu-handler}" = snpGpuRefVals; - "${metal-qemu-tdx-gpu-handler}" = tdxGpuRefVals; + "${cc-metal-qemu-tdx-handler}" = tdxRefVals; + "${cc-metal-qemu-snp-handler}" = snpRefVals; + "${cc-metal-qemu-snp-gpu-handler}" = snpGpuRefVals; + "${cc-metal-qemu-tdx-gpu-handler}" = tdxGpuRefVals; + "${insecure-metal-qemu-snp-handler}" = insecureSnpRefVals; + "${insecure-metal-qemu-snp-gpu-handler}" = insecureSnpRefVals; + "${insecure-metal-qemu-tdx-handler}" = insecureTdxRefVals; + "${insecure-metal-qemu-tdx-gpu-handler}" = insecureTdxRefVals; } ) diff --git a/packages/by-name/initdata-processor/package.nix b/packages/by-name/initdata-processor/package.nix index 7d036b11d58..d44bddc3b3a 100644 --- a/packages/by-name/initdata-processor/package.nix +++ b/packages/by-name/initdata-processor/package.nix @@ -48,6 +48,8 @@ buildGoModule (finalAttrs: { (path.append root "go.sum") (path.append root "initdata-processor/go.mod") (path.append root "initdata-processor/go.sum") + (fileset.fileFilter (file: hasSuffix ".go" file.name) (path.append root "internal/attestation")) + (fileset.fileFilter (file: hasSuffix ".go" file.name) (path.append root "internal/oid")) (fileset.fileFilter (file: hasSuffix ".go" file.name) (path.append root "internal/initdata")) (fileset.fileFilter (file: hasSuffix ".go" file.name) (path.append root "initdata-processor")) ]; diff --git a/packages/nixos/kata.nix b/packages/nixos/kata.nix index 2c45f000726..7f188d0451f 100644 --- a/packages/nixos/kata.nix +++ b/packages/nixos/kata.nix @@ -57,7 +57,12 @@ in wants = [ "initdata.target" ]; serviceConfig = { - Type = "oneshot"; + # notify: the process signals READY=1 when initdata processing is + # complete, allowing initdata.target to be reached. On insecure + # platforms the process stays alive to serve hostdata via HTTP; + # on CC platforms it exits after signaling. + Type = "notify"; + NotifyAccess = "main"; RemainAfterExit = "yes"; ExecStart = lib.getExe pkgs.contrastPkgs.initdata-processor; }; diff --git a/sdk/common.go b/sdk/common.go index 6e686c97d15..fb3692ea711 100644 --- a/sdk/common.go +++ b/sdk/common.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/contrast/internal/atls" "github.com/edgelesssys/contrast/internal/attestation/certcache" + "github.com/edgelesssys/contrast/internal/attestation/insecure" "github.com/edgelesssys/contrast/internal/attestation/snp" "github.com/edgelesssys/contrast/internal/attestation/tdx" "github.com/edgelesssys/contrast/internal/logger" @@ -57,5 +58,12 @@ func ValidatorsFromManifest(kdsGetter *certcache.CachedHTTPSGetter, m *manifest. validators = append(validators, tdx.NewValidator(opt.VerifyOpts, &tdx.StaticValidateOptsGenerator{Opts: opt.ValidateOpts}, opt.AllowedPIIDs, logger.NewWithAttrs(logger.NewNamed(log, "validator"), map[string]string{"reference-values": name}), name)) } + if m.AllowInsecure() { + validators = append(validators, insecure.NewValidator( + logger.NewWithAttrs(logger.NewNamed(log, "validator"), map[string]string{"reference-values": "insecure"}), + "insecure", + )) + } + return validators, nil } diff --git a/sdk/verify.go b/sdk/verify.go index fd0fb78beff..49de0c486f1 100644 --- a/sdk/verify.go +++ b/sdk/verify.go @@ -37,6 +37,11 @@ type Client struct { log *slog.Logger + // allowInsecure must be set to true to allow verification of manifests + // that contain insecure (non-CC) reference values. Without this, ValidateAttestation + // will return an error if the manifest allows insecure platforms. + allowInsecure bool + // validatorsFromManifestOverride is used by tests to replace the validators. validatorsFromManifestOverride func(*certcache.CachedHTTPSGetter, *manifest.Manifest, *slog.Logger) ([]atls.Validator, error) } @@ -80,6 +85,15 @@ func (c *Client) WithHTTPClient(httpClient *http.Client) *Client { return c } +// WithInsecure allows the Client to verify manifests containing insecure (non-CC) reference values. +// +// By default, [Client.ValidateAttestation] will return an error if the manifest allows insecure +// platforms. This method opts in to accepting such manifests. +func (c *Client) WithInsecure() *Client { + c.allowInsecure = true + return c +} + // GetAttestation requests attestation evidence from the Coordinator's HTTP API. // // The URL needs to map to the http://coordinator:1314/attest endpoint, but can be reverse-proxied @@ -159,6 +173,10 @@ func (c Client) ValidateAttestation(ctx context.Context, nonce []byte, attestati return nil, fmt.Errorf("validating latest manifest: %w", err) } + if latestManifest.AllowInsecure() && !c.allowInsecure { + return nil, fmt.Errorf("manifest contains insecure platforms: use WithInsecure() to allow verification of insecure deployments") + } + kdsGetter := certcache.NewCachedHTTPSGetter(c.fsstore, certcache.NeverGCTicker, c.log.WithGroup("kds-getter")) validatorsFromManifest := ValidatorsFromManifest if c.validatorsFromManifestOverride != nil { diff --git a/sdk/verify_test.go b/sdk/verify_test.go index e4f602a18ad..afaec064903 100644 --- a/sdk/verify_test.go +++ b/sdk/verify_test.go @@ -107,10 +107,11 @@ func TestGetAttestation(t *testing.T) { func TestValidateAttestation(t *testing.T) { testNonce := make([]byte, 32) for name, tc := range map[string]struct { - nonce []byte - resp *httpapi.AttestationResponse - validateErr error - wantErr string + nonce []byte + resp *httpapi.AttestationResponse + validateErr error + allowInsecure bool + wantErr string }{ "success": { nonce: testNonce, @@ -143,6 +144,26 @@ func TestValidateAttestation(t *testing.T) { validateErr: assert.AnError, wantErr: assert.AnError.Error(), }, + "insecure manifest without opt-in": { + nonce: testNonce, + resp: &httpapi.AttestationResponse{ + RawAttestationDoc: testNonce, + CoordinatorState: httpapi.CoordinatorState{ + Manifests: [][]byte{testInsecureManifest}, + }, + }, + wantErr: "WithInsecure", + }, + "insecure manifest with opt-in": { + nonce: testNonce, + allowInsecure: true, + resp: &httpapi.AttestationResponse{ + RawAttestationDoc: testNonce, + CoordinatorState: httpapi.CoordinatorState{ + Manifests: [][]byte{testInsecureManifest}, + }, + }, + }, } { t.Run(name, func(t *testing.T) { assert := assert.New(t) @@ -152,6 +173,9 @@ func TestValidateAttestation(t *testing.T) { require.NoError(err) c := New() + if tc.allowInsecure { + c = c.WithInsecure() + } c.validatorsFromManifestOverride = func(*certcache.CachedHTTPSGetter, *manifest.Manifest, *slog.Logger) ([]atls.Validator, error) { return []atls.Validator{&stubValidator{err: tc.validateErr}}, nil @@ -224,6 +248,25 @@ var testManifest = []byte(` } `) +var testInsecureManifest = []byte(` +{ + "Policies": { + "ef27c1c91a0ce044c67f0ec10d7c66ea9f178453dc96a233e97f0675578042f2": { + "SANs": ["coordinator"], + "WorkloadSecretID": "apps/v1/StatefulSet/default/coordinator", + "Role": "coordinator" + } + }, + "ReferenceValues": { + "snp": [ + { + "Platform": "metal-qemu-snp-insecure" + } + ] + } +} +`) + type stubValidator struct { atls.Validator