Skip to content

Commit 0271769

Browse files
andfasanoandfasano
committed
add initial e2e tests for the InternalReleaseImage controller
1 parent de8ed45 commit 0271769

4 files changed

Lines changed: 242 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,9 @@ test-e2e-ocl: install-go-junit-report install-skopeo
226226
# Temporarily include /tmp/skopeo/bin in our PATH variable so that the test suite can find skopeo.
227227
set -o pipefail; PATH="$(PATH):/tmp/skopeo/bin" go test -tags=$(GOTAGS) -failfast -timeout 190m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
228228

229+
test-e2e-iri: install-go-junit-report
230+
set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-iri/ | ./hack/test-with-junit.sh $(@)
231+
229232
bootstrap-e2e: install-go-junit-report install-setup-envtest
230233
@echo "Setting up KUBEBUILDER_ASSETS"
231234
@KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --index https://raw.githubusercontent.com/openshift/api/master/envtest-releases.yaml --bin-dir $(PROJECT_DIR)/bin -p path)" && \

‎test/e2e-iri/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# e2e IRI tests
2+
3+
This test package should be used for testing the behaviors of the InternalReleaseImage controller.
4+
Currently it is deployed as part of the Agent Installer for OpenShift-Virt (OVE) - and specifically for the NoRegistryClusterInstall feature.
5+
To run the tests, deploy first a baremetal cluster using the OVE ISO.

‎test/e2e-iri/iri_test.go‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package e2e_iri_test
2+
3+
import (
4+
"context"
5+
"crypto/tls"
6+
"crypto/x509"
7+
"fmt"
8+
"net/http"
9+
"reflect"
10+
"testing"
11+
"time"
12+
13+
"github.com/stretchr/testify/require"
14+
corev1 "k8s.io/api/core/v1"
15+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
16+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/apimachinery/pkg/util/wait"
18+
19+
configv1 "github.com/openshift/api/config/v1"
20+
mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
21+
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
22+
"github.com/openshift/machine-config-operator/test/framework"
23+
)
24+
25+
func TestIRIResource_Available(t *testing.T) {
26+
cs := framework.NewClientSet("")
27+
ctx := context.Background()
28+
29+
// Check that the initial InternalReleaseImage resource has been installed.
30+
_, err := cs.InternalReleaseImages().Get(ctx, "cluster", v1.GetOptions{})
31+
require.NoError(t, err)
32+
33+
// Verify that the expected MachineConfigs have been created.
34+
_, err = cs.MachineConfigs().Get(ctx, "02-master-internalreleaseimage", v1.GetOptions{})
35+
require.NoError(t, err)
36+
_, err = cs.MachineConfigs().Get(ctx, "02-worker-internalreleaseimage", v1.GetOptions{})
37+
require.NoError(t, err)
38+
}
39+
40+
func TestIRIController_VerifyIRIRegistryOnAllTheMasterNodes_NoCert(t *testing.T) {
41+
masterNodes, err := framework.NewClientSet("").CoreV1Interface.Nodes().List(context.TODO(), v1.ListOptions{LabelSelector: "node-role.kubernetes.io/master="})
42+
require.NoError(t, err)
43+
44+
// For every control plane node, ping the IRI registry at port 22625,
45+
// using directly the node IP.
46+
for _, node := range masterNodes.Items {
47+
nodeAddr := ""
48+
for _, addr := range node.Status.Addresses {
49+
if addr.Type == corev1.NodeInternalIP {
50+
nodeAddr = addr.Address
51+
break
52+
}
53+
}
54+
require.NotEmpty(t, nodeAddr)
55+
56+
client := &http.Client{
57+
Transport: &http.Transport{
58+
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
59+
},
60+
Timeout: 30 * time.Second,
61+
}
62+
pingIRIRegistry(t, client, nodeAddr)
63+
}
64+
}
65+
66+
func skipIfNoBaremetal(t *testing.T) {
67+
infra, err := framework.NewClientSet("").Infrastructures().Get(context.Background(), "cluster", v1.GetOptions{})
68+
require.NoError(t, err)
69+
if infra.Status.PlatformStatus.Type != configv1.BareMetalPlatformType {
70+
t.Skip("Skipping non-baremetal platforms")
71+
}
72+
}
73+
74+
func TestIRIController_VerifyIRIRegistryOnApiInt_WithCert(t *testing.T) {
75+
cs := framework.NewClientSet("")
76+
ctx := context.Background()
77+
78+
skipIfNoBaremetal(t)
79+
80+
// Retrieve the root CA (same of MCS).
81+
cm, err := cs.ConfigMaps("openshift-machine-config-operator").Get(ctx, "machine-config-server-ca", v1.GetOptions{})
82+
require.NoError(t, err)
83+
rootCA := []byte(cm.Data["ca-bundle.crt"])
84+
roots := x509.NewCertPool()
85+
require.True(t, roots.AppendCertsFromPEM(rootCA))
86+
87+
client := &http.Client{
88+
Transport: &http.Transport{
89+
TLSClientConfig: &tls.Config{
90+
RootCAs: roots,
91+
},
92+
},
93+
Timeout: 30 * time.Second,
94+
}
95+
96+
// The api-int DNS entry may be not resolvable in the current environment.
97+
// Let's fetch directly the related IP address.
98+
infra, err := cs.Infrastructures().Get(ctx, "cluster", v1.GetOptions{})
99+
require.NoError(t, err)
100+
require.NotEmpty(t, infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs)
101+
102+
pingIRIRegistry(t, client, infra.Status.PlatformStatus.BareMetal.APIServerInternalIPs[0])
103+
}
104+
105+
func pingIRIRegistry(t *testing.T, client *http.Client, ipAddr string) {
106+
iriRegistryUrl := fmt.Sprintf("https://%s:%d/v2/", ipAddr, 22625)
107+
108+
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, iriRegistryUrl, nil)
109+
require.NoError(t, err)
110+
resp, err := client.Do(req)
111+
require.NoError(t, err)
112+
defer resp.Body.Close()
113+
114+
require.Equal(t, resp.StatusCode, http.StatusOK)
115+
apiVersion := resp.Header.Get("Docker-Distribution-Api-Version")
116+
require.Equal(t, apiVersion, "registry/2.0")
117+
}
118+
119+
func TestIRIController_ShouldRestoreMachineConfigsWhenModified(t *testing.T) {
120+
cases := []struct {
121+
name string
122+
userAction func(t *testing.T, ctx context.Context, cs *framework.ClientSet, configs []*mcfgv1.MachineConfig)
123+
}{
124+
{
125+
name: "user deletes all the IRI machine configs",
126+
userAction: func(t *testing.T, ctx context.Context, cs *framework.ClientSet, configs []*mcfgv1.MachineConfig) {
127+
// Delete all the IRI MachineConfigs found.
128+
for _, mc := range configs {
129+
err := cs.MachineConfigs().Delete(ctx, mc.Name, v1.DeleteOptions{})
130+
require.NoError(t, err)
131+
}
132+
},
133+
},
134+
{
135+
name: "user patches all the IRI machine configs",
136+
userAction: func(t *testing.T, ctx context.Context, cs *framework.ClientSet, configs []*mcfgv1.MachineConfig) {
137+
// Update all the IRI MachineConfigs found.
138+
for _, mc := range configs {
139+
updatedMC := mc.DeepCopy()
140+
updatedMC.Spec.OSImageURL = "some-unused-value"
141+
_, err := cs.MachineConfigs().Update(ctx, updatedMC, v1.UpdateOptions{})
142+
require.NoError(t, err)
143+
}
144+
},
145+
},
146+
}
147+
for _, tc := range cases {
148+
t.Run(tc.name, func(t *testing.T) {
149+
cs := framework.NewClientSet("")
150+
ctx := context.Background()
151+
152+
// Check that initially IRI MachineConfigs are defined.
153+
machineConfigs, err := cs.MachineConfigs().List(ctx, v1.ListOptions{})
154+
require.NoError(t, err)
155+
iriMachineConfigs := []*mcfgv1.MachineConfig{}
156+
for _, mc := range machineConfigs.Items {
157+
if len(mc.OwnerReferences) != 0 && mc.OwnerReferences[0].Kind == "InternalReleaseImage" && mc.OwnerReferences[0].Name == "cluster" {
158+
iriMachineConfigs = append(iriMachineConfigs, mc.DeepCopy())
159+
}
160+
}
161+
require.NotEmpty(t, iriMachineConfigs)
162+
163+
// Apply the user action.
164+
tc.userAction(t, ctx, cs, iriMachineConfigs)
165+
166+
// Wait until all of them will be restored.
167+
err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 1*time.Minute, true, func(ctx context.Context) (done bool, err error) {
168+
for _, oldMC := range iriMachineConfigs {
169+
newMC, err := cs.MachineConfigs().Get(ctx, oldMC.Name, v1.GetOptions{})
170+
if k8serrors.IsNotFound(err) {
171+
return false, nil
172+
}
173+
if err != nil {
174+
return true, err
175+
}
176+
// Check that the ignition is really the same.
177+
oldIgn, err := ctrlcommon.ParseAndConvertConfig(oldMC.Spec.Config.Raw)
178+
require.NoError(t, err)
179+
newIgn, err := ctrlcommon.ParseAndConvertConfig(newMC.Spec.Config.Raw)
180+
require.NoError(t, err)
181+
if !reflect.DeepEqual(oldIgn, newIgn) {
182+
return true, fmt.Errorf("newer version of MachineConfig %s is different from the original one.", oldMC.Name)
183+
}
184+
}
185+
return true, nil
186+
})
187+
require.NoError(t, err)
188+
})
189+
}
190+
}

‎test/e2e-iri/main_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package e2e_iri_test
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"testing"
8+
9+
"github.com/openshift/api/features"
10+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
12+
"github.com/openshift/machine-config-operator/test/framework"
13+
)
14+
15+
func TestMain(m *testing.M) {
16+
skip, err := skipIRITests()
17+
if err != nil {
18+
fmt.Fprintf(os.Stderr, "skip IRI check failed: %v\n", err)
19+
os.Exit(1)
20+
}
21+
if skip {
22+
os.Exit(0)
23+
}
24+
os.Exit(m.Run())
25+
}
26+
27+
func skipIRITests() (bool, error) {
28+
cs := framework.NewClientSet("")
29+
ctx := context.Background()
30+
31+
// Check if NoRegistryClusterInstall feature is enabled.
32+
fg, err := cs.FeatureGates().Get(ctx, "cluster", v1.GetOptions{})
33+
if err != nil {
34+
return true, err
35+
}
36+
// Assume only one version has been installed.
37+
for _, d := range fg.Status.FeatureGates[0].Disabled {
38+
if d.Name == features.FeatureGateNoRegistryClusterInstall {
39+
return true, nil
40+
}
41+
}
42+
43+
return false, nil
44+
}

0 commit comments

Comments
 (0)