ci: test cluster with private node-groups

upcloud/kubernetes/resource_upcloud_kubernetes_test.go

@@ -230,14 +230,21 @@ func TestAccUpcloudKubernetes_storageEncryption(t *testing.T) {
 	})
 }
 
-func TestEndToEndKubernetes(t *testing.T) {
-	t.Log(`This testcase:
+func testEndToEndKubernetes(t *testing.T, cidr string, privateNodeGroups bool) {
+	serviceType := "NodePort"
+	nodeAccess := "public"
+	if privateNodeGroups {
+		serviceType = "LoadBalancer"
+		nodeAccess = "private"
+	}
+
+	t.Logf(`This testcase:
 
-- Creates a Kubernetes cluster with one node group.
+- Creates a Kubernetes cluster with one %s node group.
 - Configures Kubernetes provider to connect to the created cluster using ephemeral cluster resource.
 - Deploys hello deployment and service to the cluster.
-- Uses http data source to verify that the deployment is reachable through a node port.
-`)
+- Uses http data source to verify that the deployment is reachable through a %s.
+`, nodeAccess, serviceType)
 
 	testdata := utils.ReadTestDataFile(t, "../testdata/upcloud_kubernetes/kubernetes_e2e.tf")
 
@@ -256,13 +263,19 @@ func TestEndToEndKubernetes(t *testing.T) {
 			{
 				// Create the cluster first and add kubernetes resources in the next step.
 				Config: testdata,
+				ConfigVariables: map[string]config.Variable{
+					"private_node_groups": config.BoolVariable(privateNodeGroups),
+					"network_cidr":        config.StringVariable(cidr),
+				},
 				// OpenTofu adds open action for the ephemeral resource which causes the plan to be non-empty.
 				ExpectNonEmptyPlan: upcloud.UsingOpenTofu(),
 			},
 			{
 				Config: testdata,
 				ConfigVariables: map[string]config.Variable{
 					"enable_kubernetes_resources": config.BoolVariable(true),
+					"network_cidr":                config.StringVariable(cidr),
+					"private_node_groups":         config.BoolVariable(privateNodeGroups),
 				},
 				ConfigPlanChecks: resource.ConfigPlanChecks{
 					PreApply: []plancheck.PlanCheck{
@@ -279,3 +292,11 @@ func TestEndToEndKubernetes(t *testing.T) {
 		},
 	})
 }
+
+func TestEndToEndKubernetes_PublicNodePort(t *testing.T) {
+	testEndToEndKubernetes(t, "172.23.45.0/24", false)
+}
+
+func TestEndToEndKubernetes_PrivateLoadBalancer(t *testing.T) {
+	testEndToEndKubernetes(t, "172.23.46.0/24", true)
+}

upcloud/testdata/upcloud_kubernetes/kubernetes_e2e.tf

@@ -3,6 +3,11 @@ variable "basename" {
   type    = string
 }
 
+variable "network_cidr" {
+  default = "172.23.45.0/24"
+  type    = string
+}
+
 variable "zone" {
   default = "pl-waw1"
   type    = string
@@ -18,8 +23,13 @@ variable "enable_kubernetes_resources" {
   default = false
 }
 
+variable "private_node_groups" {
+  type    = bool
+  default = false
+}
+
 locals {
-  name_prefix = "${var.basename}k8s-e2e-"
+  name_prefix = "${var.basename}k8s-e2e-${var.private_node_groups ? "priv-lb" : "publ-np"}-"
 }
 
 resource "upcloud_router" "main" {
@@ -32,16 +42,31 @@ resource "upcloud_network" "main" {
   router = upcloud_router.main.id
 
   ip_network {
-    address = "172.23.45.0/24"
+    address = var.network_cidr
     dhcp    = true
+    dhcp_default_route = var.private_node_groups
     family  = "IPv4"
   }
 }
 
+resource "upcloud_gateway" "main" {
+  # Only deploy NAT gateway when using private node-groups
+  count = var.private_node_groups ? 1 : 0
+
+  name     = "${local.name_prefix}gw"
+  zone     = var.zone
+  features = ["nat"]
+
+  router {
+    id = upcloud_router.main.id
+  }
+}
+
 resource "upcloud_kubernetes_cluster" "main" {
   control_plane_ip_filter = ["0.0.0.0/0"]
   name                    = "${local.name_prefix}cluster"
   network                 = upcloud_network.main.id
+  private_node_groups     = var.private_node_groups
   zone                    = var.zone
 }
 
@@ -62,6 +87,10 @@ provider "kubernetes" {
   client_key             = ephemeral.upcloud_kubernetes_cluster.main.client_key
   cluster_ca_certificate = ephemeral.upcloud_kubernetes_cluster.main.cluster_ca_certificate
   host                   = ephemeral.upcloud_kubernetes_cluster.main.host
+
+  ignore_annotations = [
+    "^service\\.beta\\.kubernetes\\.io\\/.*load.*balancer.*"
+  ]
 }
 
 data "kubernetes_nodes" "this" {
@@ -126,11 +155,11 @@ resource "kubernetes_service_v1" "hello" {
     }
 
     port {
-      port        = 80
+      port = var.private_node_groups ? 443 : 80
       target_port = 80
     }
 
-    type = "NodePort"
+    type = var.private_node_groups ? "LoadBalancer" : "NodePort"
   }
 }
 
@@ -139,7 +168,8 @@ locals {
   has_external_ip = var.enable_kubernetes_resources ? contains(local.addresses.*.type, "ExternalIP") : false
   external_ip     = local.has_external_ip ? local.addresses[index(local.addresses.*.type, "ExternalIP")].address : "localhost"
   port            = var.enable_kubernetes_resources ? kubernetes_service_v1.hello[0].spec[0].port[0].node_port : 8080
-  service_url     = "http://${local.external_ip}:${local.port != null ? local.port : 8080}/"
+  lb_url = var.enable_kubernetes_resources && var.private_node_groups ? "https://${kubernetes_service_v1.hello[0].status[0].load_balancer[0].ingress[0].hostname}" : "localhost:8080"
+  service_url     = var.private_node_groups ? local.lb_url : "http://${local.external_ip}:${local.port != null ? local.port : 8080}/"
 }
 
 data "http" "hello" {
@@ -152,9 +182,11 @@ data "http" "hello" {
     kubernetes_service_v1.hello,
   ]
 
-  # Wait 5 minutes for the service to be ready.
+  # Wait for the service to be ready:
+  # - Max 5 minutes when using public node groups and NodePort service.
+  # - Max 15 minutes when using private node groups and LoadBalancer service.
   retry {
-    attempts     = 30
+    attempts     = var.private_node_groups ? 90 : 30
     min_delay_ms = 10e3
   }
 }