Skip to content

[WIP] Add GCP IP Alias Mode with Cilium CNI #18121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Camila-B wants to merge 5 commits into
base: master
Choose a base branch
from
Open

[WIP] Add GCP IP Alias Mode with Cilium CNI #18121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c696ec6
Add Cilium bool field to GCPNetworkingSpec
Camila-B Mar 26, 2026
225e054
Use CiliumNetworkingSpec in GCPNetworkingSpec
Camila-B Mar 23, 2026
99bc470
Add integration test for gcp-with-cilium
Camila-B Mar 24, 2026
e047e4a
Refactor code
Camila-B Mar 24, 2026
d40a898
Make TF SecondaryIpRanges deterministic
Camila-B Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/kops/create_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
cmd.Flags().StringVar(&options.EtcdStorageType, "etcd-storage-type", options.EtcdStorageType, "The default storage type for etcd members")
cmd.RegisterFlagCompletionFunc("etcd-storage-type", completeStorageType)

cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, kube-router, amazonvpc, cilium, cilium-etcd, kindnet, cni.")
cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, kube-router, amazonvpc, cilium, gcp-with-cilium, cilium-etcd, kindnet, cni.")
Copy link
Copy Markdown
Member

@hakman hakman Mar 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Camila-B I think gcp-with-cilium is not exactly describing the option, not is similar to the equivalent AWS option cilium-eni. Would cilium-ipam or cilium-ipam-gcp be a good match?

Copy link
Copy Markdown
Contributor

@rramkumar1 rramkumar1 Mar 31, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@hakman What did you mean by this?

"not is similar to the equivalent AWS option cilium-eni"

cmd.RegisterFlagCompletionFunc("networking", completeNetworking(options))

cmd.Flags().StringVar(&options.DNSZone, "dns-zone", options.DNSZone, "DNS hosted zone (defaults to longest matching zone)")
Expand Down Expand Up @@ -1016,7 +1016,7 @@ func completeNetworking(options *CreateClusterOptions) func(cmd *cobra.Command,
}

if options.CloudProvider == "gce" || options.CloudProvider == "" {
completions = append(completions, "gcp")
completions = append(completions, "gcp", "gcp-with-cilium")
}
}

Expand Down
5 changes: 5 additions & 0 deletions cmd/kops/create_cluster_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ func TestCreateClusterCilium(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/cilium-eni", "v1alpha2")
}

// TestCreateClusterCiliumGCE runs kops with the gcp-with-cilium networking flag
func TestCreateClusterCiliumGCE(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/cilium-gce", "v1alpha2")
}

// TestCreateClusterOverride tests the override flag
func TestCreateClusterOverride(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/overrides", "v1alpha2")
Expand Down
9 changes: 9 additions & 0 deletions cmd/kops/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,15 @@ func TestMinimalIPv6Cilium(t *testing.T) {
runTestTerraformAWS(t)
}

func TestCiliumGCE(t *testing.T) {
newIntegrationTest("cilium-gce.example.com", "cilium-gce").
withAddons(
ciliumAddon,
dnsControllerAddon,
gcpCCMAddon).
runTestTerraformGCE(t)
}

// TestMinimalIPv6NoSubnetPrefix runs the test with "/64#N" subnet notation
func TestMinimalIPv6NoSubnetPrefix(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-no-subnet-prefix").
Expand Down
2 changes: 1 addition & 1 deletion docs/cli/kops_create_cluster.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

520 changes: 520 additions & 0 deletions k8s/crds/kops.k8s.io_clusters.yaml
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion pkg/apis/kops/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,8 @@ func (c *Cluster) UsesNoneDNS() bool {
func (c *Cluster) InstallCNIAssets() bool {
return c.Spec.Networking.AmazonVPC == nil &&
c.Spec.Networking.Calico == nil &&
c.Spec.Networking.Cilium == nil
c.Spec.Networking.Cilium == nil &&
!c.Spec.Networking.NetworkingIsGCPCilium()
}

func (c *Cluster) HasImageVolumesSupport() bool {
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kops/model/features.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {

// UseCiliumEtcd is true if we are using the Cilium etcd cluster.
func UseCiliumEtcd(cluster *kops.Cluster) bool {
if cluster.Spec.Networking.Cilium == nil {
if cluster.Spec.Networking.Cilium == nil && !cluster.Spec.Networking.NetworkingIsGCPCilium() {
return false
}

Expand Down
12 changes: 10 additions & 2 deletions pkg/apis/kops/networking.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (n *NetworkingSpec) UsesKubenet() bool {
}
if n.Kubenet != nil {
return true
} else if n.GCP != nil {
} else if n.GCP != nil && n.GCP.Cilium == nil {
// GCP IP Alias networking is based on kubenet
return true
} else if n.External != nil {
Expand All @@ -117,6 +117,11 @@ func (n *NetworkingSpec) UsesKubenet() bool {
return false
}

// NetworkingIsGCPCilium returns true if our networking is derived from GCP with Cilium
func (n *NetworkingSpec) NetworkingIsGCPCilium() bool {
return n.GCP != nil && n.GCP.Cilium != nil
}

// ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes.
// Support been removed since Kubernetes 1.4.
type ClassicNetworkingSpec struct{}
Expand Down Expand Up @@ -584,7 +589,10 @@ type LyftVPCNetworkingSpec struct {
}

// GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases.
type GCPNetworkingSpec struct{}
type GCPNetworkingSpec struct {
// Cilium enables Cilium on GCP.
Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"`
}

// KindnetNetworkingSpec configures Kindnet settings.
type KindnetNetworkingSpec struct {
Expand Down
5 changes: 4 additions & 1 deletion pkg/apis/kops/v1alpha2/networking.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,10 @@ type LyftVPCNetworkingSpec struct {
}

// GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases.
type GCPNetworkingSpec struct{}
type GCPNetworkingSpec struct {
// Cilium enables Cilium on GCP.
Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"`
}

// KindnetNetworkingSpec configures Kindnet settings.
type KindnetNetworkingSpec struct {
Expand Down
18 changes: 18 additions & 0 deletions pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion pkg/apis/kops/v1alpha3/networking.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,10 @@ type HubbleSpec struct {
}

// GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases.
type GCPNetworkingSpec struct{}
type GCPNetworkingSpec struct {
// Cilium enables Cilium on GCP.
Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"`
}

// KindnetNetworkingSpec configures Kindnet settings.
type KindnetNetworkingSpec struct {
Expand Down
18 changes: 18 additions & 0 deletions pkg/apis/kops/v1alpha3/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion pkg/apis/kops/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions pkg/apis/nodeup/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,19 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
}
}

if cluster.Spec.Networking.GCP != nil {
config.Networking.GCP = &kops.GCPNetworkingSpec{}
if cluster.Spec.Networking.GCP.Cilium != nil {
config.Networking.GCP.Cilium = cluster.Spec.Networking.GCP.Cilium
if cluster.Spec.Networking.GCP.Cilium.IPAM == kops.CiliumIpamEni {
config.Networking.GCP.Cilium.IPAM = kops.CiliumIpamEni
}
if model.UseCiliumEtcd(cluster) {
config.UseCiliumEtcd = true
}
}
}

if cluster.Spec.Networking.CNI != nil && cluster.Spec.Networking.CNI.UsesSecondaryIP {
config.Networking.CNI = &kops.CNINetworkingSpec{UsesSecondaryIP: true}
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/model/components/cilium.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ var _ loader.ClusterOptionsBuilder = &CiliumOptionsBuilder{}
func (b *CiliumOptionsBuilder) BuildOptions(o *kops.Cluster) error {
clusterSpec := &o.Spec
c := clusterSpec.Networking.Cilium
if c == nil && clusterSpec.Networking.GCP != nil {
c = clusterSpec.Networking.GCP.Cilium
}
if c == nil {
return nil
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/model/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,3 +418,7 @@ func (b *KopsModelContext) NetworkingIsCalico() bool {
func (b *KopsModelContext) NetworkingIsCilium() bool {
return b.Cluster.Spec.Networking.Cilium != nil
}

func (b *KopsModelContext) NetworkingIsGCPWithCilium() bool {
return b.Cluster.Spec.Networking.GCP != nil && b.Cluster.Spec.Networking.GCP.Cilium != nil
}
2 changes: 1 addition & 1 deletion pkg/model/gcemodel/firewall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
if b.NetworkingIsCalico() {
t.Allowed = append(t.Allowed, "ipip")
}
if b.NetworkingIsCilium() {
if b.NetworkingIsCilium() || b.NetworkingIsGCPWithCilium() {
t.Allowed = append(t.Allowed, fmt.Sprintf("udp:%d", wellknownports.VxlanUDP))
if model.UseCiliumEtcd(b.Cluster) {
t.Allowed = append(t.Allowed, fmt.Sprintf("tcp:%d", wellknownports.EtcdCiliumClientPort))
Expand Down
Loading