Skip to content

Deny firewall for external services #958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k8s-ci-robot merged 1 commit into from
Jan 30, 2026
Merged

Deny firewall for external services #958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 46 additions & 37 deletions cmd/cloud-controller-manager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider-gcp/providers/gce"
_ "k8s.io/cloud-provider-gcp/providers/gce"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/names"
Expand All @@ -39,6 +37,9 @@ import (
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
"k8s.io/klog/v2"
kcmnames "k8s.io/kubernetes/cmd/kube-controller-manager/names"

"k8s.io/cloud-provider-gcp/providers/gce"
_ "k8s.io/cloud-provider-gcp/providers/gce"
)

const (
Expand All @@ -47,30 +48,36 @@ const (
gkeServiceAlias = "gke-service"
)

// enableMultiProject is bound to a command-line flag. When true, it enables the
// projectFromNodeProviderID option of the GCE cloud provider, instructing it to
// use the project specified in the Node's providerID for GCE API calls.
//
// This flag should only be enabled when the Node's providerID can be fully
// trusted.
//
// Flag binding occurs in main()
var enableMultiProject bool

// enableDiscretePortForwarding is bound to a command-line flag. It enables
// the same option of the GCE cloud provider to forward individual ports
// instead of port ranges in Forwarding Rules for external load balancers.
var enableDiscretePortForwarding bool

// enableRBSDefaultForGCEL4NetLB is bound to a command-line flag. It enables
// the option to default L4 NetLB to RBS, only controlling NetLB services with
// LoadBalancerClass
var enableRBSDefaultForL4NetLB bool

// enableL4LBAnnotations is bound to a command-line flag. It enables
// the controller to write annotations related to the provisioned resources
// for L4 Load Balancers services
var enableL4LBAnnotations bool
var (
// enableMultiProject is bound to a command-line flag. When true, it enables the
// projectFromNodeProviderID option of the GCE cloud provider, instructing it to
// use the project specified in the Node's providerID for GCE API calls.
//
// This flag should only be enabled when the Node's providerID can be fully
// trusted.
//
// Flag binding occurs in main()
enableMultiProject bool

// enableRBSDefaultForGCEL4NetLB is bound to a command-line flag. It enables
// the option to default L4 NetLB to RBS, only controlling NetLB services with
// LoadBalancerClass
enableRBSDefaultForL4NetLB bool

// enableL4LBAnnotations is bound to a command-line flag. It enables
// the controller to write annotations related to the provisioned resources
// for L4 Load Balancers services
enableL4LBAnnotations bool

// enableL4DenyFirewall creates and manages an additional deny firewall rule
// at priority 1000 and moves the node and healthcheck firewall rule to priority 999.
enableL4DenyFirewall bool

// enableL4DenyFirewallRollbackCleanup enable cleanup codepath of the deny firewalls for rollback.
// The reason for it not being enabled by default is the additional GCE API calls that are made
// for checking if the deny firewalls exist/deletion which will eat up the quota unnecessarily.
enableL4DenyFirewallRollbackCleanup bool
)

func main() {
rand.Seed(time.Now().UnixNano())
Expand All @@ -88,9 +95,10 @@ func main() {

cloudProviderFS := fss.FlagSet("GCE Cloud Provider")
cloudProviderFS.BoolVar(&enableMultiProject, "enable-multi-project", false, "Enables project selection from Node providerID for GCE API calls. CAUTION: Only enable if Node providerID is configured by a trusted source.")
cloudProviderFS.BoolVar(&enableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enables forwarding of individual ports instead of port ranges for GCE external load balancers.")
cloudProviderFS.BoolVar(&enableRBSDefaultForL4NetLB, "enable-rbs-default-l4-netlb", false, "Enables RBS defaulting for GCE L4 NetLB")
cloudProviderFS.BoolVar(&enableL4LBAnnotations, "enable-l4-lb-annotations", false, "Enables Annotations for GCE L4 LB Services")
cloudProviderFS.BoolVar(&enableL4DenyFirewall, "enable-l4-deny-firewall", false, "Enable creation and updates of Deny VPC Firewall Rules for L4 external load balancers. Requires --enable-pinhole and --enable-l4-deny-firewall-rollback-cleanup to be true.")
cloudProviderFS.BoolVar(&enableL4DenyFirewallRollbackCleanup, "enable-l4-deny-firewall-rollback-cleanup", false, "Enable cleanup codepath of the deny firewalls for rollback. The reason for it not being enabled by default is the additional GCE API calls that are made for checking if the deny firewalls exist/deletion which will eat up the quota unnecessarily.")

// add new controllers and initializers
nodeIpamController := nodeIPAMController{}
Expand Down Expand Up @@ -158,16 +166,6 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
gceCloud.SetProjectFromNodeProviderID(true)
}

if enableDiscretePortForwarding {
gceCloud, ok := (cloud).(*gce.Cloud)
if !ok {
// Fail-fast: If enableDiscretePortForwarding is set, the cloud
// provider MUST be GCE.
klog.Fatalf("enable-discrete-port-forwarding requires GCE cloud provider, but got %T", cloud)
}
gceCloud.SetEnableDiscretePortForwarding(true)
}

if enableRBSDefaultForL4NetLB {
gceCloud, ok := (cloud).(*gce.Cloud)
if !ok {
Expand All @@ -188,5 +186,16 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
gceCloud.SetEnableL4LBAnnotations(true)
}

if enableL4DenyFirewall || enableL4DenyFirewallRollbackCleanup {
gceCloud, ok := (cloud).(*gce.Cloud)
if !ok {
klog.Fatalf("enable-l4-deny-firewall and enable-l4-deny-firewall-rollback-cleanup require GCE cloud provider, but got %T", cloud)
}
if enableL4DenyFirewall && !enableL4DenyFirewallRollbackCleanup {
klog.Fatal("enable-l4-deny-firewall requires enable-l4-deny-firewall-rollback-cleanup to be true")
}
gceCloud.SetEnableL4DenyFirewallRule(enableL4DenyFirewall, enableL4DenyFirewallRollbackCleanup)
}

return cloud
}
4 changes: 4 additions & 0 deletions providers/gce/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,11 @@ go_test(
"gce_annotations_test.go",
"gce_disks_test.go",
"gce_instances_test.go",
"gce_loadbalancer_external_deny_test.go",
"gce_loadbalancer_external_test.go",
"gce_loadbalancer_internal_test.go",
"gce_loadbalancer_metrics_test.go",
"gce_loadbalancer_naming_test.go",
"gce_loadbalancer_test.go",
"gce_loadbalancer_utils_test.go",
"gce_test.go",
Expand All @@ -116,6 +118,7 @@ go_test(
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta",
"//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock",
"//vendor/github.com/google/go-cmp/cmp",
"//vendor/github.com/google/go-cmp/cmp/cmpopts",
"//vendor/github.com/stretchr/testify/assert",
"//vendor/github.com/stretchr/testify/require",
"//vendor/golang.org/x/oauth2/google",
Expand All @@ -132,6 +135,7 @@ go_test(
"//vendor/k8s.io/client-go/tools/record",
"//vendor/k8s.io/cloud-provider",
"//vendor/k8s.io/cloud-provider/service/helpers",
"//vendor/k8s.io/component-base/metrics/testutil",
"//vendor/k8s.io/utils/net",
],
)
Expand Down
39 changes: 22 additions & 17 deletions providers/gce/gce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,15 @@ const (
gceComputeAPIEndpointBeta = "https://www.googleapis.com/compute/beta/"
)

var _ cloudprovider.Interface = (*Cloud)(nil)
var _ cloudprovider.Instances = (*Cloud)(nil)
var _ cloudprovider.LoadBalancer = (*Cloud)(nil)
var _ cloudprovider.Routes = (*Cloud)(nil)
var _ cloudprovider.Zones = (*Cloud)(nil)
var _ cloudprovider.PVLabeler = (*Cloud)(nil)
var _ cloudprovider.Clusters = (*Cloud)(nil)
var (
_ cloudprovider.Interface = (*Cloud)(nil)
_ cloudprovider.Instances = (*Cloud)(nil)
_ cloudprovider.LoadBalancer = (*Cloud)(nil)
_ cloudprovider.Routes = (*Cloud)(nil)
_ cloudprovider.Zones = (*Cloud)(nil)
_ cloudprovider.PVLabeler = (*Cloud)(nil)
_ cloudprovider.Clusters = (*Cloud)(nil)
)

type StackType string

Expand Down Expand Up @@ -203,15 +205,18 @@ type Cloud struct {
// Enable this ony when the Node's .spec.providerID can be fully trusted.
projectFromNodeProviderID bool

// enableDiscretePortForwarding enables forwarding of individual ports
// instead of port ranges in Forwarding Rules for external load balancers.
enableDiscretePortForwarding bool

// enableRBSDefaultForL4NetLB disable Service controller from picking up services by default
enableRBSDefaultForL4NetLB bool

// enableL4LBAnnotations enable annotations related to provisioned resources in GCE
enableL4LBAnnotations bool

// enableL4DenyFirewallRule creates an additional deny firewall rule at priority 1000
// and moves the allow rule to priority 999 to improve security posture.
enableL4DenyFirewallRule bool

// enableL4DenyFirewallRollbackCleanup
enableL4DenyFirewallRollbackCleanup bool
}

// ConfigGlobal is the in memory representation of the gce.conf config data
Expand Down Expand Up @@ -864,11 +869,6 @@ func (g *Cloud) SetProjectFromNodeProviderID(enabled bool) {
g.projectFromNodeProviderID = enabled
}

// SetEnableDiscretePortForwarding configures enableDiscretePortForwarding option.
func (g *Cloud) SetEnableDiscretePortForwarding(enabled bool) {
g.enableDiscretePortForwarding = enabled
}

func (g *Cloud) SetEnableRBSDefaultForL4NetLB(enabled bool) {
g.enableRBSDefaultForL4NetLB = enabled
}
Expand All @@ -877,6 +877,11 @@ func (g *Cloud) SetEnableL4LBAnnotations(enabled bool) {
g.enableL4LBAnnotations = enabled
}

func (g *Cloud) SetEnableL4DenyFirewallRule(firewallEnabled, rollbackEnabled bool) {
g.enableL4DenyFirewallRule = firewallEnabled
g.enableL4DenyFirewallRollbackCleanup = rollbackEnabled
}

// getProjectsBasePath returns the compute API endpoint with the `projects/` element.
// The suffix must be added when generating compute resource urls.
func getProjectsBasePath(basePath string) string {
Expand Down Expand Up @@ -970,7 +975,7 @@ func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string
// listCall = listCall.Filter("region eq " + region)

var zones []string
var accumulator = func(response *compute.ZoneList) error {
accumulator := func(response *compute.ZoneList) error {
for _, zone := range response.Items {
regionName := lastComponent(zone.Region)
if regionName == region {
Expand Down
Loading