diff --git a/Dockerfile b/Dockerfile
index 303245bfd05..1bfcffe4f0a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -26,7 +26,7 @@ COPY pkg/ pkg/
 COPY .git/ .git/
 
 # Build
-RUN VERSION=${BUILD_VERSION} make manager
+RUN VERSION=${BUILD_VERSION} make manager-dockerfile
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/Dockerfile.adapter b/Dockerfile.adapter
index 749001fb73e..26ccfb83b71 100644
--- a/Dockerfile.adapter
+++ b/Dockerfile.adapter
@@ -28,7 +28,7 @@ COPY .git/ .git/
 RUN mkdir -p /apiserver.local.config/certificates && chmod -R 777 /apiserver.local.config
 
 # Build
-RUN VERSION=${BUILD_VERSION} make adapter
+RUN VERSION=${BUILD_VERSION} make adapter-dockerfile
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/Makefile b/Makefile
index dcd2bc63c59..14bffd57035 100644
--- a/Makefile
+++ b/Makefile
@@ -136,14 +136,28 @@ docker-build:
 
 # Build KEDA Operator binary
 .PHONY: manager
-manager: generate gofmt govet
+manager: manager-dockerfile gofmt govet
+
+# Build the manager inside the Dockerfile. This elides
+# the gofmt and govet commands. Since code quality checks
+# are already in CI, we don't need to run them every
+# time we build the image
+.PHONY: manager-dockerfile
+manager-dockerfile: generate
 	${GO_BUILD_VARS} go build \
 	-ldflags "-X=github.com/kedacore/keda/version.GitCommit=$(GIT_COMMIT) -X=github.com/kedacore/keda/version.Version=$(VERSION)" \
 	-o bin/keda main.go
 
 # Build KEDA Metrics Server Adapter binary
 .PHONY: adapter
-adapter: generate gofmt govet
+adapter: adapter-dockerfile gofmt govet
+
+# Build the adapter inside the Dockerfile. This elides
+# the gofmt and govet commands. Since code quality checks
+# are already in CI, we don't need to run them every
+# time we build the image
+.PHONY: adapter-dockerfile
+adapter-dockerfile: generate
 	${GO_BUILD_VARS} go build \
 	-ldflags "-X=github.com/kedacore/keda/version.GitCommit=$(GIT_COMMIT) -X=github.com/kedacore/keda/version.Version=$(VERSION)" \
 	-o bin/keda-adapter adapter/main.go