refactor: Split the agent and api controllers

.gitignore

@@ -30,3 +30,6 @@ e2e-fixtures/*/config*.yaml
 /.direnv
 .env
 .envrc-user
+
+# Ignore local scripts and utilities
+local/*

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.23 as builder
+FROM --platform=$BUILDPLATFORM golang:1.23 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -21,9 +21,9 @@ RUN --mount=type=cache,target=/go \
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM --platform=${TARGETPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot
+FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/bin/manager .
+COPY --from=builder /workspace/bin/api-manager /workspace/bin/agent-manager ./
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/api-manager"]

Makefile

@@ -94,17 +94,20 @@ validate: build test lint manifests helm-update-snapshots ## Validate the codeba
 ##@ Build
 
 .PHONY: build
-build: preflight generate fmt vet _build ## Build manager binary.
+build: preflight generate fmt vet _build ## Build binaries.
 
 .PHONY: _build
 _build:
-	go build -o bin/manager -trimpath -ldflags "-s -w \
+	go build -o bin/api-manager -trimpath -ldflags "-s -w \
 		-X $(REPO_URL)/internal/version.gitCommit=$(GIT_COMMIT) \
-		-X $(REPO_URL)/internal/version.version=$(VERSION)" cmd/main.go
+		-X $(REPO_URL)/internal/version.version=$(VERSION)" cmd/api/main.go
+	go build -o bin/agent-manager -trimpath -ldflags "-s -w \
+		-X $(REPO_URL)/internal/version.gitCommit=$(GIT_COMMIT) \
+		-X $(REPO_URL)/internal/version.version=$(VERSION)" cmd/agent/main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go
+	go run ./cmd/api/main.go
 
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
@@ -248,19 +251,17 @@ endef
 ##@ Helm
 
 .PHONY: _helm_setup
-_helm_setup:
-	./scripts/helm-setup.sh
-	helm repo add bitnami https://charts.bitnami.com/bitnami
-	helm dependency build $(HELM_CHART_DIR)
+_helm_setup: ## Setup helm dependencies
+	$(MAKE) -C $(HELM_CHART_DIR) setup
 
 .PHONY: helm-lint
 helm-lint: _helm_setup ## Lint the helm chart
-	helm lint $(HELM_CHART_DIR)
+	$(MAKE) -C $(HELM_CHART_DIR) lint
 
 .PHONY: helm-test
 helm-test: _helm_setup ## Run helm unittest plugin
-	helm unittest $(HELM_CHART_DIR)
+	$(MAKE) -C $(HELM_CHART_DIR) test
 
 .PHONY: helm-update-snapshots
 helm-update-snapshots: _helm_setup ## Update helm unittest snapshots
-	helm unittest -u $(HELM_CHART_DIR)
+	$(MAKE) -C $(HELM_CHART_DIR) update-snapshots

cmd/agent/main.go

@@ -0,0 +1,209 @@
+/*
+Copyright 2022.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+
+	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
+	// to ensure that exec-entrypoint and run can make use of them.
+
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+	bindingsv1alpha1 "github.com/ngrok/ngrok-operator/api/bindings/v1alpha1"
+	ingressv1alpha1 "github.com/ngrok/ngrok-operator/api/ingress/v1alpha1"
+	ngrokv1alpha1 "github.com/ngrok/ngrok-operator/api/ngrok/v1alpha1"
+	ngrokv1beta1 "github.com/ngrok/ngrok-operator/api/ngrok/v1beta1"
+	agentcontroller "github.com/ngrok/ngrok-operator/internal/controller/agent"
+	"github.com/ngrok/ngrok-operator/internal/healthcheck"
+	"github.com/ngrok/ngrok-operator/internal/version"
+	"github.com/ngrok/ngrok-operator/pkg/tunneldriver"
+	//+kubebuilder:scaffold:imports
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(gatewayv1.AddToScheme(scheme))
+	utilruntime.Must(ingressv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(ngrokv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(bindingsv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(ngrokv1beta1.AddToScheme(scheme))
+	//+kubebuilder:scaffold:scheme
+}
+
+func main() {
+	if err := cmd().Execute(); err != nil {
+		setupLog.Error(err, "error running agent-manager")
+		os.Exit(1)
+	}
+}
+
+type managerOpts struct {
+	// flags
+	metricsAddr string
+	probeAddr   string
+	serverAddr  string
+	description string
+	managerName string
+	zapOpts     *zap.Options
+
+	// feature flags
+	enableFeatureIngress  bool
+	enableFeatureGateway  bool
+	enableFeatureBindings bool
+
+	// agent(tunnel driver) flags
+	region  string
+	rootCAs string
+}
+
+func cmd() *cobra.Command {
+	var opts managerOpts
+	c := &cobra.Command{
+		Use: "agent-manager",
+		RunE: func(c *cobra.Command, args []string) error {
+			return runController(c.Context(), opts)
+		},
+	}
+
+	c.Flags().StringVar(&opts.metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to")
+	c.Flags().StringVar(&opts.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	c.Flags().StringVar(&opts.description, "description", "Created by the ngrok-operator", "Description for this installation")
+	// TODO(operator-rename): Same as above, but for the manager name.
+	c.Flags().StringVar(&opts.managerName, "manager-name", "agent-manager", "Manager name to identify unique ngrok operator agent instances")
+
+	// agent(tunnel driver) flags
+	c.Flags().StringVar(&opts.region, "region", "", "The region to use for ngrok tunnels")
+	c.Flags().StringVar(&opts.serverAddr, "server-addr", "", "The address of the ngrok server to use for tunnels")
+	c.Flags().StringVar(&opts.rootCAs, "root-cas", "trusted", "trusted (default) or host: use the trusted ngrok agent CA or the host CA")
+
+	// feature flags
+	c.Flags().BoolVar(&opts.enableFeatureIngress, "enable-feature-ingress", true, "Enables the Ingress controller")
+	c.Flags().BoolVar(&opts.enableFeatureGateway, "enable-feature-gateway", false, "Enables the Gateway controller")
+	c.Flags().BoolVar(&opts.enableFeatureBindings, "enable-feature-bindings", false, "Enables the Endpoint Bindings controller")
+
+	opts.zapOpts = &zap.Options{}
+	goFlagSet := flag.NewFlagSet("manager", flag.ContinueOnError)
+	opts.zapOpts.BindFlags(goFlagSet)
+	c.Flags().AddGoFlagSet(goFlagSet)
+
+	return c
+}
+
+func runController(ctx context.Context, opts managerOpts) error {
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(opts.zapOpts)))
+
+	buildInfo := version.Get()
+	setupLog.Info("starting agent-manager", "version", buildInfo.Version, "commit", buildInfo.GitCommit)
+
+	options := ctrl.Options{
+		Scheme: scheme,
+		Metrics: server.Options{
+			BindAddress: opts.metricsAddr,
+		},
+		WebhookServer:          webhook.NewServer(webhook.Options{Port: 9443}),
+		HealthProbeBindAddress: opts.probeAddr,
+		LeaderElection:         false,
+	}
+
+	// create default config and clientset for use outside the mgr.Start() blocking loop
+	k8sConfig := ctrl.GetConfigOrDie()
+	mgr, err := ctrl.NewManager(k8sConfig, options)
+	if err != nil {
+		return fmt.Errorf("unable to start agent-manager: %w", err)
+	}
+
+	// shared features between Ingress and Gateway (tunnels)
+	if opts.enableFeatureIngress || opts.enableFeatureGateway {
+		var comments tunneldriver.TunnelDriverComments
+		if opts.enableFeatureGateway {
+			comments = tunneldriver.TunnelDriverComments{
+				Gateway: "gateway-api",
+			}
+		}
+
+		rootCAs := "trusted"
+		if opts.rootCAs != "" {
+			rootCAs = opts.rootCAs
+		}
+
+		td, err := tunneldriver.New(ctx, ctrl.Log.WithName("drivers").WithName("tunnel"),
+			tunneldriver.TunnelDriverOpts{
+				ServerAddr: opts.serverAddr,
+				Region:     opts.region,
+				RootCAs:    rootCAs,
+				Comments:   &comments,
+			},
+		)
+
+		if err != nil {
+			return fmt.Errorf("unable to create tunnel driver: %w", err)
+		}
+
+		// register healthcheck for tunnel driver
+		healthcheck.RegisterHealthChecker(td)
+
+		if err = (&agentcontroller.TunnelReconciler{
+			Client:       mgr.GetClient(),
+			Log:          ctrl.Log.WithName("controllers").WithName("tunnel"),
+			Scheme:       mgr.GetScheme(),
+			Recorder:     mgr.GetEventRecorderFor("tunnel-controller"),
+			TunnelDriver: td,
+		}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "Tunnel")
+			os.Exit(1)
+		}
+	}
+
+	// register healthchecks
+	if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error {
+		return healthcheck.Ready(req.Context(), req)
+	}); err != nil {
+		return fmt.Errorf("error setting up readyz check: %w", err)
+	}
+	if err := mgr.AddHealthzCheck("healthz", func(req *http.Request) error {
+		return healthcheck.Alive(req.Context(), req)
+	}); err != nil {
+		return fmt.Errorf("error setting up health check: %w", err)
+	}
+
+	setupLog.Info("starting agent-manager")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		return fmt.Errorf("error starting agent-manager: %w", err)
+	}
+
+	return nil
+}