Merge pull request #26 from sp-yduck/secretref

Secretref

Author: sp-yduck

.gitignore

@@ -25,8 +25,7 @@ Dockerfile.cross
 *.swo
 *~
 
-
 *_test.go
 
-# 
-config/samples/secret.yaml
+# release
+out

Makefile

@@ -11,6 +11,9 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+GOARCH  := $(shell go env GOARCH)
+GOOS    := $(shell go env GOOS)
+
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
@@ -58,6 +61,16 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
 
+CLUSTER_NAME := cappx-test
+
+.PHONY: create-workload-cluster
+create-workload-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL)
+	export CLUSTER_NAME=$(CLUSTER_NAME) && $(KUSTOMIZE) build templates | $(ENVSUBST) | $(KUBECTL) apply -f -
+
+.PHONY: delete-workload-cluster
+delete-workload-cluster: $(KUBECTL)
+	$(KUBECTL) delete cluster $(CLUSTER_NAME)
+
 ##@ Build
 
 .PHONY: build
@@ -72,7 +85,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
-docker-build: test ## Build docker image with the manager.
+docker-build: # test ## Build docker image with the manager.
 	docker build -t ${IMG} .
 
 .PHONY: docker-push
@@ -119,6 +132,30 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
+##@ Release
+
+## Location to output for release
+RELEASE_DIR := out
+$(RELEASE_DIR):
+	mkdir -p $(RELEASE_DIR)
+
+# RELEASE_TAG := $(shell git describe --abbrev=0 2>/dev/null)
+
+# .PHONY: release
+
+.PHONY: release-manifests
+release-manifests: $(KUSTOMIZE) $(RELEASE_DIR) ## Builds the manifests to publish with a release
+	$(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml
+
+.PHONY: release-metadata
+release-metadata: $(RELEASE_DIR)
+	cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
+
+.PHONY: release-templates
+release-templates: $(RELEASE_DIR)
+	cp templates/cluster-template* $(RELEASE_DIR)/
+
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to
@@ -130,10 +167,14 @@ $(LOCALBIN):
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
+ENVSUBST ?= $(LOCALBIN)/envsubst
+KUBECTL ?= $(LOCALBIN)/kubectl
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.0.0
 CONTROLLER_TOOLS_VERSION ?= v0.11.3
+ENVSUBST_VER ?= v1.4.2
+KUBECTL_VER := v1.25.10
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
@@ -155,3 +196,14 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+.PHONY: envsubst
+envsubst: $(ENVSUBST)
+$(ENVSUBST): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install github.com/a8m/envsubst/cmd/envsubst@$(ENVSUBST_VER)
+
+.PHONY: kubectl
+kubectl: $(KUBECTL)
+$(KUBECTL): $(LOCALBIN)
+	curl --retry 3 -fsL https://dl.k8s.io/release/$(KUBECTL_VER)/bin/$(GOOS)/$(GOARCH)/kubectl -o $(LOCALBIN)/kubectl
+	chmod +x $(KUBECTL)

README.md

@@ -25,49 +25,36 @@ make deploy
 
 2. Create your first workload cluster
 ```sh
-# cluster & infra cluster
-kubectl apply -f config/samples/cluster.yaml
-kubectl apply -f config/samples/infrastructure_v1beta1_proxmoxcluster.yaml
+# export env variables
+export CONTROLPLANE_HOST=X.X.X.X
+export PROXMOX_URL=X.X.X.X:8006
+export GATEWAY_ADDRESS=X.X.X.X
+export NAMESERVER_ADDRESS=X.X.X.X
+export PROXMOX_PASSWORD_BASE64=$(echo -n <password> | base64)
+export PROXMOX_USER_BASE64=$(echo -n <user@pam> | base64)
+export NODE_URL_BASE64=$(echo -n <node.ssh.url:22> | base64)
+export NODE_USER_BASE64=$(echo -n <node-ssh-user> | base64)
+export NODE_PASSWORD_BASE64=$(echo -n <node-ssh-password> | base64)
+
+make create-workload-cluster
+```
 
-# controlplane
-kubectl apply -f config/samples/controlplane.yaml
+3. Access your first workload cluster !!
 
-# machine & bootstrap & infra machine
-kubectl apply -f config/samples/machine.yaml
-kubectl apply -f config/samples/bootstrap.yaml
-kubectl apply -f config/samples/infrastructure_v1beta1_proxmoxcluster.yaml
+Usually it takes 2~5 mins to complete bootstrap the nodes.
+```sh
+# get workload cluster's kubeconfig
+clusterctl get kubeconfig cappx-test > kubeconfig.yaml
 
-# proxmox configs
-kubetl apply -f <your-proxmox-config-secret>.yaml
+# get node command for workload cluster
+kubectl --kubeconfig=kubeconfig.yaml get node
 ```
 
-You need to provide your proxmox information through secret. 
-```yaml
-# <your-proxmox-config-secret>.yaml
-apiVersion: v1
-data:
-  # for proxmox API
-  PROXMOX_PASSWORD: "<base 64>"
-  PROXMOX_USER: "<base 64>"
-  # for ssh into the node to bootstrapping VMs
-  ## * current CAPP is compatible with only single node proxmox cluster
-  NODE_URL: "<base 64>"
-  NODE_USER: "<base 64>"
-  NODE_PASSWORD: "<base 64>"
-kind: Secret
-metadata:
-  name: proxmoxcluster-sample
-type: Opaque
+4. Tear down your workload cluster
+```sh
+make delete-workload-cluster
 ```
 
-3. Check your Cluster & Machines !!
-
-Once CAPP reconciled your `ProxmoxCluster`/`ProxmoxMachine`, you can see `READY=true` for `ProxmoxCluster` and `STATUS=running` for `ProxmoxMachine`.
-
-![kubectl-get-proxmox-cluster](./logos/k-get-proxmoxcluster.PNG)
-
-![kubectl-get-proxmox-machine](./logos/k-get-proxmoxmachine.PNG)
-
 ## Fetures
 
 - No need to prepare vm templates. You can specify any vm image in `ProxmoxMachine.Spec.Image`.

api/v1beta1/type.go

@@ -17,23 +17,22 @@ type ServerRef struct {
 
 	// to do : client options like insecure tls verify
 
-	// CredentialsRef is a reference for secret which contains proxmox login secrets
+	// SecretRef is a reference for secret which contains proxmox login secrets
 	// and ssh configs for proxmox nodes
-	CredentialsRef *ObjectReference `json:"credentialsRef"`
+	SecretRef *ObjectReference `json:"secretRef"`
 }
 
 // NodeRef
 type NodeRef struct {
-	Name           string           `json:"name"`
-	CredentialsRef *ObjectReference `json:"credentialsRef"`
+	Name      string           `json:"name"`
+	SecretRef *ObjectReference `json:"secretRef"`
 }
 
 // ObjectReference is a reference to another Kubernetes object instance.
 type ObjectReference struct {
 	// Namespace of the referent.
 	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
-	// +kubebuilder:validation:Required
-	Namespace string `json:"namespace"`
+	Namespace string `json:"namespace,omitempty"`
 	// Name of the referent.
 	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
 	// +kubebuilder:validation:Required

api/v1beta1/zz_generated.deepcopy.go

@@ -33,13 +33,13 @@ type ProxmoxServices struct {
 }
 
 func newComputeService(ctx context.Context, serverRef infrav1.ServerRef, crClient client.Client) (*service.Service, error) {
-	credentialsRef := serverRef.CredentialsRef
-	if credentialsRef == nil {
-		return nil, errors.New("failed to get proxmox client form nil credentialsRef")
+	secretRef := serverRef.SecretRef
+	if secretRef == nil {
+		return nil, errors.New("failed to get proxmox client form nil secretRef")
 	}
 
 	var secret corev1.Secret
-	key := client.ObjectKey{Namespace: credentialsRef.Namespace, Name: credentialsRef.Name}
+	key := client.ObjectKey{Namespace: secretRef.Namespace, Name: secretRef.Name}
 	if err := crClient.Get(ctx, key, &secret); err != nil {
 		return nil, err
 	}
@@ -56,13 +56,13 @@ func newComputeService(ctx context.Context, serverRef infrav1.ServerRef, crClien
 	return service.NewServiceWithLogin(serverRef.Endpoint, string(proxmoxUser), string(proxmoxPassword))
 }
 
-func newRemoteClient(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*SSHClient, error) {
-	if credentialsRef == nil {
-		return nil, errors.New("failed to get proxmox client form nil credentialsRef")
+func newRemoteClient(ctx context.Context, secretRef *infrav1.ObjectReference, crClient client.Client) (*SSHClient, error) {
+	if secretRef == nil {
+		return nil, errors.New("failed to get proxmox client form nil secretRef")
 	}
 
 	var secret corev1.Secret
-	key := client.ObjectKey{Namespace: credentialsRef.Namespace, Name: credentialsRef.Name}
+	key := client.ObjectKey{Namespace: secretRef.Namespace, Name: secretRef.Name}
 	if err := crClient.Get(ctx, key, &secret); err != nil {
 		return nil, err
 	}

cloud/scope/clients.go

@@ -42,6 +42,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
 	if params.ProxmoxCluster == nil {
 		return nil, errors.New("failed to generate new scope from nil ProxmoxCluster")
 	}
+	populateNamespace(params.ProxmoxCluster)
 
 	if params.ProxmoxServices.Compute == nil {
 		computeSvc, err := newComputeService(ctx, params.ProxmoxCluster.Spec.ServerRef, params.Client)
@@ -53,7 +54,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
 
 	if params.ProxmoxServices.Remote == nil {
 		// current CAPP is compatible with only single node proxmox cluster
-		remote, err := newRemoteClient(ctx, params.ProxmoxCluster.Spec.NodeRefs[0].CredentialsRef, params.Client)
+		remote, err := newRemoteClient(ctx, params.ProxmoxCluster.Spec.NodeRefs[0].SecretRef, params.Client)
 		if err != nil {
 			return nil, errors.Errorf("failed to create remote client: %v", err)
 		}
@@ -74,6 +75,17 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
 	}, err
 }
 
+func populateNamespace(proxmoxCluster *infrav1.ProxmoxCluster) {
+	if proxmoxCluster.Spec.ServerRef.SecretRef.Namespace == "" {
+		proxmoxCluster.Spec.ServerRef.SecretRef.Namespace = proxmoxCluster.Namespace
+	}
+	for i, nodeRef := range proxmoxCluster.Spec.NodeRefs {
+		if nodeRef.SecretRef.Namespace == "" {
+			proxmoxCluster.Spec.NodeRefs[i].SecretRef.Namespace = proxmoxCluster.Namespace
+		}
+	}
+}
+
 type ClusterScope struct {
 	ProxmoxServices
 	client         client.Client

cloud/scope/cluster.go

@@ -59,11 +59,13 @@ func (s *Service) reconcileCloudInitUser(bootstrap string) error {
 		return err
 	}
 	base := baseUserData(vmName)
-	additional, err := cloudinit.MergeUsers(*config, base)
-	if err != nil {
-		return err
+	if config != nil {
+		base, err = cloudinit.MergeUsers(*config, *base)
+		if err != nil {
+			return err
+		}
 	}
-	cloudConfig, err := cloudinit.MergeUsers(*additional, *bootstrapConfig)
+	cloudConfig, err := cloudinit.MergeUsers(*base, *bootstrapConfig)
 	if err != nil {
 		return err
 	}
@@ -101,8 +103,8 @@ func ApplyCICustom(vmid int, vmName, storageName, ciType string, ssh scope.SSHCl
 }
 
 // to do : remove these cloud-config
-func baseUserData(vmName string) infrav1.User {
-	return infrav1.User{
+func baseUserData(vmName string) *infrav1.User {
+	return &infrav1.User{
 		GrowPart:       infrav1.GrowPart{Mode: "auto", Devices: []string{"/"}, IgnoreGrowrootDisabled: false},
 		HostName:       vmName,
 		ManageEtcHosts: true,

cloud/services/compute/instance/cloudinit.go

@@ -72,7 +72,9 @@ spec:
                 items:
                   description: NodeRef
                   properties:
-                    credentialsRef:
+                    name:
+                      type: string
+                    secretRef:
                       description: ObjectReference is a reference to another Kubernetes
                         object instance.
                       properties:
@@ -84,20 +86,21 @@ spec:
                           type: string
                       required:
                       - name
-                      - namespace
                       type: object
-                    name:
-                      type: string
                   required:
-                  - credentialsRef
                   - name
+                  - secretRef
                   type: object
                 type: array
               serverRef:
                 description: ServerRef is used for configuring Proxmox client
                 properties:
-                  credentialsRef:
-                    description: CredentialsRef is a reference for secret which contains
+                  endpoint:
+                    description: endpoint is the address of the Proxmox-VE REST API
+                      endpoint.
+                    type: string
+                  secretRef:
+                    description: SecretRef is a reference for secret which contains
                       proxmox login secrets and ssh configs for proxmox nodes
                     properties:
                       name:
@@ -108,15 +111,10 @@ spec:
                         type: string
                     required:
                     - name
-                    - namespace
                     type: object
-                  endpoint:
-                    description: endpoint is the address of the Proxmox-VE REST API
-                      endpoint.
-                    type: string
                 required:
-                - credentialsRef
                 - endpoint
+                - secretRef
                 type: object
               storage:
                 description: storage is for proxmox storage used by vm instances