Migrate swf config (#21)

Co-authored-by: Michael-Kruggel <michael.kruggel@defenseunicorns.com>
Co-authored-by: Jordan McClintock <jordan@defenseunicorns.com>

.gitignore

@@ -1,18 +1,28 @@
+**/*.tfstate
+**/*.tfstate.backup
+*.tar.gz
+*.tar.zst
+~/
+.DS_Store
+defense-unicorns-distro/preflight.sh
+.terraform
+tmp
+zarf-sbom
+
 .cache/
 .idea/
-.vscode/
 build/
 .DS_Store
 *.tar.zst
 test/tf/public-ec2-instance/.tool-versions
 zarf-sbom
+tmp/
+values-*.yaml
+overlay-values-*
 
 # Terraform
 test/tf/public-ec2-instance/.test-data
 test/tf/public-ec2-instance/.terraform
 terraform.tfstate
 terraform.tfstate.backup
 .terraform.lock.hcl
-
-# SOPS stuff that should never be committed to the repo
-secret-sops-gpg.yaml

Makefile

@@ -137,7 +137,7 @@ cluster/destroy: ## Destroy the k3d cluster
 # Build Section
 ########################################################################
 
-build/all: build build/zarf build/zarf-init.sha256 build/dubbd-pull-k3d.sha256 build/uds-capability-gitlab ##
+build/all: build build/zarf build/zarf-init.sha256 build/dubbd-pull-k3d.sha256 build/test-pkg-deps build/uds-capability-gitlab ##
 
 build: ## Create build directory
 	mkdir -p build
@@ -167,20 +167,32 @@ build/dubbd-pull-k3d.sha256: | build ## Download dubbd k3d oci package
 	echo "Creating shasum of the dubbd-k3d package"
 	shasum -a 256 build/zarf-package-dubbd-k3d-amd64-$(DUBBD_K3D_VERSION).tar.zst | awk '{print $$1}' > build/dubbd-pull-k3d.sha256
 
+build/test-pkg-deps: | build ## Build package dependencies for testing
+	build/zarf package create utils/pkg-deps/namespaces/ --skip-sbom --confirm --output-directory build
+	build/zarf package create utils/pkg-deps/gitlab/postgres/ --skip-sbom --confirm --output-directory build
+	build/zarf package create utils/pkg-deps/gitlab/redis/ --skip-sbom --confirm --output-directory build
+	build/zarf package create utils/pkg-deps/gitlab/minio/ --skip-sbom --confirm --output-directory build
+
 build/uds-capability-gitlab: | build ## Build the gitlab capability
 	build/zarf package create . --skip-sbom --confirm --output-directory build
 
 ########################################################################
 # Deploy Section
 ########################################################################
 
-deploy/all: deploy/init deploy/dubbd-k3d deploy/uds-capability-gitlab ##
+deploy/all: deploy/init deploy/dubbd-k3d deploy/test-pkg-deps deploy/uds-capability-gitlab ##
 
 deploy/init: ## Deploy the zarf init package
 	./build/zarf init --confirm --components=git-server
 
 deploy/dubbd-k3d: ## Deploy the k3d flavor of DUBBD
 	cd ./build && ./zarf package deploy zarf-package-dubbd-k3d-amd64-$(DUBBD_K3D_VERSION).tar.zst --confirm
 
+deploy/test-pkg-deps: ## Deploy the package dependencies needed for testing the gitlab capability
+	cd ./build && ./zarf package deploy zarf-package-gitlab-namespaces-* --confirm
+	cd ./build && ./zarf package deploy zarf-package-gitlab-postgres-* --confirm
+	cd ./build && ./zarf package deploy zarf-package-gitlab-redis-* --confirm
+	cd ./build && ./zarf package deploy zarf-package-gitlab-minio-* --confirm
+
 deploy/uds-capability-gitlab: ## Deploy the gilab capability
-	cd ./build && ./zarf package deploy zarf-package-gitlab-*.tar.zst --confirm --components=gitlab-values
+	cd ./build && ./zarf package deploy zarf-package-gitlab-amd*.tar.zst --confirm

README.md

@@ -1,11 +1,79 @@
 # uds-capability-gitlab
-Platform One Gitlab deployed via flux
 
-## Pre-req
-- Minimum compute requirements for single node deployment are at LEAST 64 GB RAM and 32 virtual CPU threads (aws `m6i.8xlarge` instance type should do)
-- k3d installed on machine
+Bigbang [Gitlab](https://repo1.dso.mil/big-bang/product/packages/gitlab) deployed via flux by zarf
 
-## Deploy
+## Deployment Prerequisites
+
+### GitLab Capability
+
+The Gitlab Capability expects the pieces listed below to exist in the cluster before being deployed.
+
+#### General
+
+- Create `gitlab` namespace
+- Label `gitlab` namespace with `istio-injection: enabled`
+
+#### Database
+
+- A Postgres database is running on port `5432` and accessible to the cluster
+- This database can be logged into via the username `gitlab`
+- This database instance has a psql database created matching what is defined in the deploy time variable `GITLAB_DB`. Default is `gitlabdb`
+- The `gitlab` user has read/write access to the above mentioned database
+- Create `gitlab-postgres` service in `gitlab` namespace that points to the psql database
+- Create `gitlab-postgres` secret in `gitlab` namespace with the key `password` that contains the password to the `gitlab` user for the psql database
+
+#### Redis / Redis Equivalent
+
+- An instance of Redis or Redis equivalent (elasticache, etc.) is running on port `6379` and accessible to the cluster
+- The redis instance accepts anonymous auth (password only)
+- Create `gitlab-redis` service in `gitlab` namespace that points to the redis instance
+- Create `gitlab-redis` secret in `gitlab` namespace with the key `password` that contains the password to the redis instance
+
+#### Object Storage
+
+Object Storage works a bit differently as there are many kinds of file stores gitlab can be configured to use.
+
+- Create the secret `gitlab-object-store` in the `gitlab` namespace with the following keys:
+  - An example for in-cluster Minio can be found in this repository at the path `utils/pkg-deps/gitlab/minio/secret.yaml`
+  - `connection`
+    - This key refers to the configuration for the main gitlab service. The documentation for what goes in this key is located [here](https://docs.gitlab.com/16.0/ee/administration/object_storage.html#configure-the-connection-settings)
+  - `registry`
+    - This key refers to the configuration for the gitlab registry. The documentation for what goes in this key is located [here](https://docs.docker.com/registry/configuration/#storage)
+  - `backups`
+    - This key refers to the configuration for the gitlab-toolbox backup tool. It relies on a program called `s3cmd`. The documentation for what goes in this key is located [here](https://s3tools.org/kb/item14.htm)
+- Below are the list of buckets that need to be created before starting GitLab:
+  - uds-gitlab-pages
+  - uds-gitlab-registry
+  - uds-gitlab-lfs
+  - uds-gitlab-artifacts
+  - uds-gitlab-uploads
+  - uds-gitlab-packages
+  - uds-gitlab-mr-diffs
+  - uds-gitlab-terraform-state
+  - uds-gitlab-ci-secure-files
+  - uds-gitlab-dependency-proxy
+  - uds-gitlab-backups
+  - uds-gitlab-tmp
+- These buckets can have a suffix applied via the `BUCKET_SUFFIX` zarf variable (e.x. `-some-deployment-name` plus `uds-gitlab-backups` would be `uds-gitlab-backups-some-deployment-name`)
+
+## Deploying
+
+### Deploy Everything
+
+#### Via Makefile and local package
+
+```bash
+# This will destroy and create a compatible k3d cluster then it will run make build/all and make deploy/all. Follow the breadcrumbs in the Makefile to see what and how its doing it.
+make cluster/full
+```
+
+### From GHCR OCI Via Zarf
+
+```bash
+zarf package deploy ghcr.io/defenseunicorns/uds-capability/gitlab:0.0.6-amd64
+```
+
+## Building
 
 ### Use zarf to login to the needed registries i.e. registry1.dso.mil and ghcr.io
 
@@ -21,34 +89,16 @@ export REGISTRY1_USERNAME="YOUR-USERNAME-HERE"
 export REGISTRY1_TOKEN="YOUR-TOKEN-HERE"
 echo $REGISTRY1_TOKEN | build/zarf tools registry login registry1.dso.mil --username $REGISTRY1_USERNAME --password-stdin
 
-# ghcr.io (To access oci packages needed)
+# ghcr.io (If you need to push to GHCR)
 export GH_USERNAME="YOUR-USERNAME-HERE"
 export GH_TOKEN="YOUR-TOKEN-HERE"
 echo $GH_TOKEN | build/zarf tools registry login ghcr.io --username $GH_USERNAME --password-stdin
 
 set -o history
 ```
 
-### Deploy Everything
+### Creating the Package
 
 ```bash
-# This will destroy and create a compatible k3d cluster then it will run make build/all and make deploy/all. Follow the breadcrumbs in the Makefile to see what and how its doing it.
-make cluster/full
+make build/uds-capability-gitlab
 ```
-
-## Import Zarf Skeleton
-Below is an example of how to import this projects zarf skeleton into your zarf.yaml. The [uds-package-sofware-factory](https://github.com/defenseunicorns/uds-package-software-factory.git) does this with a subset of the uds-capability projects.
-
-```yaml
-components:
-  - name: values
-    required: true
-    files:
-      - source: <path-to-the-values-you-want-to-use>
-        target: values-gitlab.yaml
-  - name: gitlab
-    required: true
-    import:
-      name: gitlab
-      url: oci://ghcr.io/defenseunicorns/uds-capability/gitlab:0.0.4-skeleton
-```

gitlab-flux-values.yaml

@@ -7,7 +7,192 @@ application:
   ref:
     # renovate: datasource=gitlab-tags depName=big-bang/product/packages/gitlab versioning=loose registryUrl=https://repo1.dso.mil
     tag: 7.1.2-bb.0
-  values: |
-    ###ZARF_VAR_GITLAB_VALUES###
   dependsOn:
     ###ZARF_VAR_GITLAB_DEPENDS_ON###
+  baseValues:
+    hostname: ###ZARF_VAR_DOMAIN###
+    domain: ###ZARF_VAR_DOMAIN###
+
+    global:
+      psql:
+        host: gitlab-postgres
+        port: 5432
+        database: gitlabdb
+        username: gitlab
+        password:
+          useSecret: true
+          secret: gitlab-postgres
+          key: password
+      redis:
+        host: gitlab-redis
+        port: 6379
+        auth:
+          enabled: true
+          secret: gitlab-redis
+          key: password
+      minio:
+        enabled: false
+      # added to help with Gitlab sub-chart configuration
+      image:
+        pullPolicy: IfNotPresent
+      istio:
+        enabled: true
+        injection: enabled
+      hosts:
+        domain: ###ZARF_VAR_DOMAIN###
+
+        gitlab:
+          name: gitlab.###ZARF_VAR_DOMAIN###
+
+        registry:
+          name: registry.###ZARF_VAR_DOMAIN###
+      registry:
+        relativeurls: true
+      pages:
+        objectStore:
+          bucket: uds-gitlab-pages###ZARF_VAR_BUCKET_SUFFIX###
+      appConfig:
+        registry:
+          bucket: uds-gitlab-registry###ZARF_VAR_BUCKET_SUFFIX###
+        lfs:
+          bucket: uds-gitlab-lfs###ZARF_VAR_BUCKET_SUFFIX###
+        artifacts:
+          bucket: uds-gitlab-artifacts###ZARF_VAR_BUCKET_SUFFIX###
+        uploads:
+          bucket: uds-gitlab-uploads###ZARF_VAR_BUCKET_SUFFIX###
+        packages:
+          bucket: uds-gitlab-packages###ZARF_VAR_BUCKET_SUFFIX###
+        externalDiffs:
+          bucket: uds-gitlab-mr-diffs###ZARF_VAR_BUCKET_SUFFIX###
+        terraformState:
+          bucket: uds-gitlab-terraform-state###ZARF_VAR_BUCKET_SUFFIX###
+        ciSecureFiles:
+          bucket: uds-gitlab-ci-secure-files###ZARF_VAR_BUCKET_SUFFIX###
+        dependencyProxy:
+          bucket: uds-gitlab-dependency-proxy###ZARF_VAR_BUCKET_SUFFIX###
+        backups:
+          bucket: uds-gitlab-backups###ZARF_VAR_BUCKET_SUFFIX###
+          tmpBucket: uds-gitlab-tmp###ZARF_VAR_BUCKET_SUFFIX###
+        object_store:
+          enabled: true
+          proxy_download: true
+          connection:
+            secret: gitlab-object-store
+            key: connection
+
+    postgresql:
+      install: false
+    redis:
+      install: false
+
+    registry:
+      annotations:
+        bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+      storage:
+        secret: gitlab-object-store
+        key: registry
+      metrics:
+        serviceMonitor:
+          endpointConfig:
+            scheme: https
+            tlsConfig:
+              caFile: /etc/prom-certs/root-cert.pem
+              certFile: /etc/prom-certs/cert-chain.pem
+              keyFile: /etc/prom-certs/key.pem
+              insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+
+    gitlab:
+      toolbox:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+        backups:
+          objectStorage:
+            config:
+              secret: gitlab-object-store
+              key: backups
+      gitlab-exporter:
+        enabled: true
+        metrics:
+          annotations:
+            bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+          serviceMonitor:
+            endpointConfig:
+              scheme: https
+              tlsConfig:
+                caFile: /etc/prom-certs/root-cert.pem
+                certFile: /etc/prom-certs/cert-chain.pem
+                keyFile: /etc/prom-certs/key.pem
+                insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+      webservice:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+        metrics:
+          serviceMonitor:
+            endpointConfig:
+              scheme: https
+              tlsConfig:
+                caFile: /etc/prom-certs/root-cert.pem
+                certFile: /etc/prom-certs/cert-chain.pem
+                keyFile: /etc/prom-certs/key.pem
+                insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+        workhorse:
+          metrics:
+            serviceMonitor:
+              endpointConfig:
+                scheme: https
+                tlsConfig:
+                  caFile: /etc/prom-certs/root-cert.pem
+                  certFile: /etc/prom-certs/cert-chain.pem
+                  keyFile: /etc/prom-certs/key.pem
+                  insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+      sidekiq:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+      migrations:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+      gitaly:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+        metrics:
+          serviceMonitor:
+            endpointConfig:
+              scheme: https
+              tlsConfig:
+                caFile: /etc/prom-certs/root-cert.pem
+                certFile: /etc/prom-certs/cert-chain.pem
+                keyFile: /etc/prom-certs/key.pem
+                insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+      gitlab-shell:
+        annotations:
+          bigbang.dev/istioVersion: ###ZARF_VAR_ISTIO_VERSION###
+        metrics:
+          serviceMonitor:
+            endpointConfig:
+              scheme: https
+              tlsConfig:
+                caFile: /etc/prom-certs/root-cert.pem
+                certFile: /etc/prom-certs/cert-chain.pem
+                keyFile: /etc/prom-certs/key.pem
+                insecureSkipVerify: true  # Prometheus does not support Istio security naming, thus skip verifying target pod certificate
+
+    openshift: false
+
+    istio:
+      enabled: true
+      injection: enabled
+      gitlab:
+        gateways:
+        - istio-system/tenant
+      registry:
+        gateways:
+        - istio-system/tenant
+
+    monitoring:
+      enabled: true
+
+    networkPolicies:
+      enabled: true
+      ingressLabels:
+        app: tenant-ingressgateway
+      controlPlaneCidr: 0.0.0.0/0