Skip to content
This repository has been archived by the owner on Oct 8, 2024. It is now read-only.

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@anthonywendt
anthonywendt committed Jan 31, 2024
1 parent 3d39964 commit 6579c86
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 165 deletions.
151 changes: 8 additions & 143 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,7 @@
# the build folder.
# renovate: datasource=github-tags depName=defenseunicorns/zarf
ZARF_VERSION := v0.32.2

# The version of the build harness container to use
BUILD_HARNESS_REPO := ghcr.io/defenseunicorns/build-harness/build-harness
# renovate: datasource=docker depName=ghcr.io/defenseunicorns/build-harness/build-harness
BUILD_HARNESS_VERSION := 1.14.8
# renovate: datasource=docker depName=ghcr.io/defenseunicorns/packages/dubbd-k3d extractVersion=^(?<version>\d+\.\d+\.\d+)
DUBBD_K3D_VERSION := 0.17.0
UDS_CLI_VERSION := v0.7.0

# Figure out which Zarf binary we should use based on the operating system we are on
ZARF_BIN := zarf
Expand Down Expand Up @@ -46,91 +40,6 @@ help: ## Show a list of all targets
| sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1:\3/p' \
| column -t -s ":"

########################################################################
# Utility Section
########################################################################

.PHONY: docker-save-build-harness
docker-save-build-harness: ## Pulls the build harness docker image and saves it to a tarball
mkdir -p .cache/docker
docker pull $(BUILD_HARNESS_REPO):$(BUILD_HARNESS_VERSION)
docker save -o .cache/docker/build-harness.tar $(BUILD_HARNESS_REPO):$(BUILD_HARNESS_VERSION)

.PHONY: docker-load-build-harness
docker-load-build-harness: ## Loads the saved build harness docker image
docker load -i .cache/docker/build-harness.tar

.PHONY: run-pre-commit-hooks
run-pre-commit-hooks: ## Run all pre-commit hooks. Returns nonzero exit code if any hooks fail. Uses Docker for maximum compatibility
mkdir -p .cache/pre-commit
docker run --rm -v "${PWD}:/app" --workdir "/app" -e "PRE_COMMIT_HOME=/app/.cache/pre-commit" $(BUILD_HARNESS_REPO):$(BUILD_HARNESS_VERSION) bash -c 'git config --global --add safe.directory /app && asdf install && pre-commit run -a'

.PHONY: fix-cache-permissions
fix-cache-permissions: ## Fixes the permissions on the pre-commit cache
docker run --rm -v "${PWD}:/app" --workdir "/app" -e "PRE_COMMIT_HOME=/app/.cache/pre-commit" $(BUILD_HARNESS_REPO):$(BUILD_HARNESS_VERSION) chmod -R a+rx .cache

########################################################################
# Test Section
########################################################################

.PHONY: test
test: ## Run all automated tests. Requires access to an AWS account. Costs money. Requires env vars "REPO_URL", "GIT_BRANCH", "REGISTRY1_USERNAME", "REGISTRY1_PASSWORD", "GHCR_USERNAME", "GHCR_PASSWORD" and standard AWS env vars.
mkdir -p .cache/go
mkdir -p .cache/go-build
echo "Running automated tests. This will take several minutes. At times it does not log anything to the console. If you interrupt the test run you will need to log into AWS console and manually delete any orphaned infrastructure."
docker run $(TTY_ARG) --rm \
-v "${PWD}:/app" \
-v "${PWD}/.cache/go:/root/go" \
-v "${PWD}/.cache/go-build:/root/.cache/go-build" \
--workdir "/app/test/e2e" \
-e GOPATH=/root/go \
-e GOCACHE=/root/.cache/go-build \
-e REPO_URL \
-e GIT_BRANCH \
-e REGISTRY1_USERNAME \
-e REGISTRY1_PASSWORD \
-e GHCR_USERNAME \
-e GHCR_PASSWORD \
-e AWS_REGION \
-e AWS_DEFAULT_REGION \
-e AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY \
-e AWS_SESSION_TOKEN \
-e AWS_SECURITY_TOKEN \
-e AWS_SESSION_EXPIRATION \
-e SKIP_SETUP -e SKIP_TEST \
-e SKIP_TEARDOWN \
$(BUILD_HARNESS_REPO):$(BUILD_HARNESS_VERSION) \
bash -c 'asdf install && go test -v -timeout 2h -p 1 ./...'

.PHONY: test-ssh
test-ssh: ## Run this if you set SKIP_TEARDOWN=1 and want to SSH into the still-running test server. Don't forget to unset SKIP_TEARDOWN when you're done
cd test/tf/public-ec2-instance && terraform init
cd test/tf/public-ec2-instance/.test-data && cat Ec2KeyPair.json | jq -r .PrivateKey > privatekey.pem && chmod 600 privatekey.pem
cd test/tf/public-ec2-instance && ssh -i .test-data/privatekey.pem ubuntu@$$(terraform output public_instance_ip | tr -d '"')

########################################################################
# Cluster Section
########################################################################

cluster/reset: cluster/destroy cluster/create ## This will destroy any existing cluster and then create a new one

cluster/create: ## Create a k3d cluster with metallb installed
K3D_FIX_MOUNTS=1 k3d cluster create k3d-test-cluster --config utils/k3d/k3d-config.yaml
k3d kubeconfig merge k3d-test-cluster -o /home/${USER}/cluster-kubeconfig.yaml
echo "Installing Calico..."
kubectl apply --wait=true -f utils/calico/calico.yaml 2>&1 >/dev/null
echo "Waiting for Calico to be ready..."
kubectl rollout status deployment/calico-kube-controllers -n kube-system --watch --timeout=90s 2>&1 >/dev/null
kubectl rollout status daemonset/calico-node -n kube-system --watch --timeout=90s 2>&1 >/dev/null
kubectl wait --for=condition=Ready pods --all --all-namespaces 2>&1 >/dev/null
echo
utils/metallb/install.sh
echo "Cluster is ready!"

cluster/destroy: ## Destroy the k3d cluster
k3d cluster delete k3d-test-cluster

########################################################################
# Build Section
########################################################################
Expand All @@ -144,60 +53,16 @@ build: ## Create build directory
clean: ## Clean up build files
rm -rf ./build

.PHONY: build/uds
build/uds: | build ## Download uds-cli to the build dir
if [ -f build/uds ] && [ "$$(build/uds version)" = "$(UDS_CLI_VERSION)" ] ; then exit 0; fi && \
echo "Downloading uds-cli" && \
curl -sL https://github.com/defenseunicorns/uds-cli/releases/download/$(UDS_CLI_VERSION)/uds-cli_$(UDS_CLI_VERSION)_$(UNAME_S)_$(ARCH) -o build/uds && \
chmod +x build/uds

.PHONY: build/zarf
build/zarf: | build ## Download the Zarf to the build dir
if [ -f build/zarf ] && [ "$$(build/zarf version)" = "$(ZARF_VERSION)" ] ; then exit 0; fi && \
echo "Downloading zarf" && \
curl -sL https://github.com/defenseunicorns/zarf/releases/download/$(ZARF_VERSION)/zarf_$(ZARF_VERSION)_$(UNAME_S)_$(ARCH) -o build/zarf && \
chmod +x build/zarf

.PHONY: build/zarf-init
build/zarf-init: | build ## Download the init package
if [ -f build/zarf-init-amd64-$(ZARF_VERSION).tar.zst ] ; then exit 0; fi && \
echo "Downloading zarf-init-amd64-$(ZARF_VERSION).tar.zst" && \
curl -sL https://github.com/defenseunicorns/zarf/releases/download/$(ZARF_VERSION)/zarf-init-amd64-$(ZARF_VERSION).tar.zst -o build/zarf-init-amd64-$(ZARF_VERSION).tar.zst

.PHONY: build/dubbd-k3d
build/dubbd-k3d: | build/zarf ## Download dubbd k3d oci package
if [ -f build/zarf-package-dubbd-k3d-amd64-$(DUBBD_K3D_VERSION).tar.zst ] ; then exit 0; fi && \
cd build && ./zarf package pull oci://ghcr.io/defenseunicorns/packages/dubbd-k3d:$(DUBBD_K3D_VERSION) -a amd64 --oci-concurrency 12

build/test-pkg-deps: | build/zarf ## Build package dependencies for testing
cd build && ./zarf package create ../utils/pkg-deps/namespaces/ --skip-sbom --confirm
cd build && ./zarf package create ../utils/pkg-deps/gitlab/postgres/ --skip-sbom --confirm
cd build && ./zarf package create ../utils/pkg-deps/gitlab/redis/ --skip-sbom --confirm
cd build && ./zarf package create ../utils/pkg-deps/gitlab/minio/ --skip-sbom --confirm

build/uds-capability-gitlab: | build/zarf ## Build the gitlab capability
cd build && ./zarf package create ../ --skip-sbom --confirm

########################################################################
# Deploy Section
########################################################################

deploy/all: deploy/init deploy/dubbd-k3d deploy/test-pkg-deps deploy/uds-capability-gitlab ##

deploy/init: | build/zarf ## Deploy the zarf init package
cd build && ./zarf init --confirm --components=git-server

deploy/dubbd-k3d: | build/zarf ## Deploy the k3d flavor of DUBBD
cd build && ./zarf package deploy zarf-package-dubbd-k3d-amd64-$(DUBBD_K3D_VERSION).tar.zst --confirm

deploy/test-pkg-deps: | build/zarf ## Deploy the package dependencies needed for testing the gitlab capability
cd build && ./zarf package deploy zarf-package-gitlab-namespaces-* --confirm
cd build && ./zarf package deploy zarf-package-gitlab-postgres-* --confirm
cd build && ./zarf package deploy zarf-package-gitlab-redis-* --confirm
cd build && ./zarf package deploy zarf-package-gitlab-minio-* --confirm

deploy/uds-capability-gitlab: | build/zarf ## Deploy the gilab capability
cd build && ./zarf package deploy zarf-package-gitlab-amd*.tar.zst --confirm

########################################################################
# Macro Section
########################################################################

.PHONY: all
all: build/all cluster/reset deploy/all ## Build and deploy gitlab locally

.PHONY: rebuild
rebuild: clean build/all
39 changes: 17 additions & 22 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,51 +56,46 @@ Object Storage works a bit differently as there are many kinds of file stores gi
- uds-gitlab-tmp
- These buckets can have a suffix applied via the `BUCKET_SUFFIX` zarf variable (e.x. `-some-deployment-name` plus `uds-gitlab-backups` would be `uds-gitlab-backups-some-deployment-name`)

## Deploying

### Deploy Everything

#### Via Makefile and local package
## Zarf and UDS.
If you don't have zarf or uds installed there are `make` targets to download the configured versions of those tools.

```bash
# This will destroy and create a compatible k3d cluster then it will run make build/all and make deploy/all. Follow the breadcrumbs in the Makefile to see what and how its doing it.
make cluster/full
make build/zarf
make build/uds
```

### From GHCR OCI Via Zarf
This will place those binaries in the `build` directory. You can use those binaries there or place them on your PATH. You can also use brew to install these tools.

```bash
zarf package deploy ghcr.io/defenseunicorns/uds-capability/gitlab:0.0.6-amd64
brew tap defenseunicorns/tap && brew install uds && brew install zarf
```

## Building

### Use zarf to login to the needed registries i.e. registry1.dso.mil and ghcr.io
### Use zarf to login to the needed registries i.e. registry1.dso.mil

```bash
# Download Zarf
make build/zarf

# Login to the registry
set +o history

# registry1.dso.mil (To access registry1 images needed during build time)
export REGISTRY1_USERNAME="YOUR-USERNAME-HERE"
export REGISTRY1_TOKEN="YOUR-TOKEN-HERE"
echo $REGISTRY1_TOKEN | build/zarf tools registry login registry1.dso.mil --username $REGISTRY1_USERNAME --password-stdin

# ghcr.io (If you need to push to GHCR)
export GH_USERNAME="YOUR-USERNAME-HERE"
export GH_TOKEN="YOUR-TOKEN-HERE"
echo $GH_TOKEN | build/zarf tools registry login ghcr.io --username $GH_USERNAME --password-stdin
echo $REGISTRY1_TOKEN | zarf tools registry login registry1.dso.mil --username $REGISTRY1_USERNAME --password-stdin

set -o history
```

### Creating the Package
# Building and Deploying
There are UDS tasks in this project you can run to build and deploy different pieces.

## List availble tasks
```bash
uds run --list
```

## Build and deploy everything
```bash
make build/uds-capability-gitlab
uds run all
```

## Documentation
Expand Down

0 comments on commit 6579c86

Please sign in to comment.