From 74c248b0c1207c8d3f232b99cb966a73c774ea97 Mon Sep 17 00:00:00 2001 From: minikin Date: Thu, 29 Aug 2024 13:45:37 +0200 Subject: [PATCH 01/20] chore: wip --- utilities/local-cluster/Readme.md | 51 +++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index 7029822ff1..b0681da324 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -1,5 +1,42 @@ # Local Cluster +* [Local Cluster](#local-cluster) + * [Requirements](#requirements) + * [macOS](#macos) + * [Cluster Architecture](#cluster-architecture) + * [Default Services](#default-services) + * [Scylla DB](#scylla-db) + * [Deploying the Cluster](#deploying-the-cluster) + * [Setup hosts on windows](#setup-hosts-on-windows) + * [Startup](#startup) + * [Linux/Windows](#linuxwindows) + * [MacOS](#macos-1) + * [Getting Basic Cluster details](#getting-basic-cluster-details) + * [Suspending the Cluster](#suspending-the-cluster) + * [Resuming a suspended the Cluster](#resuming-a-suspended-the-cluster) + * [Stopping the Cluster](#stopping-the-cluster) + * [Catalyst Voices Services](#catalyst-voices-services) + * [Deploying Catalyst Voices Frontend and Backend Services](#deploying-catalyst-voices-frontend-and-backend-services) + * [Deploying Catalyst Voices Documentation Service](#deploying-catalyst-voices-documentation-service) + * [Debugging the cluster](#debugging-the-cluster) + * [SSH into a running VM](#ssh-into-a-running-vm) + * [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) + + + + +## Requirements + +https://github.com/casey/just + + +### macOS + + + + +https://www.virtualbox.org/wiki/Testbuilds + Integration Tests and local testing will require a running local cluster. To eliminate variability and simplify local deployment, we have standardized the local cluster around: @@ -11,6 +48,8 @@ To eliminate variability and simplify local deployment, we have standardized the These tools allow us to define VMs that are consistent and provide a uniform Kubernetes environment for local testing. + + ## Cluster Architecture The Cluster is based on [K3s](https://k3s.io/), which is a lightweight version of Kubernetes. @@ -56,14 +95,14 @@ For testing purposes, the ScyllaDB is accessible on the Cluster IP Address: `192 ### Setup hosts on windows -On Windows you need to setup the hosts before starting the cluster -From Windows terminal open the hosts file: +On Windows you need to setup the hosts before starting the cluster +From Windows terminal open the hosts file: ```sh -notepad %SystemRoot%\System32\drivers\etc\hosts -``` +notepad %SystemRoot%\System32\drivers\etc\hosts +``` -and copy the hosts from `./shared/extra.hosts` into the Windows host file +and copy the hosts from `./shared/extra.hosts` into the Windows host file ### Startup @@ -176,7 +215,7 @@ vagrant ssh agent99 Found (and tested) description how to connect using only open-source via DBeaver: -1. Download dbeaver (Community Edition) +1. Download dbeaver (Community Edition) https://dbeaver.io/download/ 2. Download cassandra jdbc jar files: (Downloading and Testing the Driver Binaries section have links to binary and source) 3. extract cassandra jdbc zip From 9b8475734a432e80978ac4b8e872606a4aac63a8 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 30 Aug 2024 15:04:19 +0200 Subject: [PATCH 02/20] Update Readme.md --- utilities/local-cluster/Readme.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index b0681da324..581bd4cb86 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -23,8 +23,6 @@ * [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) - - ## Requirements https://github.com/casey/just @@ -33,8 +31,6 @@ https://github.com/casey/just ### macOS - - https://www.virtualbox.org/wiki/Testbuilds Integration Tests and local testing will require a running local cluster. @@ -48,8 +44,6 @@ To eliminate variability and simplify local deployment, we have standardized the These tools allow us to define VMs that are consistent and provide a uniform Kubernetes environment for local testing. - - ## Cluster Architecture The Cluster is based on [K3s](https://k3s.io/), which is a lightweight version of Kubernetes. From e6bd6f00a51152a4b34117a44d68bf005e11417a Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 4 Sep 2024 13:03:29 +0200 Subject: [PATCH 03/20] chore: wip --- utilities/local-cluster/Readme.md | 3 ++ utilities/local-cluster/Vagrantfile | 82 ++++++++++++----------------- utilities/local-cluster/justfile | 8 +-- 3 files changed, 40 insertions(+), 53 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index 581bd4cb86..ef62e2126a 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -27,10 +27,13 @@ https://github.com/casey/just +config.vm.forward_port 80, 5656 + ### macOS +https://www.virtualbox.org/wiki/Testbuilds https://www.virtualbox.org/wiki/Testbuilds Integration Tests and local testing will require a running local cluster. diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index bca02ee762..4095e59b91 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -1,6 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : # cspell: words vcpu, noninteractive, iscsi, preconfiguration, inotify, jetstack + control_ip = "192.168.58.10" # Determine the maximum number of agents, and set their IP addresses @@ -9,11 +10,11 @@ agents = { "agent86" => "192.168.58.86", # This is sized so that a machine with 16 threads and 16GB will allocate at most # ~3/4 of its resources to the cluster. -control_vcpu = "4" # Number of vCPUs in the VM -control_memory = "5192" # 5G of Memory (An extra gig because its the control-plane node) +control_vcpu = "2" # Reduced from 4 to 2 for ARM compatibility +control_memory = "4096" # Reduced from 5192 to 4096 for ARM compatibility -agent_vcpu = "4" # Number of vCPUs in the VM -agent_memory = "4096" # 4G of Memory +agent_vcpu = "2" # Reduced from 4 to 2 for ARM compatibility +agent_memory = "4096" is_darwin_arm64 = Vagrant::Util::Platform.architecture == "arm64" && Vagrant::Util::Platform.darwin? @@ -42,7 +43,7 @@ helm_install_script = <<-SHELL curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list apt-get update - apt-get install helm + apt-get install helm SHELL control_plane_script = <<-SHELL @@ -53,9 +54,6 @@ control_plane_script = <<-SHELL cp /vagrant_shared/k3s/server/manifests/* /var/lib/rancher/k3s/server/manifests/ cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/ # Install and start k3s server node - # Extra parameters in INSTALL_K3S_EXEC variable because of - # K3s picking up the wrong interface when starting the control-plane and agents - # https://github.com/alexellis/k3sup/issues/306 export INSTALL_K3S_EXEC="--bind-address=#{control_ip} --node-external-ip=#{control_ip} --flannel-iface=eth1" curl -sfL https://get.k3s.io | sh - echo "Sleeping for 5 seconds to wait for k3s to start" @@ -82,8 +80,7 @@ agent_script = <<-SHELL SHELL cert_manager_install_script = <<-SHELL - # See: https://cert-manager.io/docs/installation/helm/ - sudo -i + sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add jetstack https://charts.jetstack.io helm repo update @@ -92,16 +89,13 @@ cert_manager_install_script = <<-SHELL SHELL longhorn_install_script = <<-SHELL - # See: https://docs.k3s.io/storage - # https://longhorn.io/docs/1.6.2/deploy/install/install-with-kubectl - # https://longhorn.io/docs/1.6.2/deploy/install/install-with-helm/ - sudo -i + sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add longhorn https://charts.longhorn.io helm repo update helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.6.2 kubectl apply -f /vagrant_shared/k3s/longhorn/storage-classes.yaml - kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml + kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml SHELL local_path_provisioner_script = <<-SHELL @@ -118,12 +112,11 @@ registry_script = <<-SHELL SHELL monitoring_install_script = <<-SHELL - # See: https://k3s.rocks/metrics/ sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo add grafana https://grafana.github.io/helm-charts - helm repo update + helm repo update helm install prometheus-stack --version 58.6.0 -f /vagrant_shared/k3s/grafana-prometheus/prometheus-values.yaml prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --wait kubectl apply -f /vagrant_shared/k3s/grafana-prometheus/alert-manager-ingress.yaml kubectl apply -f /vagrant_shared/k3s/grafana-prometheus/prometheus-ingress.yaml @@ -134,16 +127,12 @@ monitoring_install_script = <<-SHELL SHELL scylladb_install_script = <<-SHELL - # See: https://github.com/scylladb/scylla-operator/blob/master/docs/source/helm.md sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add scylla https://scylla-operator-charts.storage.googleapis.com/stable helm repo update helm install scylla-operator scylla/scylla-operator --values /vagrant_shared/k3s/scylladb/values.operator.yaml --create-namespace --namespace scylla-operator --wait - # Disable the Scylla Manager for now, consumes resources and we aren't using it. - # helm install scylla-manager scylla/scylla-manager --values /vagrant_shared/k3s/scylladb/values.manager.yaml --create-namespace --namespace scylla-manager helm install scylla scylla/scylla --values /vagrant_shared/k3s/scylladb/values.cluster.yaml --create-namespace --namespace scylla - # kubectl apply -f /vagrant_shared/k3s/scylladb/load-balancer.yaml SHELL update_sysctl_limits = <<-SHELL @@ -155,33 +144,34 @@ update_sysctl_limits = <<-SHELL Vagrant.configure("2") do |config| config.vm.box = "generic/debian12" + config.vm.provider "virtualbox" do |vb| + vb.gui = true + vb.memory = "4096" + vb.cpus = 2 + + # Disable USB 2.0, 3.0 controllers and audio. + vb.customize ["modifyvm", :id, "--usb", "off"] + vb.customize ["modifyvm", :id, "--usbehci", "off"] + vb.customize ["modifyvm", :id, "--usbxhci", "off"] + vb.customize ["modifyvm", :id, "--audio", "none"] + end + config.vm.define "control", primary: true do |control| control.vm.network "private_network", ip: control_ip control.vm.hostname = "control" + control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - if !is_darwin_arm64 - # x86 anything should work with this - control.vm.synced_folder "./shared", "/vagrant_shared" - control.vm.provider "virtualbox" do |vb| - vb.memory = control_memory - vb.cpus = control_vcpu - end - else - # Specific config just for Arm Macs. - control.vm.synced_folder "./shared", "/vagrant_shared", type: "smb" - control.vm.provider "qemu" do |qe| - qe.memory = control_memory - qe.smp = control_vcpu - end + control.vm.provider "virtualbox" do |vb| + vb.memory = control_memory + vb.cpus = control_vcpu end + control.vm.provision "shell", inline: update_sysctl_limits control.vm.provision "shell", inline: static_ips control.vm.provision "shell", inline: debian_packages control.vm.provision "shell", inline: helm_install_script control.vm.provision "shell", inline: control_plane_script control.vm.provision "shell", inline: cert_manager_install_script - # We use longhorn, so don;t setup the local-path-provisioner - # control.vm.provision "shell", inline: local_path_provisioner_script control.vm.provision "shell", inline: longhorn_install_script control.vm.provision "shell", inline: monitoring_install_script control.vm.provision "shell", inline: registry_script @@ -192,19 +182,13 @@ Vagrant.configure("2") do |config| config.vm.define agent_name do |agent| agent.vm.network "private_network", ip: agent_ip agent.vm.hostname = agent_name - if !is_darwin_arm64 - agent.vm.synced_folder "./shared", "/vagrant_shared" - agent.vm.provider "virtualbox" do |vb| - vb.memory = agent_memory - vb.cpus = agent_vcpu - end - else - agent.vm.synced_folder "./shared", "/vagrant_shared", type: "smb" - agent.vm.provider "qemu" do |qe| - qe.memory = agent_memory - qe.smp = agent_vcpu - end + agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" + + agent.vm.provider "virtualbox" do |vb| + vb.memory = agent_memory + vb.cpus = agent_vcpu end + agent.vm.provision "shell", inline: update_sysctl_limits agent.vm.provision "shell", inline: static_ips agent.vm.provision "shell", inline: debian_packages diff --git a/utilities/local-cluster/justfile b/utilities/local-cluster/justfile index 22d44ce6b5..dd9e369997 100644 --- a/utilities/local-cluster/justfile +++ b/utilities/local-cluster/justfile @@ -20,8 +20,8 @@ start-cluster: _check-cluster-prereqs # Start the development cluster - only on a aarch64 macos machine start-cluster-aarch64-macos: _check-cluster-prereqs - VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-qemu - vagrant up --provider qemu + VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-vbguest + vagrant up control --provider virtualbox # Pause the development cluster - does not destroy it suspend-cluster: @@ -49,10 +49,10 @@ ssh-into-agent86: # SSH into the agent99 node ssh-into-agent99: - vagrant ssh agent99 + vagrant ssh agent99 get-all-logs: - mkdir -p logs + mkdir -p logs kail --log-level=error --since 6h > cluster.logs # Temporary local scylla dev DB until we can get it exposed from the cluster. From 99bbc83500672e70498bb8379b8d9c3215b5864d Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 6 Sep 2024 13:28:27 +0200 Subject: [PATCH 04/20] feat: add vmware_fusion --- utilities/local-cluster/Vagrantfile | 59 ++++++++++------------------- utilities/local-cluster/justfile | 4 +- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 4095e59b91..61ed177fc4 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -2,18 +2,14 @@ # vi: set ft=ruby : # cspell: words vcpu, noninteractive, iscsi, preconfiguration, inotify, jetstack +# These variables are kept for reference in scripts, but we're not using them for network configuration control_ip = "192.168.58.10" - -# Determine the maximum number of agents, and set their IP addresses agents = { "agent86" => "192.168.58.86", "agent99" => "192.168.58.99" } -# This is sized so that a machine with 16 threads and 16GB will allocate at most -# ~3/4 of its resources to the cluster. -control_vcpu = "2" # Reduced from 4 to 2 for ARM compatibility -control_memory = "4096" # Reduced from 5192 to 4096 for ARM compatibility - -agent_vcpu = "2" # Reduced from 4 to 2 for ARM compatibility +control_vcpu = "2" +control_memory = "4096" +agent_vcpu = "2" agent_memory = "4096" is_darwin_arm64 = Vagrant::Util::Platform.architecture == "arm64" && Vagrant::Util::Platform.darwin? @@ -48,13 +44,15 @@ helm_install_script = <<-SHELL control_plane_script = <<-SHELL sudo -i + # Get the IP of the default interface + DEFAULT_IP=$(ip route get 1 | awk '{print $7;exit}') # Read preconfiguration from shared directory and install it mkdir -p /etc/rancher/k3s/ mkdir -p /var/lib/rancher/k3s/server/manifests/ cp /vagrant_shared/k3s/server/manifests/* /var/lib/rancher/k3s/server/manifests/ cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/ # Install and start k3s server node - export INSTALL_K3S_EXEC="--bind-address=#{control_ip} --node-external-ip=#{control_ip} --flannel-iface=eth1" + export INSTALL_K3S_EXEC="--bind-address=$DEFAULT_IP --node-external-ip=$DEFAULT_IP" curl -sfL https://get.k3s.io | sh - echo "Sleeping for 5 seconds to wait for k3s to start" sleep 5 @@ -69,13 +67,14 @@ control_plane_script = <<-SHELL agent_script = <<-SHELL sudo -i + # Get the IP of the control node + CONTROL_IP=$(getent hosts control | awk '{ print $1 }') # Read preconfiguration from shared directory and install it mkdir -p /etc/rancher/k3s/ cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/ # Install and start k3s agent nodes export K3S_TOKEN_FILE=/vagrant_shared/token - export K3S_URL=https://#{control_ip}:6443 - export INSTALL_K3S_EXEC="--flannel-iface=eth1" + export K3S_URL=https://$CONTROL_IP:6443 curl -sfL https://get.k3s.io | sh - SHELL @@ -98,12 +97,6 @@ longhorn_install_script = <<-SHELL kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml SHELL -local_path_provisioner_script = <<-SHELL - # See: https://github.com/rancher/local-path-provisioner/blob/master/README.md - sudo -i - kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml - SHELL - registry_script = <<-SHELL sudo -i kubectl create namespace registry @@ -142,28 +135,18 @@ update_sysctl_limits = <<-SHELL SHELL Vagrant.configure("2") do |config| - config.vm.box = "generic/debian12" - - config.vm.provider "virtualbox" do |vb| - vb.gui = true - vb.memory = "4096" - vb.cpus = 2 - - # Disable USB 2.0, 3.0 controllers and audio. - vb.customize ["modifyvm", :id, "--usb", "off"] - vb.customize ["modifyvm", :id, "--usbehci", "off"] - vb.customize ["modifyvm", :id, "--usbxhci", "off"] - vb.customize ["modifyvm", :id, "--audio", "none"] - end + config.vm.box = "bento/debian-12-arm64" + + config.vm.network "forwarded_port", guest: 6443, host: 6443 config.vm.define "control", primary: true do |control| - control.vm.network "private_network", ip: control_ip control.vm.hostname = "control" control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - control.vm.provider "virtualbox" do |vb| - vb.memory = control_memory - vb.cpus = control_vcpu + control.vm.provider "vmware_desktop" do |vmware| + vmware.cpus = control_vcpu.to_i + vmware.memory = control_memory.to_i + vmware.gui = false end control.vm.provision "shell", inline: update_sysctl_limits @@ -180,13 +163,13 @@ Vagrant.configure("2") do |config| agents.each do |agent_name, agent_ip| config.vm.define agent_name do |agent| - agent.vm.network "private_network", ip: agent_ip agent.vm.hostname = agent_name agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - agent.vm.provider "virtualbox" do |vb| - vb.memory = agent_memory - vb.cpus = agent_vcpu + agent.vm.provider "vmware_desktop" do |vmware| + vmware.cpus = agent_vcpu.to_i + vmware.memory = agent_memory.to_i + vmware.gui = false end agent.vm.provision "shell", inline: update_sysctl_limits diff --git a/utilities/local-cluster/justfile b/utilities/local-cluster/justfile index dd9e369997..e0c49b83d2 100644 --- a/utilities/local-cluster/justfile +++ b/utilities/local-cluster/justfile @@ -20,8 +20,8 @@ start-cluster: _check-cluster-prereqs # Start the development cluster - only on a aarch64 macos machine start-cluster-aarch64-macos: _check-cluster-prereqs - VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-vbguest - vagrant up control --provider virtualbox + VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-vmware-desktop + vagrant up control --provider=vmware_fusion # Pause the development cluster - does not destroy it suspend-cluster: From 8fc0e4240b88e3ac7bfc89f8d253f37d73fef05f Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 6 Sep 2024 13:30:36 +0200 Subject: [PATCH 05/20] chore: restore local_path_provisioner_script --- utilities/local-cluster/Vagrantfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 61ed177fc4..091c7de796 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -97,6 +97,11 @@ longhorn_install_script = <<-SHELL kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml SHELL +local_path_provisioner_script = <<-SHELL + # See: https://github.com/rancher/local-path-provisioner/blob/master/README.md + sudo -i + kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml + registry_script = <<-SHELL sudo -i kubectl create namespace registry From f54f593d7286978c8f55fdda28cc6c33f89fa483 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 6 Sep 2024 13:31:34 +0200 Subject: [PATCH 06/20] Update Vagrantfile --- utilities/local-cluster/Vagrantfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 091c7de796..6584ac2b76 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -87,20 +87,24 @@ cert_manager_install_script = <<-SHELL kubectl apply -f /vagrant_shared/k3s/cert-manager/self-signed-issuer.yaml SHELL -longhorn_install_script = <<-SHELL - sudo -i + longhorn_install_script = <<-SHELL + # See: https://docs.k3s.io/storage + # https://longhorn.io/docs/1.6.2/deploy/install/install-with-kubectl + # https://longhorn.io/docs/1.6.2/deploy/install/install-with-helm/ + sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add longhorn https://charts.longhorn.io helm repo update helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.6.2 kubectl apply -f /vagrant_shared/k3s/longhorn/storage-classes.yaml - kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml + kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml SHELL local_path_provisioner_script = <<-SHELL # See: https://github.com/rancher/local-path-provisioner/blob/master/README.md sudo -i kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml + SHELL registry_script = <<-SHELL sudo -i From b29deb85891eaa8c2b738e08d3a09cb62aae4d3f Mon Sep 17 00:00:00 2001 From: minikin Date: Sat, 7 Sep 2024 18:12:30 +0200 Subject: [PATCH 07/20] Update Vagrantfile --- utilities/local-cluster/Vagrantfile | 80 ++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 6584ac2b76..583a49eeb8 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -4,13 +4,18 @@ # These variables are kept for reference in scripts, but we're not using them for network configuration control_ip = "192.168.58.10" + +# Determine the maximum number of agents, and set their IP addresses agents = { "agent86" => "192.168.58.86", "agent99" => "192.168.58.99" } -control_vcpu = "2" -control_memory = "4096" -agent_vcpu = "2" -agent_memory = "4096" +# This is sized so that a machine with 16 threads and 16GB will allocate at most +# ~3/4 of its resources to the cluster. +control_vcpu = "4" # Number of vCPUs in the VM +control_memory = "5192" # 5G of Memory (An extra gig because its the control-plane node) + +agent_vcpu = "4" # Number of vCPUs in the VM +agent_memory = "4096" # 4G of Memory is_darwin_arm64 = Vagrant::Util::Platform.architecture == "arm64" && Vagrant::Util::Platform.darwin? @@ -44,15 +49,16 @@ helm_install_script = <<-SHELL control_plane_script = <<-SHELL sudo -i - # Get the IP of the default interface - DEFAULT_IP=$(ip route get 1 | awk '{print $7;exit}') # Read preconfiguration from shared directory and install it mkdir -p /etc/rancher/k3s/ mkdir -p /var/lib/rancher/k3s/server/manifests/ cp /vagrant_shared/k3s/server/manifests/* /var/lib/rancher/k3s/server/manifests/ cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/ # Install and start k3s server node - export INSTALL_K3S_EXEC="--bind-address=$DEFAULT_IP --node-external-ip=$DEFAULT_IP" + # Extra parameters in INSTALL_K3S_EXEC variable because of + # K3s picking up the wrong interface when starting the control-plane and agents + # https://github.com/alexellis/k3sup/issues/306 + export INSTALL_K3S_EXEC="--bind-address=#{control_ip} --node-external-ip=#{control_ip} --flannel-iface=eth1" curl -sfL https://get.k3s.io | sh - echo "Sleeping for 5 seconds to wait for k3s to start" sleep 5 @@ -67,19 +73,19 @@ control_plane_script = <<-SHELL agent_script = <<-SHELL sudo -i - # Get the IP of the control node - CONTROL_IP=$(getent hosts control | awk '{ print $1 }') # Read preconfiguration from shared directory and install it mkdir -p /etc/rancher/k3s/ cp /vagrant_shared/k3s/registries.yaml /etc/rancher/k3s/ # Install and start k3s agent nodes export K3S_TOKEN_FILE=/vagrant_shared/token - export K3S_URL=https://$CONTROL_IP:6443 + export K3S_URL=https://#{control_ip}:6443 + export INSTALL_K3S_EXEC="--flannel-iface=eth1" curl -sfL https://get.k3s.io | sh - SHELL cert_manager_install_script = <<-SHELL - sudo -i + # See: https://cert-manager.io/docs/installation/helm/ + sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add jetstack https://charts.jetstack.io helm repo update @@ -87,7 +93,7 @@ cert_manager_install_script = <<-SHELL kubectl apply -f /vagrant_shared/k3s/cert-manager/self-signed-issuer.yaml SHELL - longhorn_install_script = <<-SHELL +longhorn_install_script = <<-SHELL # See: https://docs.k3s.io/storage # https://longhorn.io/docs/1.6.2/deploy/install/install-with-kubectl # https://longhorn.io/docs/1.6.2/deploy/install/install-with-helm/ @@ -97,7 +103,7 @@ cert_manager_install_script = <<-SHELL helm repo update helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.6.2 kubectl apply -f /vagrant_shared/k3s/longhorn/storage-classes.yaml - kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml + kubectl apply -f /vagrant_shared/k3s/longhorn/ingress.yaml SHELL local_path_provisioner_script = <<-SHELL @@ -114,6 +120,7 @@ registry_script = <<-SHELL SHELL monitoring_install_script = <<-SHELL + # See: https://k3s.rocks/metrics/ sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add prometheus-community https://prometheus-community.github.io/helm-charts @@ -129,12 +136,16 @@ monitoring_install_script = <<-SHELL SHELL scylladb_install_script = <<-SHELL + # See: https://github.com/scylladb/scylla-operator/blob/master/docs/source/helm.md sudo -i export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm repo add scylla https://scylla-operator-charts.storage.googleapis.com/stable helm repo update helm install scylla-operator scylla/scylla-operator --values /vagrant_shared/k3s/scylladb/values.operator.yaml --create-namespace --namespace scylla-operator --wait + # Disable the Scylla Manager for now, consumes resources and we aren't using it. + # helm install scylla-manager scylla/scylla-manager --values /vagrant_shared/k3s/scylladb/values.manager.yaml --create-namespace --namespace scylla-manager helm install scylla scylla/scylla --values /vagrant_shared/k3s/scylladb/values.cluster.yaml --create-namespace --namespace scylla + # kubectl apply -f /vagrant_shared/k3s/scylladb/load-balancer.yaml SHELL update_sysctl_limits = <<-SHELL @@ -144,18 +155,31 @@ update_sysctl_limits = <<-SHELL SHELL Vagrant.configure("2") do |config| - config.vm.box = "bento/debian-12-arm64" + if is_darwin_arm64 + config.vm.box = "bento/debian-12-arm64" + else + config.vm.box = "bento/debian-12" + end config.vm.network "forwarded_port", guest: 6443, host: 6443 config.vm.define "control", primary: true do |control| control.vm.hostname = "control" - control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - control.vm.provider "vmware_desktop" do |vmware| - vmware.cpus = control_vcpu.to_i - vmware.memory = control_memory.to_i - vmware.gui = false + if is_darwin_arm64 + control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" + control.vm.provider "vmware_desktop" do |vmware| + vmware.cpus = control_vcpu + vmware.memory = control_memory + vmware.gui = false + end + else + control.vm.network "private_network", ip: control_ip + control.vm.synced_folder "./shared", "/vagrant_shared" + control.vm.provider "virtualbox" do |vb| + vb.cpus = control_vcpu + vb.memory = control_memory + end end control.vm.provision "shell", inline: update_sysctl_limits @@ -173,12 +197,20 @@ Vagrant.configure("2") do |config| agents.each do |agent_name, agent_ip| config.vm.define agent_name do |agent| agent.vm.hostname = agent_name - agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - agent.vm.provider "vmware_desktop" do |vmware| - vmware.cpus = agent_vcpu.to_i - vmware.memory = agent_memory.to_i - vmware.gui = false + if is_darwin_arm64 + agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" + agent.vm.provider "vmware_desktop" do |vmware| + vmware.cpus = agent_vcpu + vmware.memory = agent_memory + vmware.gui = false + end + else + agent.vm.synced_folder "./shared", "/vagrant_shared" + agent.vm.provider "virtualbox" do |vb| + vb.cpus = agent_vcpu + vb.memory = agent_memory + end end agent.vm.provision "shell", inline: update_sysctl_limits From c5ee210bda511addb5b35c655023a6294c3c7d8c Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 4 Oct 2024 13:39:35 +0200 Subject: [PATCH 08/20] utils: wip --- utilities/local-cluster/Readme.md | 10 ++++++++++ utilities/local-cluster/Vagrantfile | 18 +++++++++--------- utilities/local-cluster/justfile | 5 ++--- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index ab7c3f2ea8..25935521b9 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -21,6 +21,7 @@ - [Debugging the cluster](#debugging-the-cluster) - [SSH into a running VM](#ssh-into-a-running-vm) - [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) + - [Troubleshooting](#troubleshooting) ## Requirements @@ -232,3 +233,12 @@ Found (and tested) description how to connect using only open-source via DBeaver 9. click Find Class to make sure the Class Name is found okay 10. click OK 11. Create New Connection, selecting the database driver you just added + +## Troubleshooting + +If you encounter any weird issues with the Vagrant, you can try the following: + +```sh +rm -rf .vagrant +``` +and then restart the cluster. \ No newline at end of file diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index db4d0def55..1ca21f6472 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -156,7 +156,7 @@ update_sysctl_limits = <<-SHELL Vagrant.configure("2") do |config| if is_darwin_arm64 - config.vm.box = "bento/debian-12-arm64" + config.vm.box = "generic/debian12" else config.vm.box = "bento/debian-12" end @@ -168,10 +168,10 @@ Vagrant.configure("2") do |config| if is_darwin_arm64 control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - control.vm.provider "vmware_desktop" do |vmware| - vmware.cpus = control_vcpu - vmware.memory = control_memory - vmware.gui = false + control.vm.provider "qemu" do |qe| + qe.cpus = control_vcpu + qe.memory = control_memory + qe.gui = false end else control.vm.network "private_network", ip: control_ip @@ -202,10 +202,10 @@ Vagrant.configure("2") do |config| if is_darwin_arm64 agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - agent.vm.provider "vmware_desktop" do |vmware| - vmware.cpus = agent_vcpu - vmware.memory = agent_memory - vmware.gui = false + agent.vm.provider "qemu" do |qe| + qe.cpus = agent_vcpu + qe.memory = agent_memory + qe.gui = false end else agent.vm.synced_folder "./shared", "/vagrant_shared" diff --git a/utilities/local-cluster/justfile b/utilities/local-cluster/justfile index 50f6d9b315..71c78320f8 100644 --- a/utilities/local-cluster/justfile +++ b/utilities/local-cluster/justfile @@ -24,10 +24,9 @@ _check-cluster-prereqs: start-cluster: _check-cluster-prereqs vagrant up -# Start the development cluster - only on a aarch64 macos machine start-cluster-aarch64-macos: _check-cluster-prereqs - VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-vmware-desktop - vagrant up control --provider=vmware_fusion + VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-qemu + vagrant up control --provider=qemu # Pause the development cluster - does not destroy it suspend-cluster: From 6bdfba5812b712c49a6f8502f390f221f43e55a9 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 4 Oct 2024 16:06:16 +0200 Subject: [PATCH 09/20] chore: wip --- utilities/local-cluster/Readme.md | 135 +++++++++++++++++++++++----- utilities/local-cluster/Vagrantfile | 10 ++- 2 files changed, 121 insertions(+), 24 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index 25935521b9..f3a67308e8 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -22,21 +22,12 @@ - [SSH into a running VM](#ssh-into-a-running-vm) - [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) - [Troubleshooting](#troubleshooting) - + - [Issues with Vagrant configurations](#issues-with-vagrant-configurations) + - [Issues with running command with just](#issues-with-running-command-with-just) + - [Vagrant cannot forward the specified ports on a VM](#vagrant-cannot-forward-the-specified-ports-on-a-vm) ## Requirements -https://github.com/casey/just - -config.vm.forward_port 80, 5656 - - -### macOS - - -https://www.virtualbox.org/wiki/Testbuilds -https://www.virtualbox.org/wiki/Testbuilds - Integration Tests and local testing will require a running local cluster. To eliminate variability and simplify local deployment, we have standardized the local cluster around: @@ -44,6 +35,11 @@ To eliminate variability and simplify local deployment, we have standardized the * [Vagrant](https://developer.hashicorp.com/vagrant/install?product_intent=vagrant) * [kubectl](https://kubernetes.io/docs/tasks/tools/) * [helm](https://helm.sh/docs/intro/install/) +* [just](https://github.com/casey/just) + +### macOS + +https://www.virtualbox.org/wiki/Testbuilds These tools allow us to define VMs that are consistent and provide a uniform Kubernetes environment for local testing. @@ -213,13 +209,13 @@ vagrant ssh agent99 Found (and tested) description how to connect using only open-source via DBeaver: -1. Download dbeaver (Community Edition) https://dbeaver.io/download/ -2. Download cassandra jdbc jar files: http://www.dbschema.com/cassandra-jdbc-driver.html +1. Download dbeaver (Community Edition) [dbeaver](https://dbeaver.io/download) +2. Download cassandra jdbc jar files: [cassandra-jdbc-drive](http://www.dbschema.com/cassandra-jdbc-driver.html) (Downloading and Testing the Driver Binaries section have links to binary and source) -3. extract Cassandra JDBC zip -4. run DBeaver -5. go to Database > Driver Manager -6. click New +3. Extract Cassandra JDBC zip +4. Run DBeaver +5. Go to `Database` > `Driver Manager` +6. Click `New` 7. Fill in details as follows: * Driver Name: `Cassandra` (or whatever you want it to say) * Driver Type: `Generic` @@ -229,16 +225,109 @@ Found (and tested) description how to connect using only open-source via DBeaver * Embedded: `no` * Category: * Description: `Cassandra` (or whatever you want it to say) -8. click Add File and add all the jars in the Cassandra JDBC zip file. -9. click Find Class to make sure the Class Name is found okay -10. click OK -11. Create New Connection, selecting the database driver you just added +8. Click Add File and add all the jars in the Cassandra JDBC zip file. +9. Click Find Class to make sure the Class Name is found okay. +10. Click `OK`. +11. Create `New Connection`, selecting the database driver you just added. ## Troubleshooting +### Issues with Vagrant configurations + If you encounter any weird issues with the Vagrant, you can try the following: ```sh rm -rf .vagrant ``` -and then restart the cluster. \ No newline at end of file +and then restart the cluster. + +### Issues with running command with just + +If you encounter any issues with running the commands with `just`, +you can try running the commands directly. + +Instead of: + +```sh +just stop-cluster +``` + +run: + +```sh +vagrant destroy -f +``` +For more see [justfile](./justfile) + +### Vagrant cannot forward the specified ports on a VM + +1. Check if Port 6443 is in Use: + +```sh +sudo lsof -i :6443 +``` +You can see the output like this: + +```sh +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +qemu-syst 42886 username 19u IPv4 0x9af1245915ca3e76 0t0 TCP *:sun-sr-https (LISTEN) +``` + +The process qemu-syst (which stands for qemu-system), with PID 42886, is listening on port 6443. +This means that a QEMU virtual machine is currently running and using port 6443 on your host machine. + +Since you’re using the QEMU provider with Vagrant, it’s likely that an existing VM is still running and occupying port 6443. +This is preventing your new Vagrant VM from forwarding port 6443, resulting in the error + +2. List all running Vagrant VMs to see if any are currently active: + +```sh +vagrant global-status +``` + +This command will list all Vagrant environments on your system. Look for any VMs that are in the running state. + +```sh +id name provider state directory +-------------------------------------------------------------------------------------- +abcd123 control qemu running /path/to/your/project +efgh456 othervm qemu running /another/project +``` +3. Halt or Destroy the Existing VM + +If you find that a VM is running that you no longer need, you can halt or destroy it. + +**To halt the VM:** + +```sh +vagrant halt +``` + +**To destroy the VM:** + +```sh +vagrant destroy +``` + +Replace with the ID of the VM from the vagrant global-status output. +For example: `vagrant halt abcd123` + +4. Kill the QEMU Process Manually (If Necessary) + +If the VM is not managed by Vagrant, or if it did not shut down properly, +you may need to kill the QEMU process manually. + +```sh +kill 42886 +``` + +Replace 42886 with the PID from your lsof output. + +5. Confirm the Process is Terminated: + +```sh +sudo lsof -i :6443 +``` +This should return no output, indicating that no process is listening on port 6443. + +6. Retry Bringing Up the Vagrant VM. diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 1ca21f6472..4aace9ffe3 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -157,6 +157,13 @@ update_sysctl_limits = <<-SHELL Vagrant.configure("2") do |config| if is_darwin_arm64 config.vm.box = "generic/debian12" + config.ssh.insert_key = false + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + config.ssh.private_key_path = ["~/.vagrant.d/insecure_private_key"] + config.ssh.forward_agent = false + config.ssh.forward_x11 = false + config.ssh.keep_alive = true else config.vm.box = "bento/debian-12" end @@ -164,6 +171,7 @@ Vagrant.configure("2") do |config| config.vm.network "forwarded_port", guest: 6443, host: 6443 config.vm.define "control", primary: true do |control| + control.vm.network "private_network", ip: control_ip control.vm.hostname = "control" if is_darwin_arm64 @@ -171,7 +179,7 @@ Vagrant.configure("2") do |config| control.vm.provider "qemu" do |qe| qe.cpus = control_vcpu qe.memory = control_memory - qe.gui = false + qe.gui = true end else control.vm.network "private_network", ip: control_ip From b3fe0bbfd06a9a603ad9a2811e37571e762ff7da Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 12:32:56 +0200 Subject: [PATCH 10/20] chore: qemu wip --- utilities/local-cluster/Readme.md | 52 ++++++++++++++--------------- utilities/local-cluster/Vagrantfile | 22 +++++++----- utilities/local-cluster/justfile | 2 +- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index f3a67308e8..631b23411c 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -1,30 +1,30 @@ # Local Cluster -- [Local Cluster](#local-cluster) - - [Requirements](#requirements) - - [macOS](#macos) - - [Cluster Architecture](#cluster-architecture) - - [Default Services](#default-services) - - [Scylla DB](#scylla-db) - - [Deploying the Cluster](#deploying-the-cluster) - - [Setup hosts on Windows](#setup-hosts-on-windows) - - [Startup](#startup) - - [Linux/Windows](#linuxwindows) - - [macOS](#macos-1) - - [Getting Basic Cluster details](#getting-basic-cluster-details) - - [Suspending the Cluster](#suspending-the-cluster) - - [Resuming a suspended the Cluster](#resuming-a-suspended-the-cluster) - - [Stopping the Cluster](#stopping-the-cluster) - - [Catalyst Voices Services](#catalyst-voices-services) - - [Deploying Catalyst Voices Frontend and Backend Services](#deploying-catalyst-voices-frontend-and-backend-services) - - [Deploying Catalyst Voices Documentation Service](#deploying-catalyst-voices-documentation-service) - - [Debugging the cluster](#debugging-the-cluster) - - [SSH into a running VM](#ssh-into-a-running-vm) - - [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) - - [Troubleshooting](#troubleshooting) - - [Issues with Vagrant configurations](#issues-with-vagrant-configurations) - - [Issues with running command with just](#issues-with-running-command-with-just) - - [Vagrant cannot forward the specified ports on a VM](#vagrant-cannot-forward-the-specified-ports-on-a-vm) +* [Local Cluster](#local-cluster) + * [Requirements](#requirements) + * [macOS](#macos) + * [Cluster Architecture](#cluster-architecture) + * [Default Services](#default-services) + * [Scylla DB](#scylla-db) + * [Deploying the Cluster](#deploying-the-cluster) + * [Setup hosts on Windows](#setup-hosts-on-windows) + * [Startup](#startup) + * [Linux/Windows](#linuxwindows) + * [macOS](#macos-1) + * [Getting Basic Cluster details](#getting-basic-cluster-details) + * [Suspending the Cluster](#suspending-the-cluster) + * [Resuming a suspended the Cluster](#resuming-a-suspended-the-cluster) + * [Stopping the Cluster](#stopping-the-cluster) + * [Catalyst Voices Services](#catalyst-voices-services) + * [Deploying Catalyst Voices Frontend and Backend Services](#deploying-catalyst-voices-frontend-and-backend-services) + * [Deploying Catalyst Voices Documentation Service](#deploying-catalyst-voices-documentation-service) + * [Debugging the cluster](#debugging-the-cluster) + * [SSH into a running VM](#ssh-into-a-running-vm) + * [Local UI to access ScyllaDB](#local-ui-to-access-scylladb) + * [Troubleshooting](#troubleshooting) + * [Issues with Vagrant configurations](#issues-with-vagrant-configurations) + * [Issues with running command with just](#issues-with-running-command-with-just) + * [Vagrant cannot forward the specified ports on a VM](#vagrant-cannot-forward-the-specified-ports-on-a-vm) ## Requirements @@ -291,7 +291,7 @@ This command will list all Vagrant environments on your system. Look for any VMs id name provider state directory -------------------------------------------------------------------------------------- abcd123 control qemu running /path/to/your/project -efgh456 othervm qemu running /another/project +abcd456 other qemu running /another/project ``` 3. Halt or Destroy the Existing VM diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 4aace9ffe3..1548c0cd2e 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -9,7 +9,7 @@ control_ip = "192.168.58.10" agents = { "agent86" => "192.168.58.86", "agent99" => "192.168.58.99" } -# This is sized so that a machine with 16 threads and 16GB will allocate at most +# This is sized so that a machine with 16 threads and 16GB will allocate at most # ~3/4 of its resources to the cluster. control_vcpu = "4" # Number of vCPUs in the VM control_memory = "5192" # 5G of Memory (An extra gig because its the control-plane node) @@ -157,12 +157,13 @@ update_sysctl_limits = <<-SHELL Vagrant.configure("2") do |config| if is_darwin_arm64 config.vm.box = "generic/debian12" - config.ssh.insert_key = false - config.ssh.username = "vagrant" + config.vm.boot_timeout = 600 + # config.ssh.insert_key = false + # config.ssh.username = "vagrant" config.ssh.password = "vagrant" config.ssh.private_key_path = ["~/.vagrant.d/insecure_private_key"] - config.ssh.forward_agent = false - config.ssh.forward_x11 = false + # config.ssh.forward_agent = false + # config.ssh.forward_x11 = false config.ssh.keep_alive = true else config.vm.box = "bento/debian-12" @@ -177,9 +178,14 @@ Vagrant.configure("2") do |config| if is_darwin_arm64 control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" control.vm.provider "qemu" do |qe| - qe.cpus = control_vcpu - qe.memory = control_memory - qe.gui = true + qe.arch = "aarch64" + qe.machine = "virt" + qe.cpu = "cortex-a72" + qe.memory = "1G" + qe.qemu_dir = "/opt/homebrew/share/qemu" + # qe.cpus = control_vcpu + # qe.memory = control_memory + # qe.gui = true end else control.vm.network "private_network", ip: control_ip diff --git a/utilities/local-cluster/justfile b/utilities/local-cluster/justfile index 71c78320f8..63e5cec41a 100644 --- a/utilities/local-cluster/justfile +++ b/utilities/local-cluster/justfile @@ -25,7 +25,7 @@ start-cluster: _check-cluster-prereqs vagrant up start-cluster-aarch64-macos: _check-cluster-prereqs - VAGRANT_LOG=debug VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-qemu + VAGRANT_LOG=debug vagrant plugin install vagrant-qemu vagrant up control --provider=qemu # Pause the development cluster - does not destroy it From 9a668be9367074dd73a0867ca6af4a10f8033c82 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 12:51:10 +0200 Subject: [PATCH 11/20] feat: add parallels as provider --- utilities/local-cluster/Vagrantfile | 22 ++++------------------ utilities/local-cluster/justfile | 4 ++-- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 1548c0cd2e..2bca0352f5 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -156,15 +156,7 @@ update_sysctl_limits = <<-SHELL Vagrant.configure("2") do |config| if is_darwin_arm64 - config.vm.box = "generic/debian12" - config.vm.boot_timeout = 600 - # config.ssh.insert_key = false - # config.ssh.username = "vagrant" - config.ssh.password = "vagrant" - config.ssh.private_key_path = ["~/.vagrant.d/insecure_private_key"] - # config.ssh.forward_agent = false - # config.ssh.forward_x11 = false - config.ssh.keep_alive = true + config.vm.box = "bento/ubuntu-20.04-arm64" else config.vm.box = "bento/debian-12" end @@ -177,15 +169,9 @@ Vagrant.configure("2") do |config| if is_darwin_arm64 control.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - control.vm.provider "qemu" do |qe| - qe.arch = "aarch64" - qe.machine = "virt" - qe.cpu = "cortex-a72" - qe.memory = "1G" - qe.qemu_dir = "/opt/homebrew/share/qemu" - # qe.cpus = control_vcpu - # qe.memory = control_memory - # qe.gui = true + control.vm.provider "parallels" do |p| + p.cpus = control_vcpu + p.memory = control_memory end else control.vm.network "private_network", ip: control_ip diff --git a/utilities/local-cluster/justfile b/utilities/local-cluster/justfile index 63e5cec41a..24f65c0a01 100644 --- a/utilities/local-cluster/justfile +++ b/utilities/local-cluster/justfile @@ -25,8 +25,8 @@ start-cluster: _check-cluster-prereqs vagrant up start-cluster-aarch64-macos: _check-cluster-prereqs - VAGRANT_LOG=debug vagrant plugin install vagrant-qemu - vagrant up control --provider=qemu + VAGRANT_LOG=debug vagrant plugin install vagrant-parallels + vagrant up control --provider=parallels # Pause the development cluster - does not destroy it suspend-cluster: From c5ef98fcab53109ae53ef793739a813c0b4e8863 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 12:55:38 +0200 Subject: [PATCH 12/20] Update Readme.md --- utilities/local-cluster/Readme.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index 631b23411c..eedc53b132 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -2,7 +2,6 @@ * [Local Cluster](#local-cluster) * [Requirements](#requirements) - * [macOS](#macos) * [Cluster Architecture](#cluster-architecture) * [Default Services](#default-services) * [Scylla DB](#scylla-db) @@ -10,7 +9,7 @@ * [Setup hosts on Windows](#setup-hosts-on-windows) * [Startup](#startup) * [Linux/Windows](#linuxwindows) - * [macOS](#macos-1) + * [macOS](#macos) * [Getting Basic Cluster details](#getting-basic-cluster-details) * [Suspending the Cluster](#suspending-the-cluster) * [Resuming a suspended the Cluster](#resuming-a-suspended-the-cluster) @@ -31,18 +30,12 @@ Integration Tests and local testing will require a running local cluster. To eliminate variability and simplify local deployment, we have standardized the local cluster around: -* [VirtualBox](https://www.virtualbox.org/) +* [VirtualBox](https://www.virtualbox.org/) for Linux or [Parallels](https://www.parallels.com/) for macOS * [Vagrant](https://developer.hashicorp.com/vagrant/install?product_intent=vagrant) * [kubectl](https://kubernetes.io/docs/tasks/tools/) * [helm](https://helm.sh/docs/intro/install/) * [just](https://github.com/casey/just) -### macOS - -https://www.virtualbox.org/wiki/Testbuilds - -These tools allow us to define VMs that are consistent and provide a uniform Kubernetes environment -for local testing. ## Cluster Architecture @@ -330,4 +323,4 @@ sudo lsof -i :6443 ``` This should return no output, indicating that no process is listening on port 6443. -6. Retry Bringing Up the Vagrant VM. +6. Retry bringing up the Vagrant VM. From 51930f5fd6ca10536c6217e2d212f7477282a174 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 13:23:40 +0200 Subject: [PATCH 13/20] Update Readme.md --- utilities/local-cluster/Readme.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index eedc53b132..1fbf43170d 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -36,7 +36,6 @@ To eliminate variability and simplify local deployment, we have standardized the * [helm](https://helm.sh/docs/intro/install/) * [just](https://github.com/casey/just) - ## Cluster Architecture The Cluster is based on [K3s](https://k3s.io/), which is a lightweight version of Kubernetes. @@ -232,6 +231,7 @@ If you encounter any weird issues with the Vagrant, you can try the following: ```sh rm -rf .vagrant ``` + and then restart the cluster. ### Issues with running command with just @@ -250,6 +250,7 @@ run: ```sh vagrant destroy -f ``` + For more see [justfile](./justfile) ### Vagrant cannot forward the specified ports on a VM @@ -259,6 +260,7 @@ For more see [justfile](./justfile) ```sh sudo lsof -i :6443 ``` + You can see the output like this: ```sh @@ -278,7 +280,8 @@ This is preventing your new Vagrant VM from forwarding port 6443, resulting in t vagrant global-status ``` -This command will list all Vagrant environments on your system. Look for any VMs that are in the running state. +This command will list all Vagrant environments on your system. +Look for any VMs that are in the running state. ```sh id name provider state directory @@ -286,7 +289,8 @@ id name provider state directory abcd123 control qemu running /path/to/your/project abcd456 other qemu running /another/project ``` -3. Halt or Destroy the Existing VM + +3. Halt or Destroy the Existing VM If you find that a VM is running that you no longer need, you can halt or destroy it. @@ -321,6 +325,7 @@ Replace 42886 with the PID from your lsof output. ```sh sudo lsof -i :6443 ``` + This should return no output, indicating that no process is listening on port 6443. 6. Retry bringing up the Vagrant VM. From 82585643648cb14de2e8fb33dc68558df69479ec Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 14:01:50 +0200 Subject: [PATCH 14/20] Update Readme.md --- utilities/local-cluster/Readme.md | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index 1fbf43170d..c76d8a14d8 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -24,6 +24,12 @@ * [Issues with Vagrant configurations](#issues-with-vagrant-configurations) * [Issues with running command with just](#issues-with-running-command-with-just) * [Vagrant cannot forward the specified ports on a VM](#vagrant-cannot-forward-the-specified-ports-on-a-vm) + * [Check if Port 6443 is in Use](#check-if-port-6443-is-in-use) + * [List all running Vagrant VMs to see if any are currently active](#list-all-running-vagrant-vms-to-see-if-any-are-currently-active) + * [Halt or Destroy the Existing VM](#halt-or-destroy-the-existing-vm) + * [Kill the VM Process Manually (If Necessary)](#kill-the-vm-process-manually-if-necessary) + * [Confirm the Process is Terminated](#confirm-the-process-is-terminated) + * [Retry bringing up the Vagrant VM](#retry-bringing-up-the-vagrant-vm) ## Requirements @@ -255,7 +261,7 @@ For more see [justfile](./justfile) ### Vagrant cannot forward the specified ports on a VM -1. Check if Port 6443 is in Use: +#### Check if Port 6443 is in Use ```sh sudo lsof -i :6443 @@ -265,16 +271,17 @@ You can see the output like this: ```sh COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -qemu-syst 42886 username 19u IPv4 0x9af1245915ca3e76 0t0 TCP *:sun-sr-https (LISTEN) +parallels-syst 42886 username 19u IPv4 0x9af1245915ca3e76 0t0 TCP *:sun-sr-https (LISTEN) ``` -The process qemu-syst (which stands for qemu-system), with PID 42886, is listening on port 6443. -This means that a QEMU virtual machine is currently running and using port 6443 on your host machine. +The process parallels-syst, with PID 42886, is listening on port 6443. +This means that a parallels virtual machine (can be VirtualBox) is currently running and using port 6443 on your host machine. -Since you’re using the QEMU provider with Vagrant, it’s likely that an existing VM is still running and occupying port 6443. +Since you’re using the VirtualBox/Parallels provider with Vagrant +it’s likely that an existing VM is still running and occupying port 6443. This is preventing your new Vagrant VM from forwarding port 6443, resulting in the error -2. List all running Vagrant VMs to see if any are currently active: +#### List all running Vagrant VMs to see if any are currently active ```sh vagrant global-status @@ -286,11 +293,11 @@ Look for any VMs that are in the running state. ```sh id name provider state directory -------------------------------------------------------------------------------------- -abcd123 control qemu running /path/to/your/project -abcd456 other qemu running /another/project +abcd123 control parallels running /path/to/your/project +abcd456 other parallels running /another/project ``` -3. Halt or Destroy the Existing VM +#### Halt or Destroy the Existing VM If you find that a VM is running that you no longer need, you can halt or destroy it. @@ -306,13 +313,13 @@ vagrant halt vagrant destroy ``` -Replace with the ID of the VM from the vagrant global-status output. +Replace ID with the ID of the VM from the vagrant global-status output. For example: `vagrant halt abcd123` -4. Kill the QEMU Process Manually (If Necessary) +#### Kill the VM Process Manually (If Necessary) If the VM is not managed by Vagrant, or if it did not shut down properly, -you may need to kill the QEMU process manually. +you may need to kill the VM process manually. ```sh kill 42886 @@ -320,7 +327,7 @@ kill 42886 Replace 42886 with the PID from your lsof output. -5. Confirm the Process is Terminated: +#### Confirm the Process is Terminated ```sh sudo lsof -i :6443 @@ -328,4 +335,4 @@ sudo lsof -i :6443 This should return no output, indicating that no process is listening on port 6443. -6. Retry bringing up the Vagrant VM. +#### Retry bringing up the Vagrant VM From f2c79c578fe7feb7d11b1accd4a32873e93de097 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 15:22:39 +0200 Subject: [PATCH 15/20] chore: clean up cspell --- .config/dictionaries/project.dic | 5 ++++- utilities/local-cluster/Vagrantfile | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index d145a4877e..cc56e70092 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -51,6 +51,7 @@ codepoints commitlog coti coverallsapp +cpu CQLSH cryptoxide Cunego @@ -113,6 +114,7 @@ interps inversed iohk iphoneos +iscsi jdbc jetbrains Joaquín @@ -170,6 +172,7 @@ netifas netkey nextest Nodetool +noninteractive OCSP Oleksandr onboarded @@ -296,4 +299,4 @@ xctest xctestrun xcworkspace xvfb -yoroi +yoroi \ No newline at end of file diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index 2bca0352f5..e75102a20b 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -1,6 +1,5 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# cspell: words vcpu, noninteractive, iscsi, preconfiguration, inotify, jetstack # These variables are kept for reference in scripts, but we're not using them for network configuration control_ip = "192.168.58.10" From ac0a95401e62baa5a4e6c5062b606b4a4acfdb0a Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 16:08:56 +0200 Subject: [PATCH 16/20] Update project.dic --- .config/dictionaries/project.dic | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index cc56e70092..40b74505a2 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -21,6 +21,8 @@ autorecalculates autoresizing backendpython bech +bento +bento bimap bindgen bkioshn @@ -117,12 +119,14 @@ iphoneos iscsi jdbc jetbrains +jetstack Joaquín jorm jormungandr Jörmungandr junitreport junitxml +justfile Keyhash keyserver keyspace @@ -192,6 +196,7 @@ postcss Pozhylenkov Precache Precertificate +preconfiguration preprod projectcatalyst Prokhorenko @@ -274,6 +279,7 @@ Unstaked UTXO Utxos varint +vcpu Vespr vite vitss From b8fab5a634a85285b478cb89ed0dd896c2e35a98 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 16:10:32 +0200 Subject: [PATCH 17/20] Update project.dic --- .config/dictionaries/project.dic | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 40b74505a2..292f0c11f4 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -109,6 +109,7 @@ icudtl ideascale idents ilap +inotify Instantitation integ Intellij From f310e9020ec34a124ad3292e607b0e570463a535 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 16:16:01 +0200 Subject: [PATCH 18/20] Update Readme.md --- utilities/local-cluster/Readme.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utilities/local-cluster/Readme.md b/utilities/local-cluster/Readme.md index c76d8a14d8..a2b7b46fd5 100644 --- a/utilities/local-cluster/Readme.md +++ b/utilities/local-cluster/Readme.md @@ -29,7 +29,6 @@ * [Halt or Destroy the Existing VM](#halt-or-destroy-the-existing-vm) * [Kill the VM Process Manually (If Necessary)](#kill-the-vm-process-manually-if-necessary) * [Confirm the Process is Terminated](#confirm-the-process-is-terminated) - * [Retry bringing up the Vagrant VM](#retry-bringing-up-the-vagrant-vm) ## Requirements @@ -334,5 +333,4 @@ sudo lsof -i :6443 ``` This should return no output, indicating that no process is listening on port 6443. - -#### Retry bringing up the Vagrant VM +Retry bringing up the Vagrant VM From 43feb71729b89ab698fa430eff448e2d279312b6 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 16:49:05 +0200 Subject: [PATCH 19/20] Update project.dic --- .config/dictionaries/project.dic | 1 - 1 file changed, 1 deletion(-) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 292f0c11f4..fa5c7c1ca2 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -22,7 +22,6 @@ autoresizing backendpython bech bento -bento bimap bindgen bkioshn From 88ea242c01b94d99d8183160e1c833f836e0d347 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 9 Oct 2024 17:12:34 +0200 Subject: [PATCH 20/20] Update Vagrantfile --- utilities/local-cluster/Vagrantfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utilities/local-cluster/Vagrantfile b/utilities/local-cluster/Vagrantfile index e75102a20b..71c413a2d0 100644 --- a/utilities/local-cluster/Vagrantfile +++ b/utilities/local-cluster/Vagrantfile @@ -201,10 +201,9 @@ Vagrant.configure("2") do |config| if is_darwin_arm64 agent.vm.synced_folder "./shared", "/vagrant_shared", type: "rsync" - agent.vm.provider "qemu" do |qe| - qe.cpus = agent_vcpu - qe.memory = agent_memory - qe.gui = false + agent.vm.provider "parallels" do |p| + p.cpus = agent_vcpu + p.memory = agent_memory end else agent.vm.synced_folder "./shared", "/vagrant_shared"