From 71be058e082cd193b7d1754a25bb7adf192dac4f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:47:40 +0200 Subject: [PATCH 01/23] test: harbor operator initialization --- package.json | 2 ++ src/operator/harbor.ts | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 src/operator/harbor.ts diff --git a/package.json b/package.json index 49931958..443f8320 100644 --- a/package.json +++ b/package.json @@ -141,6 +141,8 @@ "operator:secrets": "node dist/operator/secrets.js", "operator:gitea-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/operator/gitea.ts", "operator:gitea": "node dist/operator/gitea.js", + "operator:harbor-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/operator/harbor.ts", + "operator:harbor": "node dist/operator/harbor.js", "test": "NODE_ENV=test mocha -r ts-node/register -r ts-custom-error --exit src/**/*.test.*" }, "standard-version": { diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts new file mode 100644 index 00000000..5d491827 --- /dev/null +++ b/src/operator/harbor.ts @@ -0,0 +1,51 @@ +import Operator from '@dot-i/k8s-operator' +import * as k8s from '@kubernetes/client-node' +import { KubeConfig } from '@kubernetes/client-node' + +interface groupMapping { + [key: string]: { + otomi: string[] + } +} + +const kc = new KubeConfig() +// loadFromCluster when deploying on cluster +// loadFromDefault when locally connecting to cluster +kc.loadFromCluster() +const k8sApi = kc.makeApiClient(k8s.CoreV1Api) + +export default class MyOperator extends Operator { + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types + protected async init() { + // Watch all namespaces + try { + await this.watchResource('', 'v1', 'namespaces', async (e) => { + const { object }: { object: k8s.V1Pod } = e + const { metadata } = object + // Check if namespace starts with prefix 'team-' + if (metadata && !metadata.name?.startsWith('team-')) return + if (metadata && metadata.name === 'team-admin') return + await new Promise((resolve) => setTimeout(resolve, 1000)) + console.info(`Namespace:`, metadata?.name) + }) + } catch (error) { + console.debug(error) + } + } +} + +async function main(): Promise { + const operator = new MyOperator() + console.info(`Listening to team namespace changes in all namespaces`) + await operator.start() + const exit = (reason: string) => { + operator.stop() + process.exit(0) + } + + process.on('SIGTERM', () => exit('SIGTERM')).on('SIGINT', () => exit('SIGINT')) +} + +if (typeof require !== 'undefined' && require.main === module) { + main() +} From 9224f494c2b4777473a4c1e827931d820ed01cb9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:02:59 +0200 Subject: [PATCH 02/23] test: harbor operator --- src/operator/harbor.ts | 527 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 515 insertions(+), 12 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 5d491827..d85d7d32 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -1,6 +1,36 @@ -import Operator from '@dot-i/k8s-operator' +import Operator, { ResourceEventType } from '@dot-i/k8s-operator' import * as k8s from '@kubernetes/client-node' import { KubeConfig } from '@kubernetes/client-node' +import { + ConfigureApi, + HttpBearerAuth, + MemberApi, + ProjectApi, + // eslint-disable-next-line no-unused-vars + ProjectMember, + // eslint-disable-next-line no-unused-vars + ProjectReq, + RobotApi, + // eslint-disable-next-line no-unused-vars + RobotCreate, + // eslint-disable-next-line no-unused-vars + RobotCreated, +} from '@redkubes/harbor-client-node' +import { createBuildsK8sSecret, createK8sSecret, createSecret, getSecret } from '../k8s' +import { doApiCall, handleErrors, waitTillAvailable } from '../utils' +import { + HARBOR_BASE_REPO_URL, + HARBOR_BASE_URL, + HARBOR_PASSWORD, + HARBOR_USER, + OIDC_AUTO_ONBOARD, + OIDC_CLIENT_SECRET, + OIDC_ENDPOINT, + OIDC_USER_CLAIM, + OIDC_VERIFY_CERT, + TEAM_IDS, + cleanEnv, +} from '../validators' interface groupMapping { [key: string]: { @@ -11,23 +41,70 @@ interface groupMapping { const kc = new KubeConfig() // loadFromCluster when deploying on cluster // loadFromDefault when locally connecting to cluster -kc.loadFromCluster() +if (process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) { + kc.loadFromCluster() +} else { + kc.loadFromDefault() +} const k8sApi = kc.makeApiClient(k8s.CoreV1Api) +// Callbacks +const secretsAndConfigmapsCallback = async (e: any) => { + const { object } = e + const { metadata, data } = object + + if (object.kind === 'Secret' && metadata.name === 'harbor-admin') { + console.log('Secret:', metadata.name) + console.log('Data:', data) + } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { + console.log('ConfigMap:', metadata.name) + console.log('Data:', data) + } else return + + switch (e.type) { + case ResourceEventType.Added: + case ResourceEventType.Modified: { + try { + await runSetupHarbor() + } catch (error) { + console.debug(error) + } + break + } + default: + break + } +} + +const namespacesCallback = async (e: any) => { + const { object }: { object: k8s.V1Pod } = e + const { metadata } = object + // Check if namespace starts with prefix 'team-' + if (metadata && !metadata.name?.startsWith('team-')) return + if (metadata && metadata.name === 'team-admin') return + await new Promise((resolve) => setTimeout(resolve, 1000)) + console.info(`Namespace:`, metadata?.name) +} + +// Operator export default class MyOperator extends Operator { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types protected async init() { + // Watch harbor-operator-secrets + try { + await this.watchResource('', 'v1', 'secrets', secretsAndConfigmapsCallback, 'harbor-operator') + } catch (error) { + console.debug(error) + } + // Watch harbor-operator-cm + try { + await this.watchResource('', 'v1', 'configmaps', secretsAndConfigmapsCallback, 'harbor-operator') + } catch (error) { + console.debug(error) + } // Watch all namespaces try { - await this.watchResource('', 'v1', 'namespaces', async (e) => { - const { object }: { object: k8s.V1Pod } = e - const { metadata } = object - // Check if namespace starts with prefix 'team-' - if (metadata && !metadata.name?.startsWith('team-')) return - if (metadata && metadata.name === 'team-admin') return - await new Promise((resolve) => setTimeout(resolve, 1000)) - console.info(`Namespace:`, metadata?.name) - }) + await this.watchResource('', 'v1', 'namespaces', namespacesCallback) } catch (error) { console.debug(error) } @@ -36,7 +113,7 @@ export default class MyOperator extends Operator { async function main(): Promise { const operator = new MyOperator() - console.info(`Listening to team namespace changes in all namespaces`) + console.info(`Listening to secrets, configmaps and namespaces`) await operator.start() const exit = (reason: string) => { operator.stop() @@ -49,3 +126,429 @@ async function main(): Promise { if (typeof require !== 'undefined' && require.main === module) { main() } + +// Runners +async function runSetupHarbor() { + try { + await setupHarbor() + } catch (error) { + console.debug('Error could not run setup harbor', error) + console.debug('Retrying in 30 seconds') + await new Promise((resolve) => setTimeout(resolve, 30000)) + console.debug('Retrying to setup harbor') + await runSetupHarbor() + } +} + +// Setup Harbor + +const env = cleanEnv({ + HARBOR_BASE_URL, + HARBOR_BASE_REPO_URL, + HARBOR_PASSWORD, + HARBOR_USER, + OIDC_USER_CLAIM, + OIDC_AUTO_ONBOARD, + OIDC_CLIENT_SECRET, + OIDC_ENDPOINT, + OIDC_VERIFY_CERT, + TEAM_IDS, +}) + +const HarborRole = { + admin: 1, + developer: 2, + guest: 3, + master: 4, +} + +const HarborGroupType = { + ldap: 1, + http: 2, +} + +const errors: string[] = [] + +export interface RobotSecret { + id: number + name: string + secret: string +} + +const systemRobot: any = { + name: 'harbor', + duration: -1, + description: 'Used by Otomi Harbor task runner', + disable: false, + level: 'system', + permissions: [ + { + kind: 'system', + namespace: '/', + access: [ + { + resource: '*', + action: '*', + }, + ], + }, + ], +} + +const robotPrefix = 'otomi-' +const config: any = { + auth_mode: 'oidc_auth', + oidc_admin_group: 'admin', + oidc_client_id: 'otomi', + oidc_client_secret: env.OIDC_CLIENT_SECRET, + oidc_endpoint: env.OIDC_ENDPOINT, + oidc_groups_claim: 'groups', + oidc_name: 'otomi', + oidc_scope: 'openid', + oidc_verify_cert: env.OIDC_VERIFY_CERT, + oidc_user_claim: env.OIDC_USER_CLAIM, + oidc_auto_onboard: env.OIDC_AUTO_ONBOARD, + project_creation_restriction: 'adminonly', + robot_name_prefix: robotPrefix, + self_registration: false, +} + +const systemNamespace = 'harbor' +const systemSecretName = 'harbor-robot-admin' +const projectPullSecretName = 'harbor-pullsecret' +const projectPushSecretName = 'harbor-pushsecret' +const projectBuildPushSecretName = 'harbor-pushsecret-builds' +const harborBaseUrl = `${env.HARBOR_BASE_URL}/api/v2.0` +const harborHealthUrl = `${harborBaseUrl}/systeminfo` +const robotApi = new RobotApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) +const configureApi = new ConfigureApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) +const projectsApi = new ProjectApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) +const memberApi = new MemberApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) + +/** + * Create Harbor robot account that is used by Otomi tasks + * @note assumes OIDC is not yet configured, otherwise this operation is NOT possible + */ +async function createSystemRobotSecret(): Promise { + const { body: robotList } = await robotApi.listRobot() + const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) + if (existing?.id) { + const existingId = existing.id + await doApiCall(errors, `Deleting previous robot account ${systemRobot.name}`, () => + robotApi.deleteRobot(existingId), + ) + } + const robotAccount = (await doApiCall( + errors, + `Create robot account ${systemRobot.name} with system level perms`, + () => robotApi.createRobot(systemRobot), + )) as RobotCreated + const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } + await createSecret(systemSecretName, systemNamespace, robotSecret) + return robotSecret +} + +/** + * Create Harbor system robot account that is scoped to a given Harbor project with pull access only. + * @param projectName Harbor project name + */ +async function createTeamPullRobotAccount(projectName: string): Promise { + const projectRobot: RobotCreate = { + name: `${projectName}-pull`, + duration: -1, + description: 'Allow team to pull from its own registry', + disable: false, + level: 'system', + permissions: [ + { + kind: 'project', + namespace: projectName, + access: [ + { + resource: 'repository', + action: 'pull', + }, + ], + }, + ], + } + const fullName = `${robotPrefix}${projectRobot.name}` + + const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) + const existing = robotList.find((i) => i.name === fullName) + + if (existing?.id) { + const existingId = existing.id + await doApiCall(errors, `Deleting previous pull robot account ${fullName}`, () => robotApi.deleteRobot(existingId)) + } + + const robotPullAccount = (await doApiCall( + errors, + `Creating pull robot account ${fullName} with project level perms`, + () => robotApi.createRobot(projectRobot), + )) as RobotCreated + if (!robotPullAccount?.id) { + throw new Error( + `RobotPullAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, + ) + } + return robotPullAccount +} + +/** + * Create Harbor system robot account that is scoped to a given Harbor project with push and push access + * to offer team members the option to download the kubeconfig. + * @param projectName Harbor project name + */ +async function ensureTeamPushRobotAccount(projectName: string): Promise { + const projectRobot: RobotCreate = { + name: `${projectName}-push`, + duration: -1, + description: 'Allow team to push to its own registry', + disable: false, + level: 'system', + permissions: [ + { + kind: 'project', + namespace: projectName, + access: [ + { + resource: 'repository', + action: 'push', + }, + { + resource: 'repository', + action: 'pull', + }, + ], + }, + ], + } + const fullName = `${robotPrefix}${projectRobot.name}` + + const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) + const existing = robotList.find((i) => i.name === fullName) + + if (existing?.name) { + return existing + } + + const robotPushAccount = (await doApiCall( + errors, + `Creating push robot account ${fullName} with project level perms`, + () => robotApi.createRobot(projectRobot), + )) as RobotCreated + if (!robotPushAccount?.id) { + throw new Error( + `RobotPushAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, + ) + } + return robotPushAccount +} + +/** + * Create Harbor system robot account that is scoped to a given Harbor project with push access + * for Kaniko (used for builds) task to push images. + * @param projectName Harbor project name + */ +async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { + const projectRobot: RobotCreate = { + name: `${projectName}-builds`, + duration: -1, + description: 'Allow builds to push images', + disable: false, + level: 'system', + permissions: [ + { + kind: 'project', + namespace: projectName, + access: [ + { + resource: 'repository', + action: 'push', + }, + { + resource: 'repository', + action: 'pull', + }, + ], + }, + ], + } + const fullName = `${robotPrefix}${projectRobot.name}` + + const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) + const existing = robotList.find((i) => i.name === fullName) + + if (existing?.name) { + return existing + } + + const robotBuildsPushAccount = (await doApiCall( + errors, + `Creating push robot account ${fullName} with project level perms`, + () => robotApi.createRobot(projectRobot), + )) as RobotCreated + if (!robotBuildsPushAccount?.id) { + throw new Error( + `RobotBuildsPushAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, + ) + } + return robotBuildsPushAccount +} + +/** + * Get token by reading access token from kubernetes secret. + * If the secret does not exists then create Harbor robot account and populate credentials to kubernetes secret. + */ +async function getBearerToken(): Promise { + const bearerAuth: HttpBearerAuth = new HttpBearerAuth() + + let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret + if (!robotSecret) { + // not existing yet, create robot account and keep creds in secret + robotSecret = await createSystemRobotSecret() + } else { + // test if secret still works + try { + bearerAuth.accessToken = robotSecret.secret + robotApi.setDefaultAuthentication(bearerAuth) + await robotApi.listRobot() + } catch (e) { + // throw everything except 401, which is what we test for + if (e.status !== 401) throw e + // unauthenticated, so remove and recreate secret + await k8sApi.deleteNamespacedSecret(systemSecretName, systemNamespace) + // now, the next call might throw IF: + // - authMode oidc was already turned on and an otomi admin accidentally removed the secret + // but that is very unlikely, an unresolvable problem and needs a manual db fix + robotSecret = await createSystemRobotSecret() + } + } + bearerAuth.accessToken = robotSecret.secret + return bearerAuth +} + +/** + * Ensure that Harbor robot account and corresponding Kubernetes pull secret exist + * @param namespace Kubernetes namespace where pull secret is created + * @param projectName Harbor project name + */ +async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { + const k8sSecret = await getSecret(projectPullSecretName, namespace) + if (k8sSecret) { + console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) + await k8sApi.deleteNamespacedSecret(projectPullSecretName, namespace) + } + + const robotPullAccount = await createTeamPullRobotAccount(projectName) + console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) + await createK8sSecret({ + namespace, + name: projectPullSecretName, + server: `${env.HARBOR_BASE_REPO_URL}`, + username: robotPullAccount.name!, + password: robotPullAccount.secret!, + }) +} + +/** + * Ensure that Harbor robot account and corresponding Kubernetes push secret exist + * @param namespace Kubernetes namespace where push secret is created + * @param projectName Harbor project name + */ +async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): Promise { + const k8sSecret = await getSecret(projectPushSecretName, namespace) + if (!k8sSecret) { + const robotPushAccount = await ensureTeamPushRobotAccount(projectName) + console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) + await createK8sSecret({ + namespace, + name: projectPushSecretName, + server: `${env.HARBOR_BASE_REPO_URL}`, + username: robotPushAccount.name!, + password: robotPushAccount.secret!, + }) + } +} + +/** + * Ensure that Harbor robot account and corresponding Kubernetes push secret for builds exist + * @param namespace Kubernetes namespace where push secret is created + * @param projectName Harbor project name + */ +async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { + const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) + if (!k8sSecret) { + const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) + console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) + await createBuildsK8sSecret({ + namespace, + name: projectBuildPushSecretName, + server: `${env.HARBOR_BASE_REPO_URL}`, + username: robotBuildsPushAccount.name!, + password: robotBuildsPushAccount.secret!, + }) + } +} + +async function setupHarbor() { + // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed + await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) + const bearerAuth = await getBearerToken() + robotApi.setDefaultAuthentication(bearerAuth) + configureApi.setDefaultAuthentication(bearerAuth) + projectsApi.setDefaultAuthentication(bearerAuth) + memberApi.setDefaultAuthentication(bearerAuth) + + await doApiCall(errors, 'Putting Harbor configuration', () => configureApi.configurationsPut(config)) + await Promise.all( + env.TEAM_IDS.map(async (teamId: string) => { + const projectName = `team-${teamId}` + const teamNamespce = projectName + const projectReq: ProjectReq = { + projectName, + } + await doApiCall(errors, `Creating project for team ${teamId}`, () => projectsApi.createProject(projectReq)) + + const project = await doApiCall(errors, `Get project for team ${teamId}`, () => + projectsApi.getProject(projectName), + ) + if (!project) return '' + const projectId = `${project.projectId}` + + const projMember: ProjectMember = { + roleId: HarborRole.developer, + memberGroup: { + groupName: projectName, + groupType: HarborGroupType.http, + }, + } + const projAdminMember: ProjectMember = { + roleId: HarborRole.admin, + memberGroup: { + groupName: 'team-admin', + groupType: HarborGroupType.http, + }, + } + await doApiCall( + errors, + `Associating "developer" role for team "${teamId}" with harbor project "${projectName}"`, + () => memberApi.createProjectMember(projectId, undefined, undefined, projMember), + ) + await doApiCall( + errors, + `Associating "project-admin" role for "team-admin" with harbor project "${projectName}"`, + () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), + ) + + await ensureTeamPullRobotAccountSecret(teamNamespce, projectName) + await ensureTeamPushRobotAccountSecret(teamNamespce, projectName) + await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName) + + return null + }), + ) + + handleErrors(errors) +} From 621dc1b29414305fbc9ea1fbf792bac812f31ada Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:32:57 +0200 Subject: [PATCH 03/23] test: harbor operator --- src/operator/harbor.ts | 204 +++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 100 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index d85d7d32..fce19326 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -18,19 +18,6 @@ import { } from '@redkubes/harbor-client-node' import { createBuildsK8sSecret, createK8sSecret, createSecret, getSecret } from '../k8s' import { doApiCall, handleErrors, waitTillAvailable } from '../utils' -import { - HARBOR_BASE_REPO_URL, - HARBOR_BASE_URL, - HARBOR_PASSWORD, - HARBOR_USER, - OIDC_AUTO_ONBOARD, - OIDC_CLIENT_SECRET, - OIDC_ENDPOINT, - OIDC_USER_CLAIM, - OIDC_VERIFY_CERT, - TEAM_IDS, - cleanEnv, -} from '../validators' interface groupMapping { [key: string]: { @@ -38,6 +25,92 @@ interface groupMapping { } } +const HarborRole = { + admin: 1, + developer: 2, + guest: 3, + master: 4, +} + +const HarborGroupType = { + ldap: 1, + http: 2, +} + +const errors: string[] = [] + +export interface RobotSecret { + id: number + name: string + secret: string +} + +const systemRobot: any = { + name: 'harbor', + duration: -1, + description: 'Used by Otomi Harbor task runner', + disable: false, + level: 'system', + permissions: [ + { + kind: 'system', + namespace: '/', + access: [ + { + resource: '*', + action: '*', + }, + ], + }, + ], +} + +const robotPrefix = 'otomi-' +const harborOperator = { + harborBaseUrl: '', + harborBaseRepoUrl: '', + harborUser: '', + harborPassword: '', + teamIds: [], + oidcClientId: '', + oidcClientSecret: '', + oidcEndpoint: '', + oidcVerifyCert: true, + oidcUserClaim: 'email', + oidcAutoOnboard: true, + oidcGroupsClaim: 'groups', + oidcName: 'keycloak', + oidcScope: 'openid', +} +const config: any = { + auth_mode: 'oidc_auth', + oidc_admin_group: 'admin', + oidc_client_id: 'otomi', + oidc_client_secret: harborOperator.oidcClientSecret, + oidc_endpoint: harborOperator.oidcEndpoint, + oidc_groups_claim: 'groups', + oidc_name: 'otomi', + oidc_scope: 'openid', + oidc_verify_cert: harborOperator.oidcVerifyCert, + oidc_user_claim: harborOperator.oidcUserClaim, + oidc_auto_onboard: harborOperator.oidcAutoOnboard, + project_creation_restriction: 'adminonly', + robot_name_prefix: robotPrefix, + self_registration: false, +} + +const systemNamespace = 'harbor' +const systemSecretName = 'harbor-robot-admin' +const projectPullSecretName = 'harbor-pullsecret' +const projectPushSecretName = 'harbor-pushsecret' +const projectBuildPushSecretName = 'harbor-pushsecret-builds' +const harborBaseUrl = `${harborOperator.harborBaseUrl}/api/v2.0` +const harborHealthUrl = `${harborBaseUrl}/systeminfo` +const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) +const configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) +const projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) +const memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + const kc = new KubeConfig() // loadFromCluster when deploying on cluster // loadFromDefault when locally connecting to cluster @@ -56,9 +129,23 @@ const secretsAndConfigmapsCallback = async (e: any) => { if (object.kind === 'Secret' && metadata.name === 'harbor-admin') { console.log('Secret:', metadata.name) console.log('Data:', data) + harborOperator.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() + harborOperator.harborUser = Buffer.from(data.harborUser, 'base64').toString() + harborOperator.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() + harborOperator.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() + harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { console.log('ConfigMap:', metadata.name) console.log('Data:', data) + harborOperator.harborBaseUrl = data.harborBaseUrl + harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl + harborOperator.teamIds = JSON.parse(data.teamIds) + harborOperator.oidcAutoOnboard = data.oidcAutoOnboard + harborOperator.oidcUserClaim = data.oidcUserClaim + harborOperator.oidcGroupsClaim = data.oidcGroupsClaim + harborOperator.oidcName = data.oidcName + harborOperator.oidcScope = data.oidcScope + harborOperator.oidcVerifyCert = data.oidcVerifyCert } else return switch (e.type) { @@ -142,89 +229,6 @@ async function runSetupHarbor() { // Setup Harbor -const env = cleanEnv({ - HARBOR_BASE_URL, - HARBOR_BASE_REPO_URL, - HARBOR_PASSWORD, - HARBOR_USER, - OIDC_USER_CLAIM, - OIDC_AUTO_ONBOARD, - OIDC_CLIENT_SECRET, - OIDC_ENDPOINT, - OIDC_VERIFY_CERT, - TEAM_IDS, -}) - -const HarborRole = { - admin: 1, - developer: 2, - guest: 3, - master: 4, -} - -const HarborGroupType = { - ldap: 1, - http: 2, -} - -const errors: string[] = [] - -export interface RobotSecret { - id: number - name: string - secret: string -} - -const systemRobot: any = { - name: 'harbor', - duration: -1, - description: 'Used by Otomi Harbor task runner', - disable: false, - level: 'system', - permissions: [ - { - kind: 'system', - namespace: '/', - access: [ - { - resource: '*', - action: '*', - }, - ], - }, - ], -} - -const robotPrefix = 'otomi-' -const config: any = { - auth_mode: 'oidc_auth', - oidc_admin_group: 'admin', - oidc_client_id: 'otomi', - oidc_client_secret: env.OIDC_CLIENT_SECRET, - oidc_endpoint: env.OIDC_ENDPOINT, - oidc_groups_claim: 'groups', - oidc_name: 'otomi', - oidc_scope: 'openid', - oidc_verify_cert: env.OIDC_VERIFY_CERT, - oidc_user_claim: env.OIDC_USER_CLAIM, - oidc_auto_onboard: env.OIDC_AUTO_ONBOARD, - project_creation_restriction: 'adminonly', - robot_name_prefix: robotPrefix, - self_registration: false, -} - -const systemNamespace = 'harbor' -const systemSecretName = 'harbor-robot-admin' -const projectPullSecretName = 'harbor-pullsecret' -const projectPushSecretName = 'harbor-pushsecret' -const projectBuildPushSecretName = 'harbor-pushsecret-builds' -const harborBaseUrl = `${env.HARBOR_BASE_URL}/api/v2.0` -const harborHealthUrl = `${harborBaseUrl}/systeminfo` -const robotApi = new RobotApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const configureApi = new ConfigureApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const projectsApi = new ProjectApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const memberApi = new MemberApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) - /** * Create Harbor robot account that is used by Otomi tasks * @note assumes OIDC is not yet configured, otherwise this operation is NOT possible @@ -446,7 +450,7 @@ async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): await createK8sSecret({ namespace, name: projectPullSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, + server: `${harborOperator.harborBaseRepoUrl}`, username: robotPullAccount.name!, password: robotPullAccount.secret!, }) @@ -465,7 +469,7 @@ async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): await createK8sSecret({ namespace, name: projectPushSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, + server: `${harborOperator.harborBaseRepoUrl}`, username: robotPushAccount.name!, password: robotPushAccount.secret!, }) @@ -485,7 +489,7 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN await createBuildsK8sSecret({ namespace, name: projectBuildPushSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, + server: `${harborOperator.harborBaseRepoUrl}`, username: robotBuildsPushAccount.name!, password: robotBuildsPushAccount.secret!, }) @@ -503,7 +507,7 @@ async function setupHarbor() { await doApiCall(errors, 'Putting Harbor configuration', () => configureApi.configurationsPut(config)) await Promise.all( - env.TEAM_IDS.map(async (teamId: string) => { + harborOperator.teamIds.map(async (teamId: string) => { const projectName = `team-${teamId}` const teamNamespce = projectName const projectReq: ProjectReq = { From 37a08a875281f8694a85966de4fb6629691b11e1 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:50:21 +0200 Subject: [PATCH 04/23] test: harbor operator --- src/operator/harbor.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index fce19326..de14e2ab 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -82,6 +82,7 @@ const harborOperator = { oidcName: 'keycloak', oidcScope: 'openid', } +console.log('harborOperator', harborOperator) const config: any = { auth_mode: 'oidc_auth', oidc_admin_group: 'admin', @@ -99,13 +100,12 @@ const config: any = { self_registration: false, } -const systemNamespace = 'harbor' +const systemNamespace = 'harbor-operator' const systemSecretName = 'harbor-robot-admin' const projectPullSecretName = 'harbor-pullsecret' const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' const harborBaseUrl = `${harborOperator.harborBaseUrl}/api/v2.0` -const harborHealthUrl = `${harborBaseUrl}/systeminfo` const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) const configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) const projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) @@ -128,7 +128,6 @@ const secretsAndConfigmapsCallback = async (e: any) => { if (object.kind === 'Secret' && metadata.name === 'harbor-admin') { console.log('Secret:', metadata.name) - console.log('Data:', data) harborOperator.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() harborOperator.harborUser = Buffer.from(data.harborUser, 'base64').toString() harborOperator.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() @@ -136,7 +135,6 @@ const secretsAndConfigmapsCallback = async (e: any) => { harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { console.log('ConfigMap:', metadata.name) - console.log('Data:', data) harborOperator.harborBaseUrl = data.harborBaseUrl harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl harborOperator.teamIds = JSON.parse(data.teamIds) @@ -497,6 +495,11 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN } async function setupHarbor() { + console.log('setupHarbor') + if (!harborOperator.harborBaseUrl) return + console.log('harborOperator', harborOperator) + const harborHealthUrl = `${harborOperator.harborBaseUrl}/api/v2.0/systeminfo` + console.log('harborHealthUrl', harborHealthUrl) // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) const bearerAuth = await getBearerToken() From 0d340e5db266ff6c9a8225ad10540dba2d78c2d4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:22:14 +0200 Subject: [PATCH 05/23] test: harbor operator --- src/operator/harbor.ts | 51 +++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index de14e2ab..1f2a2aa5 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -82,7 +82,6 @@ const harborOperator = { oidcName: 'keycloak', oidcScope: 'openid', } -console.log('harborOperator', harborOperator) const config: any = { auth_mode: 'oidc_auth', oidc_admin_group: 'admin', @@ -106,7 +105,7 @@ const projectPullSecretName = 'harbor-pullsecret' const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' const harborBaseUrl = `${harborOperator.harborBaseUrl}/api/v2.0` -const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + const configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) const projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) const memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) @@ -231,8 +230,10 @@ async function runSetupHarbor() { * Create Harbor robot account that is used by Otomi tasks * @note assumes OIDC is not yet configured, otherwise this operation is NOT possible */ -async function createSystemRobotSecret(): Promise { +async function createSystemRobotSecret(robotApi): Promise { + console.log('ROBOT 1') const { body: robotList } = await robotApi.listRobot() + console.log('ROBOT 2') const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) if (existing?.id) { const existingId = existing.id @@ -245,6 +246,8 @@ async function createSystemRobotSecret(): Promise { `Create robot account ${systemRobot.name} with system level perms`, () => robotApi.createRobot(systemRobot), )) as RobotCreated + console.log('errors', errors) + console.log('robotAccount', robotAccount) const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } await createSecret(systemSecretName, systemNamespace, robotSecret) return robotSecret @@ -254,7 +257,7 @@ async function createSystemRobotSecret(): Promise { * Create Harbor system robot account that is scoped to a given Harbor project with pull access only. * @param projectName Harbor project name */ -async function createTeamPullRobotAccount(projectName: string): Promise { +async function createTeamPullRobotAccount(projectName: string, robotApi): Promise { const projectRobot: RobotCreate = { name: `${projectName}-pull`, duration: -1, @@ -302,7 +305,7 @@ async function createTeamPullRobotAccount(projectName: string): Promise { +async function ensureTeamPushRobotAccount(projectName: string, robotApi): Promise { const projectRobot: RobotCreate = { name: `${projectName}-push`, duration: -1, @@ -353,7 +356,7 @@ async function ensureTeamPushRobotAccount(projectName: string): Promise { * for Kaniko (used for builds) task to push images. * @param projectName Harbor project name */ -async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { +async function ensureTeamBuildsPushRobotAccount(projectName: string, robotApi): Promise { const projectRobot: RobotCreate = { name: `${projectName}-builds`, duration: -1, @@ -403,13 +406,13 @@ async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { +async function getBearerToken(robotApi): Promise { const bearerAuth: HttpBearerAuth = new HttpBearerAuth() let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret if (!robotSecret) { // not existing yet, create robot account and keep creds in secret - robotSecret = await createSystemRobotSecret() + robotSecret = await createSystemRobotSecret(robotApi) } else { // test if secret still works try { @@ -424,7 +427,7 @@ async function getBearerToken(): Promise { // now, the next call might throw IF: // - authMode oidc was already turned on and an otomi admin accidentally removed the secret // but that is very unlikely, an unresolvable problem and needs a manual db fix - robotSecret = await createSystemRobotSecret() + robotSecret = await createSystemRobotSecret(robotApi) } } bearerAuth.accessToken = robotSecret.secret @@ -436,14 +439,14 @@ async function getBearerToken(): Promise { * @param namespace Kubernetes namespace where pull secret is created * @param projectName Harbor project name */ -async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { +async function ensureTeamPullRobotAccountSecret(namespace: string, projectName, robotApi): Promise { const k8sSecret = await getSecret(projectPullSecretName, namespace) if (k8sSecret) { console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) await k8sApi.deleteNamespacedSecret(projectPullSecretName, namespace) } - const robotPullAccount = await createTeamPullRobotAccount(projectName) + const robotPullAccount = await createTeamPullRobotAccount(projectName, robotApi) console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) await createK8sSecret({ namespace, @@ -459,10 +462,10 @@ async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): * @param namespace Kubernetes namespace where push secret is created * @param projectName Harbor project name */ -async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): Promise { +async function ensureTeamPushRobotAccountSecret(namespace: string, projectName, robotApi): Promise { const k8sSecret = await getSecret(projectPushSecretName, namespace) if (!k8sSecret) { - const robotPushAccount = await ensureTeamPushRobotAccount(projectName) + const robotPushAccount = await ensureTeamPushRobotAccount(projectName, robotApi) console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) await createK8sSecret({ namespace, @@ -479,10 +482,10 @@ async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): * @param namespace Kubernetes namespace where push secret is created * @param projectName Harbor project name */ -async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { +async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName, robotApi): Promise { const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) if (!k8sSecret) { - const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) + const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName, robotApi) console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) await createBuildsK8sSecret({ namespace, @@ -495,14 +498,16 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN } async function setupHarbor() { - console.log('setupHarbor') - if (!harborOperator.harborBaseUrl) return console.log('harborOperator', harborOperator) - const harborHealthUrl = `${harborOperator.harborBaseUrl}/api/v2.0/systeminfo` - console.log('harborHealthUrl', harborHealthUrl) + if (!harborOperator.harborBaseUrl || !harborOperator.harborUser || !harborOperator.harborPassword) return + const harborUrl = `${harborOperator.harborBaseUrl}/api/v2.0` + const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborUrl) + console.log('robotApi', robotApi) + const harborHealthUrl = `https://${harborOperator.harborBaseRepoUrl}/api/v2.0/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) - const bearerAuth = await getBearerToken() + const bearerAuth = await getBearerToken(robotApi) + console.log('bearerAuth', bearerAuth) robotApi.setDefaultAuthentication(bearerAuth) configureApi.setDefaultAuthentication(bearerAuth) projectsApi.setDefaultAuthentication(bearerAuth) @@ -549,9 +554,9 @@ async function setupHarbor() { () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), ) - await ensureTeamPullRobotAccountSecret(teamNamespce, projectName) - await ensureTeamPushRobotAccountSecret(teamNamespce, projectName) - await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName) + await ensureTeamPullRobotAccountSecret(teamNamespce, projectName, robotApi) + await ensureTeamPushRobotAccountSecret(teamNamespce, projectName, robotApi) + await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName, robotApi) return null }), From 734e7ac71a10c4fc032cf41d3cd58f647b87e450 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:27:04 +0200 Subject: [PATCH 06/23] test: harbor operator --- src/operator/harbor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 1f2a2aa5..6cf5e2cf 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -503,7 +503,7 @@ async function setupHarbor() { const harborUrl = `${harborOperator.harborBaseUrl}/api/v2.0` const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborUrl) console.log('robotApi', robotApi) - const harborHealthUrl = `https://${harborOperator.harborBaseRepoUrl}/api/v2.0/systeminfo` + const harborHealthUrl = `${harborUrl}/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) const bearerAuth = await getBearerToken(robotApi) From 24c793de6e9b44964610108d8a11a511a2bf104d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 23:48:56 +0200 Subject: [PATCH 07/23] test: harbor operator --- src/operator/harbor.ts | 76 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 6cf5e2cf..4cd65d8c 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -67,10 +67,10 @@ const systemRobot: any = { const robotPrefix = 'otomi-' const harborOperator = { - harborBaseUrl: '', - harborBaseRepoUrl: '', - harborUser: '', - harborPassword: '', + harborBaseUrl: 'http://harbor-core.harbor', + harborBaseRepoUrl: 'harbor.172.233.37.26.nip.io', + harborUser: 'admin', + harborPassword: 'welcomeotomi', teamIds: [], oidcClientId: '', oidcClientSecret: '', @@ -99,16 +99,25 @@ const config: any = { self_registration: false, } -const systemNamespace = 'harbor-operator' +const systemNamespace = 'harbor' const systemSecretName = 'harbor-robot-admin' const projectPullSecretName = 'harbor-pullsecret' const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' -const harborBaseUrl = `${harborOperator.harborBaseUrl}/api/v2.0` - -const configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) -const projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) -const memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) +// const harborBaseUrl = `https://harbor.172.233.37.26.nip.io/api/v2.0` +// const harborHealthUrl = `${harborBaseUrl}/systeminfo` +const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborOperator.harborBaseUrl) +const configureApi = new ConfigureApi( + harborOperator.harborUser, + harborOperator.harborPassword, + harborOperator.harborBaseUrl, +) +const projectsApi = new ProjectApi( + harborOperator.harborUser, + harborOperator.harborPassword, + harborOperator.harborBaseUrl, +) +const memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborOperator.harborBaseUrl) const kc = new KubeConfig() // loadFromCluster when deploying on cluster @@ -230,10 +239,9 @@ async function runSetupHarbor() { * Create Harbor robot account that is used by Otomi tasks * @note assumes OIDC is not yet configured, otherwise this operation is NOT possible */ -async function createSystemRobotSecret(robotApi): Promise { - console.log('ROBOT 1') +async function createSystemRobotSecret(): Promise { const { body: robotList } = await robotApi.listRobot() - console.log('ROBOT 2') + console.log('robotApi', robotApi) const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) if (existing?.id) { const existingId = existing.id @@ -246,8 +254,8 @@ async function createSystemRobotSecret(robotApi): Promise { `Create robot account ${systemRobot.name} with system level perms`, () => robotApi.createRobot(systemRobot), )) as RobotCreated - console.log('errors', errors) console.log('robotAccount', robotAccount) + console.log('errors', errors) const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } await createSecret(systemSecretName, systemNamespace, robotSecret) return robotSecret @@ -257,7 +265,7 @@ async function createSystemRobotSecret(robotApi): Promise { * Create Harbor system robot account that is scoped to a given Harbor project with pull access only. * @param projectName Harbor project name */ -async function createTeamPullRobotAccount(projectName: string, robotApi): Promise { +async function createTeamPullRobotAccount(projectName: string): Promise { const projectRobot: RobotCreate = { name: `${projectName}-pull`, duration: -1, @@ -305,7 +313,7 @@ async function createTeamPullRobotAccount(projectName: string, robotApi): Promis * to offer team members the option to download the kubeconfig. * @param projectName Harbor project name */ -async function ensureTeamPushRobotAccount(projectName: string, robotApi): Promise { +async function ensureTeamPushRobotAccount(projectName: string): Promise { const projectRobot: RobotCreate = { name: `${projectName}-push`, duration: -1, @@ -356,7 +364,7 @@ async function ensureTeamPushRobotAccount(projectName: string, robotApi): Promis * for Kaniko (used for builds) task to push images. * @param projectName Harbor project name */ -async function ensureTeamBuildsPushRobotAccount(projectName: string, robotApi): Promise { +async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { const projectRobot: RobotCreate = { name: `${projectName}-builds`, duration: -1, @@ -406,13 +414,13 @@ async function ensureTeamBuildsPushRobotAccount(projectName: string, robotApi): * Get token by reading access token from kubernetes secret. * If the secret does not exists then create Harbor robot account and populate credentials to kubernetes secret. */ -async function getBearerToken(robotApi): Promise { +async function getBearerToken(): Promise { const bearerAuth: HttpBearerAuth = new HttpBearerAuth() let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret if (!robotSecret) { // not existing yet, create robot account and keep creds in secret - robotSecret = await createSystemRobotSecret(robotApi) + robotSecret = await createSystemRobotSecret() } else { // test if secret still works try { @@ -427,7 +435,7 @@ async function getBearerToken(robotApi): Promise { // now, the next call might throw IF: // - authMode oidc was already turned on and an otomi admin accidentally removed the secret // but that is very unlikely, an unresolvable problem and needs a manual db fix - robotSecret = await createSystemRobotSecret(robotApi) + robotSecret = await createSystemRobotSecret() } } bearerAuth.accessToken = robotSecret.secret @@ -439,14 +447,14 @@ async function getBearerToken(robotApi): Promise { * @param namespace Kubernetes namespace where pull secret is created * @param projectName Harbor project name */ -async function ensureTeamPullRobotAccountSecret(namespace: string, projectName, robotApi): Promise { +async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { const k8sSecret = await getSecret(projectPullSecretName, namespace) if (k8sSecret) { console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) await k8sApi.deleteNamespacedSecret(projectPullSecretName, namespace) } - const robotPullAccount = await createTeamPullRobotAccount(projectName, robotApi) + const robotPullAccount = await createTeamPullRobotAccount(projectName) console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) await createK8sSecret({ namespace, @@ -462,10 +470,10 @@ async function ensureTeamPullRobotAccountSecret(namespace: string, projectName, * @param namespace Kubernetes namespace where push secret is created * @param projectName Harbor project name */ -async function ensureTeamPushRobotAccountSecret(namespace: string, projectName, robotApi): Promise { +async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): Promise { const k8sSecret = await getSecret(projectPushSecretName, namespace) if (!k8sSecret) { - const robotPushAccount = await ensureTeamPushRobotAccount(projectName, robotApi) + const robotPushAccount = await ensureTeamPushRobotAccount(projectName) console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) await createK8sSecret({ namespace, @@ -482,10 +490,10 @@ async function ensureTeamPushRobotAccountSecret(namespace: string, projectName, * @param namespace Kubernetes namespace where push secret is created * @param projectName Harbor project name */ -async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName, robotApi): Promise { +async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) if (!k8sSecret) { - const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName, robotApi) + const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) await createBuildsK8sSecret({ namespace, @@ -499,15 +507,11 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN async function setupHarbor() { console.log('harborOperator', harborOperator) - if (!harborOperator.harborBaseUrl || !harborOperator.harborUser || !harborOperator.harborPassword) return - const harborUrl = `${harborOperator.harborBaseUrl}/api/v2.0` - const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborUrl) - console.log('robotApi', robotApi) - const harborHealthUrl = `${harborUrl}/systeminfo` + if (!harborOperator.harborBaseUrl) return + const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) - const bearerAuth = await getBearerToken(robotApi) - console.log('bearerAuth', bearerAuth) + const bearerAuth = await getBearerToken() robotApi.setDefaultAuthentication(bearerAuth) configureApi.setDefaultAuthentication(bearerAuth) projectsApi.setDefaultAuthentication(bearerAuth) @@ -554,9 +558,9 @@ async function setupHarbor() { () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), ) - await ensureTeamPullRobotAccountSecret(teamNamespce, projectName, robotApi) - await ensureTeamPushRobotAccountSecret(teamNamespce, projectName, robotApi) - await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName, robotApi) + await ensureTeamPullRobotAccountSecret(teamNamespce, projectName) + await ensureTeamPushRobotAccountSecret(teamNamespce, projectName) + await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName) return null }), From 32d2388459dc22362317479e94dc641458dbd9d6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Jun 2024 23:53:42 +0200 Subject: [PATCH 08/23] test: harbor operator --- src/operator/harbor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 4cd65d8c..1059e205 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -508,7 +508,7 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN async function setupHarbor() { console.log('harborOperator', harborOperator) if (!harborOperator.harborBaseUrl) return - const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` + const harborHealthUrl = `${harborOperator.harborBaseUrl}/api/v2.0/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) const bearerAuth = await getBearerToken() From 14239e84d8e6d2b11f14bf7a78591efb04601515 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:20:53 +0200 Subject: [PATCH 09/23] test: harbor task --- src/tasks/harbor/harbor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tasks/harbor/harbor.ts b/src/tasks/harbor/harbor.ts index ef7246ac..9e2eafb8 100644 --- a/src/tasks/harbor/harbor.ts +++ b/src/tasks/harbor/harbor.ts @@ -111,6 +111,7 @@ const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' const harborBaseUrl = `${env.HARBOR_BASE_URL}/api/v2.0` const harborHealthUrl = `${harborBaseUrl}/systeminfo` +console.log('harborHealthUrl', harborHealthUrl) const robotApi = new RobotApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) const configureApi = new ConfigureApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) const projectsApi = new ProjectApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) @@ -122,6 +123,7 @@ const memberApi = new MemberApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBase */ async function createSystemRobotSecret(): Promise { const { body: robotList } = await robotApi.listRobot() + console.log('robotApi', robotApi) const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) if (existing?.id) { const existingId = existing.id @@ -134,6 +136,8 @@ async function createSystemRobotSecret(): Promise { `Create robot account ${systemRobot.name} with system level perms`, () => robotApi.createRobot(systemRobot), )) as RobotCreated + console.log('errors', errors) + console.log('robotAccount', robotAccount) const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } await createSecret(systemSecretName, systemNamespace, robotSecret) return robotSecret From 03c41715d3df05da0f30e5436235836f9278d6ac Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 11:47:19 +0200 Subject: [PATCH 10/23] test: harbor operator --- src/operator/harbor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 1059e205..6ed5f8d7 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -68,7 +68,7 @@ const systemRobot: any = { const robotPrefix = 'otomi-' const harborOperator = { harborBaseUrl: 'http://harbor-core.harbor', - harborBaseRepoUrl: 'harbor.172.233.37.26.nip.io', + harborBaseRepoUrl: '', harborUser: 'admin', harborPassword: 'welcomeotomi', teamIds: [], @@ -143,7 +143,7 @@ const secretsAndConfigmapsCallback = async (e: any) => { harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { console.log('ConfigMap:', metadata.name) - harborOperator.harborBaseUrl = data.harborBaseUrl + // harborOperator.harborBaseUrl = data.harborBaseUrl harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl harborOperator.teamIds = JSON.parse(data.teamIds) harborOperator.oidcAutoOnboard = data.oidcAutoOnboard From 08707ed1419145e2b7b20bb4ca5a07e516b47ead Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:58:38 +0200 Subject: [PATCH 11/23] test: harbor operator --- src/operator/harbor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 6ed5f8d7..78d44a1f 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -67,7 +67,7 @@ const systemRobot: any = { const robotPrefix = 'otomi-' const harborOperator = { - harborBaseUrl: 'http://harbor-core.harbor', + harborBaseUrl: 'http://harbor-core.harbor:8083/api/v2.0', harborBaseRepoUrl: '', harborUser: 'admin', harborPassword: 'welcomeotomi', @@ -508,7 +508,7 @@ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectN async function setupHarbor() { console.log('harborOperator', harborOperator) if (!harborOperator.harborBaseUrl) return - const harborHealthUrl = `${harborOperator.harborBaseUrl}/api/v2.0/systeminfo` + const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) const bearerAuth = await getBearerToken() From 4e49e11111e1cdc3e8a37c176bb9f317ab75f24a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:20:38 +0200 Subject: [PATCH 12/23] test: harbor operator port forwarding --- src/operator/harbor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 78d44a1f..ba08fd6f 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -67,7 +67,7 @@ const systemRobot: any = { const robotPrefix = 'otomi-' const harborOperator = { - harborBaseUrl: 'http://harbor-core.harbor:8083/api/v2.0', + harborBaseUrl: 'http://harbor-core.harbor:80/api/v2.0', harborBaseRepoUrl: '', harborUser: 'admin', harborPassword: 'welcomeotomi', From c397adc609ec4181e89e25e7ae84027de27b92e6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:13:18 +0200 Subject: [PATCH 13/23] test: harbor operator --- src/operator/harbor.ts | 342 +++++++++++++++++++++-------------------- 1 file changed, 178 insertions(+), 164 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index ba08fd6f..076424f2 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -172,11 +172,10 @@ const secretsAndConfigmapsCallback = async (e: any) => { const namespacesCallback = async (e: any) => { const { object }: { object: k8s.V1Pod } = e const { metadata } = object - // Check if namespace starts with prefix 'team-' - if (metadata && !metadata.name?.startsWith('team-')) return - if (metadata && metadata.name === 'team-admin') return - await new Promise((resolve) => setTimeout(resolve, 1000)) - console.info(`Namespace:`, metadata?.name) + if (metadata && metadata.name?.startsWith('team-')) { + console.info(`Processing Namespace:`, metadata.name) + await runProcessNamespace(metadata.name) + } } // Operator @@ -233,7 +232,66 @@ async function runSetupHarbor() { } } +async function runProcessNamespace(namespace: string) { + try { + await processNamespace(namespace) + } catch (error) { + console.debug('Error could not process namespace', error) + console.debug('Retrying in 30 seconds') + await new Promise((resolve) => setTimeout(resolve, 30000)) + console.debug('Retrying to process namespace') + await runProcessNamespace(namespace) + } +} + // Setup Harbor +async function setupHarbor() { + console.log('harborOperator', harborOperator) + if (!harborOperator.harborBaseUrl) return + const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` + // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed + await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) + const bearerAuth = await getBearerToken() + robotApi.setDefaultAuthentication(bearerAuth) + configureApi.setDefaultAuthentication(bearerAuth) + projectsApi.setDefaultAuthentication(bearerAuth) + memberApi.setDefaultAuthentication(bearerAuth) + + await doApiCall(errors, 'Putting Harbor configuration', () => configureApi.configurationsPut(config)) + handleErrors(errors) +} + +/** + * Get token by reading access token from kubernetes secret. + * If the secret does not exists then create Harbor robot account and populate credentials to kubernetes secret. + */ +async function getBearerToken(): Promise { + const bearerAuth: HttpBearerAuth = new HttpBearerAuth() + + let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret + if (!robotSecret) { + // not existing yet, create robot account and keep creds in secret + robotSecret = await createSystemRobotSecret() + } else { + // test if secret still works + try { + bearerAuth.accessToken = robotSecret.secret + robotApi.setDefaultAuthentication(bearerAuth) + await robotApi.listRobot() + } catch (e) { + // throw everything except 401, which is what we test for + if (e.status !== 401) throw e + // unauthenticated, so remove and recreate secret + await k8sApi.deleteNamespacedSecret(systemSecretName, systemNamespace) + // now, the next call might throw IF: + // - authMode oidc was already turned on and an otomi admin accidentally removed the secret + // but that is very unlikely, an unresolvable problem and needs a manual db fix + robotSecret = await createSystemRobotSecret() + } + } + bearerAuth.accessToken = robotSecret.secret + return bearerAuth +} /** * Create Harbor robot account that is used by Otomi tasks @@ -261,6 +319,81 @@ async function createSystemRobotSecret(): Promise { return robotSecret } +// Process Namespace +async function processNamespace(namespace: string) { + try { + const projectName = namespace + const projectReq: ProjectReq = { + projectName, + } + await doApiCall(errors, `Creating project for team ${namespace}`, () => projectsApi.createProject(projectReq)) + + const project = await doApiCall(errors, `Get project for team ${namespace}`, () => + projectsApi.getProject(projectName), + ) + if (!project) return '' + const projectId = `${project.projectId}` + + const projMember: ProjectMember = { + roleId: HarborRole.developer, + memberGroup: { + groupName: projectName, + groupType: HarborGroupType.http, + }, + } + const projAdminMember: ProjectMember = { + roleId: HarborRole.admin, + memberGroup: { + groupName: 'team-admin', + groupType: HarborGroupType.http, + }, + } + await doApiCall( + errors, + `Associating "developer" role for team "${namespace}" with harbor project "${projectName}"`, + () => memberApi.createProjectMember(projectId, undefined, undefined, projMember), + ) + await doApiCall( + errors, + `Associating "project-admin" role for "team-admin" with harbor project "${projectName}"`, + () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), + ) + + await ensureTeamPullRobotAccountSecret(namespace, projectName) + await ensureTeamPushRobotAccountSecret(namespace, projectName) + await ensureTeamBuildPushRobotAccountSecret(namespace, projectName) + + console.info(`Successfully processed namespace: ${namespace}`) + return null + } catch (error) { + console.error(`Error processing namespace ${namespace}:`, error) + return null + } +} + +/** + * Ensure that Harbor robot account and corresponding Kubernetes pull secret exist + * @param namespace Kubernetes namespace where pull secret is created + * @param projectName Harbor project name + */ +async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { + const k8sSecret = await getSecret(projectPullSecretName, namespace) + if (k8sSecret) { + console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) + await k8sApi.deleteNamespacedSecret(projectPullSecretName, namespace) + } + + const robotPullAccount = await createTeamPullRobotAccount(projectName) + console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) + await createK8sSecret({ + namespace, + name: projectPullSecretName, + server: `${harborOperator.harborBaseRepoUrl}`, + username: robotPullAccount.name!, + password: robotPullAccount.secret!, + }) +} + /** * Create Harbor system robot account that is scoped to a given Harbor project with pull access only. * @param projectName Harbor project name @@ -308,6 +441,26 @@ async function createTeamPullRobotAccount(projectName: string): Promise { + const k8sSecret = await getSecret(projectPushSecretName, namespace) + if (!k8sSecret) { + const robotPushAccount = await ensureTeamPushRobotAccount(projectName) + console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) + await createK8sSecret({ + namespace, + name: projectPushSecretName, + server: `${harborOperator.harborBaseRepoUrl}`, + username: robotPushAccount.name!, + password: robotPushAccount.secret!, + }) + } +} + /** * Create Harbor system robot account that is scoped to a given Harbor project with push and push access * to offer team members the option to download the kubeconfig. @@ -359,6 +512,26 @@ async function ensureTeamPushRobotAccount(projectName: string): Promise { return robotPushAccount } +/** + * Ensure that Harbor robot account and corresponding Kubernetes push secret for builds exist + * @param namespace Kubernetes namespace where push secret is created + * @param projectName Harbor project name + */ +async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { + const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) + if (!k8sSecret) { + const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) + console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) + await createBuildsK8sSecret({ + namespace, + name: projectBuildPushSecretName, + server: `${harborOperator.harborBaseRepoUrl}`, + username: robotBuildsPushAccount.name!, + password: robotBuildsPushAccount.secret!, + }) + } +} + /** * Create Harbor system robot account that is scoped to a given Harbor project with push access * for Kaniko (used for builds) task to push images. @@ -409,162 +582,3 @@ async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { - const bearerAuth: HttpBearerAuth = new HttpBearerAuth() - - let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret - if (!robotSecret) { - // not existing yet, create robot account and keep creds in secret - robotSecret = await createSystemRobotSecret() - } else { - // test if secret still works - try { - bearerAuth.accessToken = robotSecret.secret - robotApi.setDefaultAuthentication(bearerAuth) - await robotApi.listRobot() - } catch (e) { - // throw everything except 401, which is what we test for - if (e.status !== 401) throw e - // unauthenticated, so remove and recreate secret - await k8sApi.deleteNamespacedSecret(systemSecretName, systemNamespace) - // now, the next call might throw IF: - // - authMode oidc was already turned on and an otomi admin accidentally removed the secret - // but that is very unlikely, an unresolvable problem and needs a manual db fix - robotSecret = await createSystemRobotSecret() - } - } - bearerAuth.accessToken = robotSecret.secret - return bearerAuth -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes pull secret exist - * @param namespace Kubernetes namespace where pull secret is created - * @param projectName Harbor project name - */ -async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectPullSecretName, namespace) - if (k8sSecret) { - console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) - await k8sApi.deleteNamespacedSecret(projectPullSecretName, namespace) - } - - const robotPullAccount = await createTeamPullRobotAccount(projectName) - console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) - await createK8sSecret({ - namespace, - name: projectPullSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, - username: robotPullAccount.name!, - password: robotPullAccount.secret!, - }) -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes push secret exist - * @param namespace Kubernetes namespace where push secret is created - * @param projectName Harbor project name - */ -async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectPushSecretName, namespace) - if (!k8sSecret) { - const robotPushAccount = await ensureTeamPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) - await createK8sSecret({ - namespace, - name: projectPushSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, - username: robotPushAccount.name!, - password: robotPushAccount.secret!, - }) - } -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes push secret for builds exist - * @param namespace Kubernetes namespace where push secret is created - * @param projectName Harbor project name - */ -async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) - if (!k8sSecret) { - const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) - await createBuildsK8sSecret({ - namespace, - name: projectBuildPushSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, - username: robotBuildsPushAccount.name!, - password: robotBuildsPushAccount.secret!, - }) - } -} - -async function setupHarbor() { - console.log('harborOperator', harborOperator) - if (!harborOperator.harborBaseUrl) return - const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` - // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed - await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) - const bearerAuth = await getBearerToken() - robotApi.setDefaultAuthentication(bearerAuth) - configureApi.setDefaultAuthentication(bearerAuth) - projectsApi.setDefaultAuthentication(bearerAuth) - memberApi.setDefaultAuthentication(bearerAuth) - - await doApiCall(errors, 'Putting Harbor configuration', () => configureApi.configurationsPut(config)) - await Promise.all( - harborOperator.teamIds.map(async (teamId: string) => { - const projectName = `team-${teamId}` - const teamNamespce = projectName - const projectReq: ProjectReq = { - projectName, - } - await doApiCall(errors, `Creating project for team ${teamId}`, () => projectsApi.createProject(projectReq)) - - const project = await doApiCall(errors, `Get project for team ${teamId}`, () => - projectsApi.getProject(projectName), - ) - if (!project) return '' - const projectId = `${project.projectId}` - - const projMember: ProjectMember = { - roleId: HarborRole.developer, - memberGroup: { - groupName: projectName, - groupType: HarborGroupType.http, - }, - } - const projAdminMember: ProjectMember = { - roleId: HarborRole.admin, - memberGroup: { - groupName: 'team-admin', - groupType: HarborGroupType.http, - }, - } - await doApiCall( - errors, - `Associating "developer" role for team "${teamId}" with harbor project "${projectName}"`, - () => memberApi.createProjectMember(projectId, undefined, undefined, projMember), - ) - await doApiCall( - errors, - `Associating "project-admin" role for "team-admin" with harbor project "${projectName}"`, - () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), - ) - - await ensureTeamPullRobotAccountSecret(teamNamespce, projectName) - await ensureTeamPushRobotAccountSecret(teamNamespce, projectName) - await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName) - - return null - }), - ) - - handleErrors(errors) -} From 6bdaacd974d9613b7ddaeebbec7651337cf5a3b4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:25:24 +0200 Subject: [PATCH 14/23] test: harbor operator --- src/operator/harbor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 076424f2..47a8a19b 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -367,7 +367,7 @@ async function processNamespace(namespace: string) { return null } catch (error) { console.error(`Error processing namespace ${namespace}:`, error) - return null + throw error } } From 86fb1890ff36fb98a866681f9ce92be579a8c62f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:11:10 +0200 Subject: [PATCH 15/23] feat: add harbor operator env --- src/operator/harbor.ts | 79 ++++++++++++++++++++---------------------- src/validators.ts | 3 ++ 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 47a8a19b..0e6e62a8 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -18,13 +18,29 @@ import { } from '@redkubes/harbor-client-node' import { createBuildsK8sSecret, createK8sSecret, createSecret, getSecret } from '../k8s' import { doApiCall, handleErrors, waitTillAvailable } from '../utils' - -interface groupMapping { - [key: string]: { - otomi: string[] - } +import { + HARBOR_BASE_URL, + HARBOR_BASE_URL_PORT, + HARBOR_OPERATOR_NAMESPACE, + HARBOR_SYSTEM_NAMESPACE, + cleanEnv, +} from '../validators' + +// Interfaces +interface RobotSecret { + id: number + name: string + secret: string } +// Constants +const env = cleanEnv({ + HARBOR_BASE_URL, + HARBOR_BASE_URL_PORT, + HARBOR_OPERATOR_NAMESPACE, + HARBOR_SYSTEM_NAMESPACE, +}) + const HarborRole = { admin: 1, developer: 2, @@ -38,13 +54,6 @@ const HarborGroupType = { } const errors: string[] = [] - -export interface RobotSecret { - id: number - name: string - secret: string -} - const systemRobot: any = { name: 'harbor', duration: -1, @@ -67,11 +76,9 @@ const systemRobot: any = { const robotPrefix = 'otomi-' const harborOperator = { - harborBaseUrl: 'http://harbor-core.harbor:80/api/v2.0', harborBaseRepoUrl: '', - harborUser: 'admin', - harborPassword: 'welcomeotomi', - teamIds: [], + harborUser: '', + harborPassword: '', oidcClientId: '', oidcClientSecret: '', oidcEndpoint: '', @@ -99,25 +106,18 @@ const config: any = { self_registration: false, } -const systemNamespace = 'harbor' +const systemNamespace = env.HARBOR_SYSTEM_NAMESPACE const systemSecretName = 'harbor-robot-admin' const projectPullSecretName = 'harbor-pullsecret' const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' -// const harborBaseUrl = `https://harbor.172.233.37.26.nip.io/api/v2.0` -// const harborHealthUrl = `${harborBaseUrl}/systeminfo` -const robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborOperator.harborBaseUrl) -const configureApi = new ConfigureApi( - harborOperator.harborUser, - harborOperator.harborPassword, - harborOperator.harborBaseUrl, -) -const projectsApi = new ProjectApi( - harborOperator.harborUser, - harborOperator.harborPassword, - harborOperator.harborBaseUrl, -) -const memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborOperator.harborBaseUrl) +const harborBaseUrl = `${env.HARBOR_BASE_URL}:${env.HARBOR_BASE_URL_PORT}/api/v2.0` +const harborHealthUrl = `${harborBaseUrl}/systeminfo` +const harborOperatorNamespace = env.HARBOR_OPERATOR_NAMESPACE +let robotApi +let configureApi +let projectsApi +let memberApi const kc = new KubeConfig() // loadFromCluster when deploying on cluster @@ -135,17 +135,13 @@ const secretsAndConfigmapsCallback = async (e: any) => { const { metadata, data } = object if (object.kind === 'Secret' && metadata.name === 'harbor-admin') { - console.log('Secret:', metadata.name) harborOperator.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() harborOperator.harborUser = Buffer.from(data.harborUser, 'base64').toString() harborOperator.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() harborOperator.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { - console.log('ConfigMap:', metadata.name) - // harborOperator.harborBaseUrl = data.harborBaseUrl harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl - harborOperator.teamIds = JSON.parse(data.teamIds) harborOperator.oidcAutoOnboard = data.oidcAutoOnboard harborOperator.oidcUserClaim = data.oidcUserClaim harborOperator.oidcGroupsClaim = data.oidcGroupsClaim @@ -184,13 +180,13 @@ export default class MyOperator extends Operator { protected async init() { // Watch harbor-operator-secrets try { - await this.watchResource('', 'v1', 'secrets', secretsAndConfigmapsCallback, 'harbor-operator') + await this.watchResource('', 'v1', 'secrets', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) { console.debug(error) } // Watch harbor-operator-cm try { - await this.watchResource('', 'v1', 'configmaps', secretsAndConfigmapsCallback, 'harbor-operator') + await this.watchResource('', 'v1', 'configmaps', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) { console.debug(error) } @@ -246,11 +242,12 @@ async function runProcessNamespace(namespace: string) { // Setup Harbor async function setupHarbor() { - console.log('harborOperator', harborOperator) - if (!harborOperator.harborBaseUrl) return - const harborHealthUrl = `${harborOperator.harborBaseUrl}/systeminfo` // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) + robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) const bearerAuth = await getBearerToken() robotApi.setDefaultAuthentication(bearerAuth) configureApi.setDefaultAuthentication(bearerAuth) @@ -367,7 +364,7 @@ async function processNamespace(namespace: string) { return null } catch (error) { console.error(`Error processing namespace ${namespace}:`, error) - throw error + return null } } diff --git a/src/validators.ts b/src/validators.ts index a6225334..27d88ba2 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -19,7 +19,10 @@ const feat = cleanEnv({ FEAT_EXTERNAL_IDP }) export const CERT_ROTATION_DAYS = num({ desc: 'The amount of days for the cert rotation', default: 75 }) export const DOMAINS = json({ desc: 'A list of domains and their cert status' }) export const HARBOR_BASE_URL = str({ desc: 'The harbor core service URL' }) +export const HARBOR_BASE_URL_PORT = str({ desc: 'The harbor core service URL port' }) export const HARBOR_BASE_REPO_URL = str({ desc: 'The harbor repository base URL' }) +export const HARBOR_OPERATOR_NAMESPACE = str({ desc: 'The harbor operator namespace' }) +export const HARBOR_SYSTEM_NAMESPACE = str({ desc: 'The harbor system namespace' }) export const HARBOR_PASSWORD = str({ desc: 'The harbor admin password' }) export const HARBOR_USER = str({ desc: 'The harbor admin username' }) export const IDP_ALIAS = str({ desc: 'An alias for the IDP', default: 'otomi-idp' }) From bb00ce2dbfefe8bfcd2de47ea122f8d85fc5699a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:53:47 +0200 Subject: [PATCH 16/23] feat: update harbor operator oidc config --- src/operator/harbor.ts | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 0e6e62a8..8ef5affc 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -89,22 +89,6 @@ const harborOperator = { oidcName: 'keycloak', oidcScope: 'openid', } -const config: any = { - auth_mode: 'oidc_auth', - oidc_admin_group: 'admin', - oidc_client_id: 'otomi', - oidc_client_secret: harborOperator.oidcClientSecret, - oidc_endpoint: harborOperator.oidcEndpoint, - oidc_groups_claim: 'groups', - oidc_name: 'otomi', - oidc_scope: 'openid', - oidc_verify_cert: harborOperator.oidcVerifyCert, - oidc_user_claim: harborOperator.oidcUserClaim, - oidc_auto_onboard: harborOperator.oidcAutoOnboard, - project_creation_restriction: 'adminonly', - robot_name_prefix: robotPrefix, - self_registration: false, -} const systemNamespace = env.HARBOR_SYSTEM_NAMESPACE const systemSecretName = 'harbor-robot-admin' @@ -142,12 +126,12 @@ const secretsAndConfigmapsCallback = async (e: any) => { harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl - harborOperator.oidcAutoOnboard = data.oidcAutoOnboard + harborOperator.oidcAutoOnboard = data.oidcAutoOnboard === 'true' harborOperator.oidcUserClaim = data.oidcUserClaim harborOperator.oidcGroupsClaim = data.oidcGroupsClaim harborOperator.oidcName = data.oidcName harborOperator.oidcScope = data.oidcScope - harborOperator.oidcVerifyCert = data.oidcVerifyCert + harborOperator.oidcVerifyCert = data.oidcVerifyCert === 'true' } else return switch (e.type) { @@ -244,10 +228,29 @@ async function runProcessNamespace(namespace: string) { async function setupHarbor() { // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) + robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + + const config: any = { + auth_mode: 'oidc_auth', + oidc_admin_group: 'admin', + oidc_client_id: 'otomi', + oidc_client_secret: harborOperator.oidcClientSecret, + oidc_endpoint: harborOperator.oidcEndpoint, + oidc_groups_claim: 'groups', + oidc_name: 'otomi', + oidc_scope: 'openid', + oidc_verify_cert: harborOperator.oidcVerifyCert, + oidc_user_claim: harborOperator.oidcUserClaim, + oidc_auto_onboard: harborOperator.oidcAutoOnboard, + project_creation_restriction: 'adminonly', + robot_name_prefix: robotPrefix, + self_registration: false, + } + const bearerAuth = await getBearerToken() robotApi.setDefaultAuthentication(bearerAuth) configureApi.setDefaultAuthentication(bearerAuth) From 4cbf6de832b59a39bb08fc33295f5b5a9a315130 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:33:47 +0200 Subject: [PATCH 17/23] feat: remove harbor task & update harbor operator name & namespace --- .vscode/launch.json | 31 +-- .vscode/tasks.json | 16 ++ package.json | 2 - src/operator/harbor.ts | 8 +- src/tasks/harbor/harbor.ts | 453 ------------------------------------- 5 files changed, 38 insertions(+), 472 deletions(-) create mode 100644 .vscode/tasks.json delete mode 100644 src/tasks/harbor/harbor.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index e232548c..39975f08 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -85,6 +85,24 @@ "console": "integratedTerminal", "envFile": "${workspaceFolder}/.env", }, + { + "type": "node", + "request": "launch", + "name": "Debug harbor operator", + "runtimeExecutable": "npm", + "runtimeArgs": ["run-script", "operator:harbor-dev"], + "cwd": "${workspaceRoot}", + "console": "integratedTerminal", + "envFile": "${workspaceFolder}/.env", + "env" : { + "KUBECONFIG": "/path/to/your/kubeconfig.yaml", + "HARBOR_BASE_URL" : "http://localhost", + "HARBOR_BASE_URL_PORT" : "8083", + "HARBOR_OPERATOR_NAMESPACE" : "harbor-app-operator", + "HARBOR_SYSTEM_NAMESPACE" : "harbor", + }, + "preLaunchTask": "port-forward-harbor" + }, { "type": "node", "request": "launch", @@ -124,19 +142,6 @@ "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca" } }, - { - "type": "node", - "request": "launch", - "name": "Debug Harbor task", - "runtimeExecutable": "npm", - "runtimeArgs": ["run-script", "tasks:harbor-dev"], - "cwd": "${workspaceRoot}", - "console": "integratedTerminal", - "envFile": "${workspaceFolder}/.env" - // "env": { - // "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca" - // } - }, { "type": "node", "request": "launch", diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..21ed7a87 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,16 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "port-forward-harbor", + "type": "shell", + "command": "export KUBECONFIG=/path/to/your/kubeconfig.yaml && kubectl -n harbor port-forward svc/harbor-core 8083:80", + "problemMatcher": [], + "isBackground": true, + "presentation": { + "reveal": "always", + "panel": "new" + } + } + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 443f8320..e7362b2f 100644 --- a/package.json +++ b/package.json @@ -129,8 +129,6 @@ "tasks:gitea-drone-auth-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/gitea/gitea-drone-oauth.ts", "tasks:gitea-drone-auth": "node dist/tasks/gitea/gitea-drone-oauth.js", "tasks:gitea": "node dist/tasks/gitea/gitea.js && node dist/tasks/gitea/gitea-drone-oauth.js", - "tasks:harbor-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/harbor/harbor.ts", - "tasks:harbor": "node dist/tasks/harbor/harbor.js", "tasks:keycloak-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/keycloak/keycloak.ts", "tasks:keycloak": "node dist/tasks/keycloak/keycloak.js", "tasks:otomi-chart-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/otomi/otomi-chart.ts", diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 8ef5affc..7c2de02a 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -118,13 +118,13 @@ const secretsAndConfigmapsCallback = async (e: any) => { const { object } = e const { metadata, data } = object - if (object.kind === 'Secret' && metadata.name === 'harbor-admin') { + if (object.kind === 'Secret' && metadata.name === 'harbor-app-operator-secret') { harborOperator.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() harborOperator.harborUser = Buffer.from(data.harborUser, 'base64').toString() harborOperator.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() harborOperator.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() - } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-operator-cm') { + } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-app-operator-cm') { harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl harborOperator.oidcAutoOnboard = data.oidcAutoOnboard === 'true' harborOperator.oidcUserClaim = data.oidcUserClaim @@ -162,13 +162,13 @@ const namespacesCallback = async (e: any) => { export default class MyOperator extends Operator { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types protected async init() { - // Watch harbor-operator-secrets + // Watch harbor-app-operator-secret try { await this.watchResource('', 'v1', 'secrets', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) { console.debug(error) } - // Watch harbor-operator-cm + // Watch harbor-app-operator-cm try { await this.watchResource('', 'v1', 'configmaps', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) { diff --git a/src/tasks/harbor/harbor.ts b/src/tasks/harbor/harbor.ts deleted file mode 100644 index 9e2eafb8..00000000 --- a/src/tasks/harbor/harbor.ts +++ /dev/null @@ -1,453 +0,0 @@ -// eslint-disable @typescript-eslint/camelcase - -import { - ConfigureApi, - HttpBearerAuth, - MemberApi, - // eslint-disable-next-line no-unused-vars - Project, - ProjectApi, - // eslint-disable-next-line no-unused-vars - ProjectMember, - // eslint-disable-next-line no-unused-vars - ProjectReq, - RobotApi, - // eslint-disable-next-line no-unused-vars - RobotCreate, - // eslint-disable-next-line no-unused-vars - RobotCreated, -} from '@redkubes/harbor-client-node' -import { createBuildsK8sSecret, createK8sSecret, createSecret, getSecret, k8s } from '../../k8s' -import { doApiCall, handleErrors, waitTillAvailable } from '../../utils' -import { - HARBOR_BASE_REPO_URL, - HARBOR_BASE_URL, - HARBOR_PASSWORD, - HARBOR_USER, - OIDC_AUTO_ONBOARD, - OIDC_CLIENT_SECRET, - OIDC_ENDPOINT, - OIDC_USER_CLAIM, - OIDC_VERIFY_CERT, - TEAM_IDS, - cleanEnv, -} from '../../validators' - -const env = cleanEnv({ - HARBOR_BASE_URL, - HARBOR_BASE_REPO_URL, - HARBOR_PASSWORD, - HARBOR_USER, - OIDC_USER_CLAIM, - OIDC_AUTO_ONBOARD, - OIDC_CLIENT_SECRET, - OIDC_ENDPOINT, - OIDC_VERIFY_CERT, - TEAM_IDS, -}) - -const HarborRole = { - admin: 1, - developer: 2, - guest: 3, - master: 4, -} - -const HarborGroupType = { - ldap: 1, - http: 2, -} - -const errors: string[] = [] - -export interface RobotSecret { - id: number - name: string - secret: string -} - -const systemRobot: any = { - name: 'harbor', - duration: -1, - description: 'Used by Otomi Harbor task runner', - disable: false, - level: 'system', - permissions: [ - { - kind: 'system', - namespace: '/', - access: [ - { - resource: '*', - action: '*', - }, - ], - }, - ], -} - -const robotPrefix = 'otomi-' -const config: any = { - auth_mode: 'oidc_auth', - oidc_admin_group: 'admin', - oidc_client_id: 'otomi', - oidc_client_secret: env.OIDC_CLIENT_SECRET, - oidc_endpoint: env.OIDC_ENDPOINT, - oidc_groups_claim: 'groups', - oidc_name: 'otomi', - oidc_scope: 'openid', - oidc_verify_cert: env.OIDC_VERIFY_CERT, - oidc_user_claim: env.OIDC_USER_CLAIM, - oidc_auto_onboard: env.OIDC_AUTO_ONBOARD, - project_creation_restriction: 'adminonly', - robot_name_prefix: robotPrefix, - self_registration: false, -} - -const systemNamespace = 'harbor' -const systemSecretName = 'harbor-robot-admin' -const projectPullSecretName = 'harbor-pullsecret' -const projectPushSecretName = 'harbor-pushsecret' -const projectBuildPushSecretName = 'harbor-pushsecret-builds' -const harborBaseUrl = `${env.HARBOR_BASE_URL}/api/v2.0` -const harborHealthUrl = `${harborBaseUrl}/systeminfo` -console.log('harborHealthUrl', harborHealthUrl) -const robotApi = new RobotApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const configureApi = new ConfigureApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const projectsApi = new ProjectApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) -const memberApi = new MemberApi(env.HARBOR_USER, env.HARBOR_PASSWORD, harborBaseUrl) - -/** - * Create Harbor robot account that is used by Otomi tasks - * @note assumes OIDC is not yet configured, otherwise this operation is NOT possible - */ -async function createSystemRobotSecret(): Promise { - const { body: robotList } = await robotApi.listRobot() - console.log('robotApi', robotApi) - const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) - if (existing?.id) { - const existingId = existing.id - await doApiCall(errors, `Deleting previous robot account ${systemRobot.name}`, () => - robotApi.deleteRobot(existingId), - ) - } - const robotAccount = (await doApiCall( - errors, - `Create robot account ${systemRobot.name} with system level perms`, - () => robotApi.createRobot(systemRobot), - )) as RobotCreated - console.log('errors', errors) - console.log('robotAccount', robotAccount) - const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } - await createSecret(systemSecretName, systemNamespace, robotSecret) - return robotSecret -} - -/** - * Create Harbor system robot account that is scoped to a given Harbor project with pull access only. - * @param projectName Harbor project name - */ -async function createTeamPullRobotAccount(projectName: string): Promise { - const projectRobot: RobotCreate = { - name: `${projectName}-pull`, - duration: -1, - description: 'Allow team to pull from its own registry', - disable: false, - level: 'system', - permissions: [ - { - kind: 'project', - namespace: projectName, - access: [ - { - resource: 'repository', - action: 'pull', - }, - ], - }, - ], - } - const fullName = `${robotPrefix}${projectRobot.name}` - - const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) - const existing = robotList.find((i) => i.name === fullName) - - if (existing?.id) { - const existingId = existing.id - await doApiCall(errors, `Deleting previous pull robot account ${fullName}`, () => robotApi.deleteRobot(existingId)) - } - - const robotPullAccount = (await doApiCall( - errors, - `Creating pull robot account ${fullName} with project level perms`, - () => robotApi.createRobot(projectRobot), - )) as RobotCreated - if (!robotPullAccount?.id) { - throw new Error( - `RobotPullAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, - ) - } - return robotPullAccount -} - -/** - * Create Harbor system robot account that is scoped to a given Harbor project with push and push access - * to offer team members the option to download the kubeconfig. - * @param projectName Harbor project name - */ -async function ensureTeamPushRobotAccount(projectName: string): Promise { - const projectRobot: RobotCreate = { - name: `${projectName}-push`, - duration: -1, - description: 'Allow team to push to its own registry', - disable: false, - level: 'system', - permissions: [ - { - kind: 'project', - namespace: projectName, - access: [ - { - resource: 'repository', - action: 'push', - }, - { - resource: 'repository', - action: 'pull', - }, - ], - }, - ], - } - const fullName = `${robotPrefix}${projectRobot.name}` - - const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) - const existing = robotList.find((i) => i.name === fullName) - - if (existing?.name) { - return existing - } - - const robotPushAccount = (await doApiCall( - errors, - `Creating push robot account ${fullName} with project level perms`, - () => robotApi.createRobot(projectRobot), - )) as RobotCreated - if (!robotPushAccount?.id) { - throw new Error( - `RobotPushAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, - ) - } - return robotPushAccount -} - -/** - * Create Harbor system robot account that is scoped to a given Harbor project with push access - * for Kaniko (used for builds) task to push images. - * @param projectName Harbor project name - */ -async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise { - const projectRobot: RobotCreate = { - name: `${projectName}-builds`, - duration: -1, - description: 'Allow builds to push images', - disable: false, - level: 'system', - permissions: [ - { - kind: 'project', - namespace: projectName, - access: [ - { - resource: 'repository', - action: 'push', - }, - { - resource: 'repository', - action: 'pull', - }, - ], - }, - ], - } - const fullName = `${robotPrefix}${projectRobot.name}` - - const { body: robotList } = await robotApi.listRobot(undefined, undefined, undefined, undefined, 100) - const existing = robotList.find((i) => i.name === fullName) - - if (existing?.name) { - return existing - } - - const robotBuildsPushAccount = (await doApiCall( - errors, - `Creating push robot account ${fullName} with project level perms`, - () => robotApi.createRobot(projectRobot), - )) as RobotCreated - if (!robotBuildsPushAccount?.id) { - throw new Error( - `RobotBuildsPushAccount already exists and should have been deleted beforehand. This happens when more than 100 robot accounts exist.`, - ) - } - return robotBuildsPushAccount -} - -/** - * Get token by reading access token from kubernetes secret. - * If the secret does not exists then create Harbor robot account and populate credentials to kubernetes secret. - */ -async function getBearerToken(): Promise { - const bearerAuth: HttpBearerAuth = new HttpBearerAuth() - - let robotSecret = (await getSecret(systemSecretName, systemNamespace)) as RobotSecret - if (!robotSecret) { - // not existing yet, create robot account and keep creds in secret - robotSecret = await createSystemRobotSecret() - } else { - // test if secret still works - try { - bearerAuth.accessToken = robotSecret.secret - robotApi.setDefaultAuthentication(bearerAuth) - await robotApi.listRobot() - } catch (e) { - // throw everything except 401, which is what we test for - if (e.status !== 401) throw e - // unauthenticated, so remove and recreate secret - await k8s.core().deleteNamespacedSecret(systemSecretName, systemNamespace) - // now, the next call might throw IF: - // - authMode oidc was already turned on and an otomi admin accidentally removed the secret - // but that is very unlikely, an unresolvable problem and needs a manual db fix - robotSecret = await createSystemRobotSecret() - } - } - bearerAuth.accessToken = robotSecret.secret - return bearerAuth -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes pull secret exist - * @param namespace Kubernetes namespace where pull secret is created - * @param projectName Harbor project name - */ -async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectPullSecretName, namespace) - if (k8sSecret) { - console.debug(`Deleting pull secret/${projectPullSecretName} from ${namespace} namespace`) - await k8s.core().deleteNamespacedSecret(projectPullSecretName, namespace) - } - - const robotPullAccount = await createTeamPullRobotAccount(projectName) - console.debug(`Creating pull secret/${projectPullSecretName} at ${namespace} namespace`) - await createK8sSecret({ - namespace, - name: projectPullSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, - username: robotPullAccount.name!, - password: robotPullAccount.secret!, - }) -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes push secret exist - * @param namespace Kubernetes namespace where push secret is created - * @param projectName Harbor project name - */ -async function ensureTeamPushRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectPushSecretName, namespace) - if (!k8sSecret) { - const robotPushAccount = await ensureTeamPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) - await createK8sSecret({ - namespace, - name: projectPushSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, - username: robotPushAccount.name!, - password: robotPushAccount.secret!, - }) - } -} - -/** - * Ensure that Harbor robot account and corresponding Kubernetes push secret for builds exist - * @param namespace Kubernetes namespace where push secret is created - * @param projectName Harbor project name - */ -async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { - const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) - if (!k8sSecret) { - const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) - await createBuildsK8sSecret({ - namespace, - name: projectBuildPushSecretName, - server: `${env.HARBOR_BASE_REPO_URL}`, - username: robotBuildsPushAccount.name!, - password: robotBuildsPushAccount.secret!, - }) - } -} - -async function main(): Promise { - // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed - await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) - const bearerAuth = await getBearerToken() - robotApi.setDefaultAuthentication(bearerAuth) - configureApi.setDefaultAuthentication(bearerAuth) - projectsApi.setDefaultAuthentication(bearerAuth) - memberApi.setDefaultAuthentication(bearerAuth) - - await doApiCall(errors, 'Putting Harbor configuration', () => configureApi.configurationsPut(config)) - await Promise.all( - env.TEAM_IDS.map(async (teamId: string) => { - const projectName = `team-${teamId}` - const teamNamespce = projectName - const projectReq: ProjectReq = { - projectName, - } - await doApiCall(errors, `Creating project for team ${teamId}`, () => projectsApi.createProject(projectReq)) - - const project = (await doApiCall(errors, `Get project for team ${teamId}`, () => - projectsApi.getProject(projectName), - )) as Project - if (!project) return '' - const projectId = `${project.projectId}` - - const projMember: ProjectMember = { - roleId: HarborRole.developer, - memberGroup: { - groupName: projectName, - groupType: HarborGroupType.http, - }, - } - const projAdminMember: ProjectMember = { - roleId: HarborRole.admin, - memberGroup: { - groupName: 'team-admin', - groupType: HarborGroupType.http, - }, - } - await doApiCall( - errors, - `Associating "developer" role for team "${teamId}" with harbor project "${projectName}"`, - () => memberApi.createProjectMember(projectId, undefined, undefined, projMember), - ) - await doApiCall( - errors, - `Associating "project-admin" role for "team-admin" with harbor project "${projectName}"`, - () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), - ) - - await ensureTeamPullRobotAccountSecret(teamNamespce, projectName) - await ensureTeamPushRobotAccountSecret(teamNamespce, projectName) - await ensureTeamBuildPushRobotAccountSecret(teamNamespce, projectName) - - return null - }), - ) - - handleErrors(errors) -} - -if (typeof require !== 'undefined' && require.main === module) { - main() -} From 3bf1245d3c3fb7e63a51cf2d293eb56174fa02a3 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:54:01 +0200 Subject: [PATCH 18/23] fix: remove unnecessary logs --- .vscode/launch.json | 2 +- src/operator/harbor.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 39975f08..8f19be66 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -98,7 +98,7 @@ "KUBECONFIG": "/path/to/your/kubeconfig.yaml", "HARBOR_BASE_URL" : "http://localhost", "HARBOR_BASE_URL_PORT" : "8083", - "HARBOR_OPERATOR_NAMESPACE" : "harbor-app-operator", + "HARBOR_OPERATOR_NAMESPACE" : "harbor-app", "HARBOR_SYSTEM_NAMESPACE" : "harbor", }, "preLaunchTask": "port-forward-harbor" diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 7c2de02a..53369492 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -299,7 +299,6 @@ async function getBearerToken(): Promise { */ async function createSystemRobotSecret(): Promise { const { body: robotList } = await robotApi.listRobot() - console.log('robotApi', robotApi) const existing = robotList.find((i) => i.name === `${robotPrefix}${systemRobot.name}`) if (existing?.id) { const existingId = existing.id @@ -312,8 +311,6 @@ async function createSystemRobotSecret(): Promise { `Create robot account ${systemRobot.name} with system level perms`, () => robotApi.createRobot(systemRobot), )) as RobotCreated - console.log('robotAccount', robotAccount) - console.log('errors', errors) const robotSecret: RobotSecret = { id: robotAccount.id!, name: robotAccount.name!, secret: robotAccount.secret! } await createSecret(systemSecretName, systemNamespace, robotSecret) return robotSecret From f66bb9a9595585cece75af3b316541ccb1d5028d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:42:43 +0200 Subject: [PATCH 19/23] test: harbor operator base repo url --- src/operator/harbor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 53369492..eb58e7f3 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -356,6 +356,7 @@ async function processNamespace(namespace: string) { () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), ) + if (!harborOperator.harborBaseRepoUrl) throw new Error('Harbor base repo url is not set') await ensureTeamPullRobotAccountSecret(namespace, projectName) await ensureTeamPushRobotAccountSecret(namespace, projectName) await ensureTeamBuildPushRobotAccountSecret(namespace, projectName) From 48484ffe79348034fb430e226d0a5ad23651a637 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:54:59 +0200 Subject: [PATCH 20/23] test: harbor operator base repo url --- src/operator/harbor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index eb58e7f3..3deac08e 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -214,6 +214,7 @@ async function runSetupHarbor() { async function runProcessNamespace(namespace: string) { try { + if (!harborOperator.harborBaseRepoUrl) throw new Error('Harbor base repo url is not set') await processNamespace(namespace) } catch (error) { console.debug('Error could not process namespace', error) @@ -356,7 +357,6 @@ async function processNamespace(namespace: string) { () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), ) - if (!harborOperator.harborBaseRepoUrl) throw new Error('Harbor base repo url is not set') await ensureTeamPullRobotAccountSecret(namespace, projectName) await ensureTeamPushRobotAccountSecret(namespace, projectName) await ensureTeamBuildPushRobotAccountSecret(namespace, projectName) From 5d06cdff7ae4b3d5f7a9ce59dbc34c642585fd39 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:31:19 +0200 Subject: [PATCH 21/23] test: harbor operator team namespaces --- src/operator/harbor.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 3deac08e..5306a4fe 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -88,6 +88,7 @@ const harborOperator = { oidcGroupsClaim: 'groups', oidcName: 'keycloak', oidcScope: 'openid', + teamNamespaces: [], } const systemNamespace = env.HARBOR_SYSTEM_NAMESPACE @@ -132,6 +133,7 @@ const secretsAndConfigmapsCallback = async (e: any) => { harborOperator.oidcName = data.oidcName harborOperator.oidcScope = data.oidcScope harborOperator.oidcVerifyCert = data.oidcVerifyCert === 'true' + harborOperator.teamNamespaces = JSON.parse(data.teamNamespaces) } else return switch (e.type) { @@ -174,12 +176,12 @@ export default class MyOperator extends Operator { } catch (error) { console.debug(error) } - // Watch all namespaces - try { - await this.watchResource('', 'v1', 'namespaces', namespacesCallback) - } catch (error) { - console.debug(error) - } + // // Watch all namespaces + // try { + // await this.watchResource('', 'v1', 'namespaces', namespacesCallback) + // } catch (error) { + // console.debug(error) + // } } } @@ -203,6 +205,9 @@ if (typeof require !== 'undefined' && require.main === module) { async function runSetupHarbor() { try { await setupHarbor() + console.log('teamNamespaces', harborOperator.teamNamespaces) + if (harborOperator.teamNamespaces.length > 0) + await Promise.all(harborOperator.teamNamespaces.map((namespace) => processNamespace(`team-${namespace}`))) } catch (error) { console.debug('Error could not run setup harbor', error) console.debug('Retrying in 30 seconds') @@ -214,6 +219,7 @@ async function runSetupHarbor() { async function runProcessNamespace(namespace: string) { try { + console.log('harborOperator', harborOperator) if (!harborOperator.harborBaseRepoUrl) throw new Error('Harbor base repo url is not set') await processNamespace(namespace) } catch (error) { From 0781c758fab086c8c081ffa13b8adc71edb9b791 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 3 Jul 2024 00:47:31 +0200 Subject: [PATCH 22/23] feat: update harbor operator --- src/operator/harbor.ts | 181 ++++++++++++++++++++++------------------- 1 file changed, 98 insertions(+), 83 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 5306a4fe..fd987a0c 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -27,6 +27,10 @@ import { } from '../validators' // Interfaces +interface DependencyState { + [key: string]: any +} + interface RobotSecret { id: number name: string @@ -34,7 +38,7 @@ interface RobotSecret { } // Constants -const env = cleanEnv({ +const localEnv = cleanEnv({ HARBOR_BASE_URL, HARBOR_BASE_URL_PORT, HARBOR_OPERATOR_NAMESPACE, @@ -53,6 +57,7 @@ const HarborGroupType = { http: 2, } +let lastState: DependencyState = {} const errors: string[] = [] const systemRobot: any = { name: 'harbor', @@ -75,7 +80,7 @@ const systemRobot: any = { } const robotPrefix = 'otomi-' -const harborOperator = { +const env = { harborBaseRepoUrl: '', harborUser: '', harborPassword: '', @@ -91,14 +96,14 @@ const harborOperator = { teamNamespaces: [], } -const systemNamespace = env.HARBOR_SYSTEM_NAMESPACE +const systemNamespace = localEnv.HARBOR_SYSTEM_NAMESPACE const systemSecretName = 'harbor-robot-admin' const projectPullSecretName = 'harbor-pullsecret' const projectPushSecretName = 'harbor-pushsecret' const projectBuildPushSecretName = 'harbor-pushsecret-builds' -const harborBaseUrl = `${env.HARBOR_BASE_URL}:${env.HARBOR_BASE_URL_PORT}/api/v2.0` +const harborBaseUrl = `${localEnv.HARBOR_BASE_URL}:${localEnv.HARBOR_BASE_URL_PORT}/api/v2.0` const harborHealthUrl = `${harborBaseUrl}/systeminfo` -const harborOperatorNamespace = env.HARBOR_OPERATOR_NAMESPACE +const harborOperatorNamespace = localEnv.HARBOR_OPERATOR_NAMESPACE let robotApi let configureApi let projectsApi @@ -114,26 +119,31 @@ if (process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) } const k8sApi = kc.makeApiClient(k8s.CoreV1Api) +// Utility function to compare states +function hasStateChanged(currentState: DependencyState, _lastState: DependencyState): boolean { + return Object.entries(currentState).some(([key, value]) => !value || value !== _lastState[key]) +} + // Callbacks const secretsAndConfigmapsCallback = async (e: any) => { const { object } = e const { metadata, data } = object if (object.kind === 'Secret' && metadata.name === 'harbor-app-operator-secret') { - harborOperator.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() - harborOperator.harborUser = Buffer.from(data.harborUser, 'base64').toString() - harborOperator.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() - harborOperator.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() - harborOperator.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() + env.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() + env.harborUser = Buffer.from(data.harborUser, 'base64').toString() + env.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() + env.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() + env.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-app-operator-cm') { - harborOperator.harborBaseRepoUrl = data.harborBaseRepoUrl - harborOperator.oidcAutoOnboard = data.oidcAutoOnboard === 'true' - harborOperator.oidcUserClaim = data.oidcUserClaim - harborOperator.oidcGroupsClaim = data.oidcGroupsClaim - harborOperator.oidcName = data.oidcName - harborOperator.oidcScope = data.oidcScope - harborOperator.oidcVerifyCert = data.oidcVerifyCert === 'true' - harborOperator.teamNamespaces = JSON.parse(data.teamNamespaces) + env.harborBaseRepoUrl = data.harborBaseRepoUrl + env.oidcAutoOnboard = data.oidcAutoOnboard === 'true' + env.oidcUserClaim = data.oidcUserClaim + env.oidcGroupsClaim = data.oidcGroupsClaim + env.oidcName = data.oidcName + env.oidcScope = data.oidcScope + env.oidcVerifyCert = data.oidcVerifyCert === 'true' + env.teamNamespaces = JSON.parse(data.teamNamespaces) } else return switch (e.type) { @@ -151,15 +161,6 @@ const secretsAndConfigmapsCallback = async (e: any) => { } } -const namespacesCallback = async (e: any) => { - const { object }: { object: k8s.V1Pod } = e - const { metadata } = object - if (metadata && metadata.name?.startsWith('team-')) { - console.info(`Processing Namespace:`, metadata.name) - await runProcessNamespace(metadata.name) - } -} - // Operator export default class MyOperator extends Operator { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types @@ -176,12 +177,6 @@ export default class MyOperator extends Operator { } catch (error) { console.debug(error) } - // // Watch all namespaces - // try { - // await this.watchResource('', 'v1', 'namespaces', namespacesCallback) - // } catch (error) { - // console.debug(error) - // } } } @@ -202,32 +197,43 @@ if (typeof require !== 'undefined' && require.main === module) { } // Runners -async function runSetupHarbor() { - try { +async function checkAndExecute() { + const currentState: DependencyState = { + harborBaseRepoUrl: env.harborBaseRepoUrl, + harborUser: env.harborUser, + harborPassword: env.harborPassword, + oidcClientId: env.oidcClientId, + oidcClientSecret: env.oidcClientSecret, + oidcEndpoint: env.oidcEndpoint, + oidcVerifyCert: env.oidcVerifyCert, + oidcUserClaim: env.oidcUserClaim, + oidcAutoOnboard: env.oidcAutoOnboard, + oidcGroupsClaim: env.oidcGroupsClaim, + oidcName: env.oidcName, + oidcScope: env.oidcScope, + teamNames: env.teamNamespaces, + } + + if (hasStateChanged(currentState, lastState)) { await setupHarbor() - console.log('teamNamespaces', harborOperator.teamNamespaces) - if (harborOperator.teamNamespaces.length > 0) - await Promise.all(harborOperator.teamNamespaces.map((namespace) => processNamespace(`team-${namespace}`))) - } catch (error) { - console.debug('Error could not run setup harbor', error) - console.debug('Retrying in 30 seconds') - await new Promise((resolve) => setTimeout(resolve, 30000)) - console.debug('Retrying to setup harbor') - await runSetupHarbor() } + + if (currentState.teamNames && currentState.teamNames.length > 0 && currentState.teamNames !== lastState.teamNames) { + await Promise.all(currentState.teamNames.map((namespace) => processNamespace(`team-${namespace}`))) + } + + lastState = { ...currentState } } -async function runProcessNamespace(namespace: string) { +async function runSetupHarbor() { try { - console.log('harborOperator', harborOperator) - if (!harborOperator.harborBaseRepoUrl) throw new Error('Harbor base repo url is not set') - await processNamespace(namespace) + await checkAndExecute() } catch (error) { - console.debug('Error could not process namespace', error) + console.debug('Error could not run setup harbor', error) console.debug('Retrying in 30 seconds') await new Promise((resolve) => setTimeout(resolve, 30000)) - console.debug('Retrying to process namespace') - await runProcessNamespace(namespace) + console.debug('Retrying to setup harbor') + await runSetupHarbor() } } @@ -235,24 +241,25 @@ async function runProcessNamespace(namespace: string) { async function setupHarbor() { // harborHealthUrl is an in-cluster http svc, so no multiple external dns confirmations are needed await waitTillAvailable(harborHealthUrl, undefined, { confirmations: 1 }) + if (!env.harborUser) return - robotApi = new RobotApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) - configureApi = new ConfigureApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) - projectsApi = new ProjectApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) - memberApi = new MemberApi(harborOperator.harborUser, harborOperator.harborPassword, harborBaseUrl) + robotApi = new RobotApi(env.harborUser, env.harborPassword, harborBaseUrl) + configureApi = new ConfigureApi(env.harborUser, env.harborPassword, harborBaseUrl) + projectsApi = new ProjectApi(env.harborUser, env.harborPassword, harborBaseUrl) + memberApi = new MemberApi(env.harborUser, env.harborPassword, harborBaseUrl) const config: any = { auth_mode: 'oidc_auth', oidc_admin_group: 'admin', oidc_client_id: 'otomi', - oidc_client_secret: harborOperator.oidcClientSecret, - oidc_endpoint: harborOperator.oidcEndpoint, + oidc_client_secret: env.oidcClientSecret, + oidc_endpoint: env.oidcEndpoint, oidc_groups_claim: 'groups', oidc_name: 'otomi', oidc_scope: 'openid', - oidc_verify_cert: harborOperator.oidcVerifyCert, - oidc_user_claim: harborOperator.oidcUserClaim, - oidc_auto_onboard: harborOperator.oidcAutoOnboard, + oidc_verify_cert: env.oidcVerifyCert, + oidc_user_claim: env.oidcUserClaim, + oidc_auto_onboard: env.oidcAutoOnboard, project_creation_restriction: 'adminonly', robot_name_prefix: robotPrefix, self_registration: false, @@ -392,7 +399,7 @@ async function ensureTeamPullRobotAccountSecret(namespace: string, projectName): await createK8sSecret({ namespace, name: projectPullSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, + server: `${env.harborBaseRepoUrl}`, username: robotPullAccount.name!, password: robotPullAccount.secret!, }) @@ -452,17 +459,19 @@ async function createTeamPullRobotAccount(projectName: string): Promise { const k8sSecret = await getSecret(projectPushSecretName, namespace) - if (!k8sSecret) { - const robotPushAccount = await ensureTeamPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) - await createK8sSecret({ - namespace, - name: projectPushSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, - username: robotPushAccount.name!, - password: robotPushAccount.secret!, - }) + if (k8sSecret) { + console.debug(`Deleting push secret/${projectPushSecretName} from ${namespace} namespace`) + await k8sApi.deleteNamespacedSecret(projectPushSecretName, namespace) } + const robotPushAccount = await ensureTeamPushRobotAccount(projectName) + console.debug(`Creating push secret/${projectPushSecretName} at ${namespace} namespace`) + await createK8sSecret({ + namespace, + name: projectPushSecretName, + server: `${env.harborBaseRepoUrl}`, + username: robotPushAccount.name!, + password: robotPushAccount.secret!, + }) } /** @@ -500,7 +509,8 @@ async function ensureTeamPushRobotAccount(projectName: string): Promise { const existing = robotList.find((i) => i.name === fullName) if (existing?.name) { - return existing + const existingId = existing.id + await doApiCall(errors, `Deleting previous push robot account ${fullName}`, () => robotApi.deleteRobot(existingId)) } const robotPushAccount = (await doApiCall( @@ -523,17 +533,19 @@ async function ensureTeamPushRobotAccount(projectName: string): Promise { */ async function ensureTeamBuildPushRobotAccountSecret(namespace: string, projectName): Promise { const k8sSecret = await getSecret(projectBuildPushSecretName, namespace) - if (!k8sSecret) { - const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) - console.debug(`Creating push secret/${projectBuildPushSecretName} at ${namespace} namespace`) - await createBuildsK8sSecret({ - namespace, - name: projectBuildPushSecretName, - server: `${harborOperator.harborBaseRepoUrl}`, - username: robotBuildsPushAccount.name!, - password: robotBuildsPushAccount.secret!, - }) + if (k8sSecret) { + console.debug(`Deleting build push secret/${projectBuildPushSecretName} from ${namespace} namespace`) + await k8sApi.deleteNamespacedSecret(projectBuildPushSecretName, namespace) } + const robotBuildsPushAccount = await ensureTeamBuildsPushRobotAccount(projectName) + console.debug(`Creating build push secret/${projectBuildPushSecretName} at ${namespace} namespace`) + await createBuildsK8sSecret({ + namespace, + name: projectBuildPushSecretName, + server: `${env.harborBaseRepoUrl}`, + username: robotBuildsPushAccount.name!, + password: robotBuildsPushAccount.secret!, + }) } /** @@ -571,7 +583,10 @@ async function ensureTeamBuildsPushRobotAccount(projectName: string): Promise i.name === fullName) if (existing?.name) { - return existing + const existingId = existing.id + await doApiCall(errors, `Deleting previous build push robot account ${fullName}`, () => + robotApi.deleteRobot(existingId), + ) } const robotBuildsPushAccount = (await doApiCall( From 8ab0058e47ebf16602c75d9f20b3cc0ce28ceaae Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:52:19 +0200 Subject: [PATCH 23/23] feat: update harbor operator name and namespace --- src/operator/harbor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index 70e81e7b..40940e20 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -129,13 +129,13 @@ const secretsAndConfigmapsCallback = async (e: any) => { const { object } = e const { metadata, data } = object - if (object.kind === 'Secret' && metadata.name === 'harbor-app-operator-secret') { + if (object.kind === 'Secret' && metadata.name === 'apl-harbor-operator-secret') { env.harborPassword = Buffer.from(data.harborPassword, 'base64').toString() env.harborUser = Buffer.from(data.harborUser, 'base64').toString() env.oidcEndpoint = Buffer.from(data.oidcEndpoint, 'base64').toString() env.oidcClientId = Buffer.from(data.oidcClientId, 'base64').toString() env.oidcClientSecret = Buffer.from(data.oidcClientSecret, 'base64').toString() - } else if (object.kind === 'ConfigMap' && metadata.name === 'harbor-app-operator-cm') { + } else if (object.kind === 'ConfigMap' && metadata.name === 'apl-harbor-operator-cm') { env.harborBaseRepoUrl = data.harborBaseRepoUrl env.oidcAutoOnboard = data.oidcAutoOnboard === 'true' env.oidcUserClaim = data.oidcUserClaim @@ -165,13 +165,13 @@ const secretsAndConfigmapsCallback = async (e: any) => { export default class MyOperator extends Operator { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types protected async init() { - // Watch harbor-app-operator-secret + // Watch apl-harbor-operator-secret try { await this.watchResource('', 'v1', 'secrets', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) { console.debug(error) } - // Watch harbor-app-operator-cm + // Watch apl-harbor-operator-cm try { await this.watchResource('', 'v1', 'configmaps', secretsAndConfigmapsCallback, harborOperatorNamespace) } catch (error) {