From 40c229503bd9af2f5c1f5054c2de3a9a712426f5 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 19:58:55 +0200 Subject: [PATCH 01/17] Fix esbuild-related issues * Fix Swagger UI is not available #369 * Fix Error class names are wrong in logfile #370 * Improve logging and health check Signed-off-by: Marvin A. Ruder --- .pnp-ts.loader.mjs | 25 ----- .pnp.cjs | 21 +++- docker/Dockerfile | 2 +- docker/Dockerfile-build | 15 +-- package.json | 9 +- packages/backend/package.json | 4 +- packages/backend/src/server.ts | 14 ++- packages/backend/src/utils/logger.ts | 154 ++++++++++++--------------- packages/commons/package.json | 2 +- yarn.lock | 23 +++- 10 files changed, 137 insertions(+), 132 deletions(-) delete mode 100644 .pnp-ts.loader.mjs diff --git a/.pnp-ts.loader.mjs b/.pnp-ts.loader.mjs deleted file mode 100644 index 4c80a4289..000000000 --- a/.pnp-ts.loader.mjs +++ /dev/null @@ -1,25 +0,0 @@ -import Path from "node:path"; -import URL from "node:url"; -import FS from "node:fs"; - -export function resolve(specifier, context, next) { - if (!specifier.startsWith(".") || !specifier.endsWith(".js")) { - return next(specifier, context); - } - - const parentURL = context.parentURL; - if (!parentURL || !parentURL.startsWith("file:") || parentURL.includes("/.yarn/")) { - return next(specifier, context); - } - - const dirName = Path.dirname(URL.fileURLToPath(parentURL)); - const baseName = specifier.slice(0, -3); - const path = Path.join(dirName, baseName); - for (const extension of [".js", ".ts", ".tsx"]) { - if (FS.existsSync(path + extension)) { - return next(baseName + extension, context); - } - } - - return next(specifier, context); -} diff --git a/.pnp.cjs b/.pnp.cjs index 1ffe4827a..d7dbd3151 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -1641,6 +1641,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/node", "npm:18.17.5"],\ ["@types/selenium-webdriver", "npm:4.1.15"],\ ["@types/supertest", "npm:2.0.12"],\ + ["@types/swagger-ui-express", "npm:4.1.3"],\ ["@typescript-eslint/eslint-plugin", "virtual:9e2d75c26d812ba07f2548643e31c2f0eb2cb6f6eca268f33f7e7f2f00bc9a60e5174f4187df59beb0c43929d43a06842c0155865f0f7f541c96499f2ed6aada#npm:6.3.0"],\ ["@typescript-eslint/parser", "virtual:9e2d75c26d812ba07f2548643e31c2f0eb2cb6f6eca268f33f7e7f2f00bc9a60e5174f4187df59beb0c43929d43a06842c0155865f0f7f541c96499f2ed6aada#npm:6.3.0"],\ ["@vitest/coverage-v8", "virtual:9e2d75c26d812ba07f2548643e31c2f0eb2cb6f6eca268f33f7e7f2f00bc9a60e5174f4187df59beb0c43929d43a06842c0155865f0f7f541c96499f2ed6aada#npm:0.34.1"],\ @@ -1672,6 +1673,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["response-time", "npm:2.3.2"],\ ["selenium-webdriver", "npm:4.11.1"],\ ["supertest", "npm:6.3.3"],\ + ["swagger-ui-dist", "npm:5.3.1"],\ ["swagger-ui-express", "virtual:143f48b3b02030f94479b3a5988283313dc9859fa4cb6844b22a91f8f01b9f50780f19a990f97707ad409e89183b3679a8db4d101263b8caf9410c781cd5e728#npm:5.0.0"],\ ["typescript", "patch:typescript@npm%3A5.1.6#~builtin::version=5.1.6&hash=5da071"],\ ["vite", "virtual:143f48b3b02030f94479b3a5988283313dc9859fa4cb6844b22a91f8f01b9f50780f19a990f97707ad409e89183b3679a8db4d101263b8caf9410c781cd5e728#npm:4.4.9"],\ @@ -2534,6 +2536,17 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@types/swagger-ui-express", [\ + ["npm:4.1.3", {\ + "packageLocation": "./.yarn/cache/@types-swagger-ui-express-npm-4.1.3-0c91a9cfb5-1c990fa8c1.zip/node_modules/@types/swagger-ui-express/",\ + "packageDependencies": [\ + ["@types/swagger-ui-express", "npm:4.1.3"],\ + ["@types/express", "npm:4.17.17"],\ + ["@types/serve-static", "npm:1.15.1"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@types/treeify", [\ ["npm:1.0.0", {\ "packageLocation": "./.yarn/cache/@types-treeify-npm-1.0.0-b5e04e9cd3-1b2397030d.zip/node_modules/@types/treeify/",\ @@ -9677,10 +9690,10 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { }]\ ]],\ ["swagger-ui-dist", [\ - ["npm:5.1.0", {\ - "packageLocation": "./.yarn/cache/swagger-ui-dist-npm-5.1.0-2571f16bc3-41b91708e7.zip/node_modules/swagger-ui-dist/",\ + ["npm:5.3.1", {\ + "packageLocation": "./.yarn/unplugged/swagger-ui-dist-npm-5.3.1-af568da2af/node_modules/swagger-ui-dist/",\ "packageDependencies": [\ - ["swagger-ui-dist", "npm:5.1.0"]\ + ["swagger-ui-dist", "npm:5.3.1"]\ ],\ "linkType": "HARD"\ }]\ @@ -9699,7 +9712,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["swagger-ui-express", "virtual:143f48b3b02030f94479b3a5988283313dc9859fa4cb6844b22a91f8f01b9f50780f19a990f97707ad409e89183b3679a8db4d101263b8caf9410c781cd5e728#npm:5.0.0"],\ ["@types/express", "npm:4.17.17"],\ ["express", "npm:4.18.2"],\ - ["swagger-ui-dist", "npm:5.1.0"]\ + ["swagger-ui-dist", "npm:5.3.1"]\ ],\ "packagePeers": [\ "@types/express",\ diff --git a/docker/Dockerfile b/docker/Dockerfile index 66185d123..b32ea70ae 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,6 @@ USER node COPY --from=assemble --chown=node:node /workdir/app . # Define health check -HEALTHCHECK --interval=5m --start-period=15s CMD wget -qO /dev/null http://localhost:$PORT/api/status || exit 1 +HEALTHCHECK CMD wget -qO /dev/null http://localhost:$PORT/api/status || exit 1 CMD [ "dumb-init", "node", "server.mjs" ] diff --git a/docker/Dockerfile-build b/docker/Dockerfile-build index 9419ccda8..2a911e525 100644 --- a/docker/Dockerfile-build +++ b/docker/Dockerfile-build @@ -5,15 +5,16 @@ ENV FORCE_COLOR true WORKDIR /workdir -# Copy project files and WebAssembly package -COPY . . +# Copy project files as well as Swagger UI files and WebAssembly package COPY --from=marvinruder/rating-tracker:wasm /workdir/pkg packages/wasm +COPY .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui.css .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-bundle.js .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-standalone-preset.js /workdir/app/packages/backend/dist/public/api-docs/ +COPY . . -# Build Node.js bundle -RUN yarn build - -# Create directories for target container and copy only necessary files -RUN mkdir -p app/public app/prisma/client && \ +RUN \ + # Build Node.js bundle + yarn build && \ + # Create directories for target container and copy only necessary files + mkdir -p app/public app/prisma/client && \ cp packages/backend/dist/server.mjs app && \ cp -r packages/backend/prisma/client/schema.prisma packages/backend/prisma/client/libquery_engine-* app/prisma/client && \ cp -r packages/frontend/dist/* app/public diff --git a/package.json b/package.json index 983cfe35a..6be68ee8f 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "scripts": { "dev:server": "conc --kill-others \"yarn workspace @rating-tracker/backend dev:run\" \"yarn workspace @rating-tracker/backend dev:watch\" \"yarn workspace @rating-tracker/frontend dev:vite\" \"yarn workspace @rating-tracker/commons dev:build\" -n \",,,ﯤ\" -p \"{name}\" -c #339933,#3178C6,#61DAFB,#3178C6 --timings", "dev:tools": "conc --kill-others \"yarn workspace @rating-tracker/backend dev:tools\" -n \"\" -p \"{name}\" -c grey --timings", - "dev:wasm": "wasm-pack build -s rating-tracker -d ../packages/wasm --debug wasm", "prisma:migrate:dev": "yarn workspace @rating-tracker/backend prisma:migrate:dev", "prisma:studio": "yarn workspace @rating-tracker/backend prisma:studio", "test": "yarn workspaces foreach -pt run test", @@ -21,7 +20,8 @@ "test:prisma:migrate:init": "yarn workspace @rating-tracker/backend test:prisma:migrate:init", "build": "yarn workspaces foreach -pt run build", "build:wasm": "wasm-pack build -s rating-tracker -d ../packages/wasm --release wasm && sed -E -i.bak 's/\"module\": \"([A-Za-z0-9\\-\\.]+)\",/\"main\": \"\\1\",\\\n \"module\": \"\\1\",/g ; s/^}$/}\\\n/' packages/wasm/package.json && rm packages/wasm/package.json.bak", - "lint": "yarn workspaces foreach -pt run lint" + "lint": "yarn workspaces foreach -pt run lint", + "fix:swagger": "mkdir -p packages/backend/dist/public/api-docs && cp .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui{.css,-bundle.js,-standalone-preset.js} packages/backend/dist/public/api-docs/" }, "packageManager": "yarn@3.6.1", "devDependencies": { @@ -32,5 +32,10 @@ }, "resolutions": { "vite/esbuild": "0.19.1" + }, + "dependenciesMeta": { + "swagger-ui-dist@5.3.1": { + "unplugged": true + } } } diff --git a/packages/backend/package.json b/packages/backend/package.json index 7dde9a579..f080ea6f4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -26,7 +26,7 @@ "prisma:studio": "pnpify prisma studio", "prisma:generate": "pnpify prisma generate", "prisma:migrate:dev": "pnpify prisma migrate dev", - "build": "esbuild --color=true src/server.ts --bundle --minify --platform=node --format=esm --legal-comments=none --banner:js=\"import r from'path';import{createRequire as e}from'module';import{fileURLToPath as m}from'url';const require=e(import.meta.url),__filename=m(import.meta.url),__dirname=r.dirname(__filename);\" --outfile=dist/server.mjs", + "build": "esbuild --color=true src/server.ts --bundle --minify --keep-names --platform=node --format=esm --legal-comments=none --banner:js=\"import __path from'path';import{createRequire}from'module';import{fileURLToPath}from'url';const require=createRequire(import.meta.url),__filename=fileURLToPath(import.meta.url),__dirname=__path.dirname(__filename);\" --outfile=dist/server.mjs", "lint": "eslint --cache --ext .ts src/", "lint:fix": "eslint --cache --ext .ts src/ --fix" }, @@ -50,6 +50,7 @@ "redis-om": "0.4.2", "response-time": "2.3.2", "selenium-webdriver": "4.11.1", + "swagger-ui-dist": "5.3.1", "swagger-ui-express": "5.0.0" }, "devDependencies": { @@ -58,6 +59,7 @@ "@types/node": "18.17.5", "@types/selenium-webdriver": "4.1.15", "@types/supertest": "2.0.12", + "@types/swagger-ui-express": "4.1.3", "@typescript-eslint/eslint-plugin": "6.3.0", "@typescript-eslint/parser": "6.3.0", "@vitest/coverage-v8": "0.34.1", diff --git a/packages/backend/src/server.ts b/packages/backend/src/server.ts index fc0412ad4..5b2c6fa84 100644 --- a/packages/backend/src/server.ts +++ b/packages/backend/src/server.ts @@ -140,7 +140,19 @@ server.app.use((req, res, next) => { }); // Host the OpenAPI UI -server.app.use("/api-docs", SwaggerUI.serve, SwaggerUI.setup(openapiDocument)); +server.app.use( + "/api-docs", + SwaggerUI.serve, + SwaggerUI.setup( + openapiDocument, + undefined, + undefined, + undefined, + "/assets/images/favicon-dev/favicon-192.png", + undefined, + "Rating Tracker API", + ), +); // Host the OpenAPI JSON configuration server.app.get("/api-spec/v3", (_, res) => res.json(openapiDocument)); diff --git a/packages/backend/src/utils/logger.ts b/packages/backend/src/utils/logger.ts index 88dd33c59..0cd280382 100644 --- a/packages/backend/src/utils/logger.ts +++ b/packages/backend/src/utils/logger.ts @@ -30,19 +30,14 @@ const levelIcons = { /** * The stream used to log messages to the standard output. */ -const prettyStream = pretty({ - include: "level", - customPrettifiers: { - level: (level) => levelIcons[Number(level)], - }, -}); +const prettyStream = pretty({ include: "level", customPrettifiers: { level: (level) => levelIcons[Number(level)] } }); /** * Provides the path of the log file for the current day. * * @returns {string} The path of the log file. */ -const getLogFilePath = () => { +const getLogFilePath = (): string => { return (process.env.LOG_FILE ?? "/tmp/rating-tracker-log-(DATE).log").replaceAll( "(DATE)", new Date().toISOString().split("T")[0], @@ -54,11 +49,7 @@ const getLogFilePath = () => { * * @returns {fs.WriteStream} The stream to write to the log file. */ -const getNewFileStream = () => { - return fs.createWriteStream(getLogFilePath(), { - flags: "a", - }); -}; +const getNewFileStream = (): fs.WriteStream => fs.createWriteStream(getLogFilePath(), { flags: "a" }); let fileStream = getNewFileStream(); @@ -66,25 +57,14 @@ let fileStream = getNewFileStream(); * A multistream which writes to both the standard output and the log file. */ const multistream = pino.multistream([ - { - level: (process.env.LOG_LEVEL as pino.Level) ?? "info", - stream: prettyStream, - }, - { - level: (process.env.LOG_LEVEL as pino.Level) ?? "info", - stream: fileStream, - }, + { level: (process.env.LOG_LEVEL as pino.Level) ?? "info", stream: prettyStream }, + { level: (process.env.LOG_LEVEL as pino.Level) ?? "info", stream: fileStream }, ]); /** * The logger used to log messages to both the standard output and the log file. */ -const logger = pino( - { - level: process.env.LOG_LEVEL ?? "info", - }, - multistream, -); +const logger = pino({ level: process.env.LOG_LEVEL ?? "info" }, multistream); // Rotate the log file every day new cron.CronJob( @@ -104,7 +84,7 @@ new cron.CronJob( * @param {string} method The HTTP method. * @returns {string} A colored pretty prefix string. */ -const highlightMethod = (method: string) => { +const highlightMethod = (method: string): string => { switch (method) { case "GET": return chalk.whiteBright.bgBlue(` ${method} `) + chalk.blue.bgGrey(""); @@ -127,7 +107,7 @@ const highlightMethod = (method: string) => { * @param {number} statusCode The HTTP status code. * @returns {string} A colored pretty prefix string. */ -const statusCodeDescription = (statusCode: number) => { +const statusCodeDescription = (statusCode: number): string => { const statusCodeString = ` ${statusCode}  ${STATUS_CODES[statusCode]} `; switch (Math.floor(statusCode / 100)) { case 2: // Successful responses @@ -147,64 +127,66 @@ const statusCodeDescription = (statusCode: number) => { * @param {Request} req Request object * @param {Response} res Response object * @param {number} time The response time of the request. + * @returns {void} */ -export const requestLogger = (req: Request, res: Response, time: number) => { - // Do not log requests for resources such as logos – those are far too many and only mildly interesting - if (!req.originalUrl.startsWith(`/api${stockLogoEndpointPath}`)) { - chalk - .white( - chalk.whiteBright.bgHex("#339933")(" \uf898 ") + - chalk.bgGrey.hex("#339933")("") + - chalk.bgGrey( - chalk.cyanBright(" \uf5ef " + new Date().toISOString()) + // Timestamp - "  " + - chalk.yellow( - res.locals.user - ? `\uf007 ${res.locals.user.name} (${res.locals.user.email})` // Authenticated user - : /* c8 ignore next */ // We do not test Cron jobs - res.locals.userIsCron - ? "\ufba7 cron" // Cron job - : "\uf21b", // Unauthenticated user - ) + - "  " + - chalk.magentaBright("\uf98c" + req.ip) + // IP address - " ", - ) + - chalk.grey("") + - "\n ├─" + - highlightMethod(req.method) + // HTTP request method - chalk.bgGrey( - ` ${req.originalUrl // URL path - .slice(1, req.originalUrl.indexOf("?") == -1 ? undefined : req.originalUrl.indexOf("?")) - .replaceAll("/", "  ")} `, - ) + - chalk.grey("") + - Object.entries(req.cookies) // Cookies - .map( - ([key, value]) => - "\n ├─" + chalk.bgGrey(chalk.yellow(" \uf697") + `  ${key} `) + chalk.grey("") + " " + value, - ) - .join(" ") + - Object.entries(req.query) // Query parameters - .map( - ([key, value]) => - "\n ├─" + chalk.bgGrey(chalk.cyan(" \uf002") + `  ${key} `) + chalk.grey("") + " " + value, - ) - .join(" ") + - "\n ╰─" + - statusCodeDescription(res.statusCode) + // HTTP response status code - ` ${ - res.hasHeader("Content-Length") && res.hasHeader("Content-Type") - ? `sent ${res.getHeader("Content-Length")} bytes of type “${ - res.getHeader("Content-Type").toString().split(";")[0] - }” ` - : "" - }after ${Math.round(time)} ms`, // Response time - ) - .split("\n") - .forEach((line) => logger.info(line)); // Show newlines in the log in a pretty way - logger.info(""); - } -}; +export const requestLogger = (req: Request, res: Response, time: number): void => + chalk + .white( + chalk.whiteBright.bgHex("#339933")(" \uf898 ") + + chalk.bgGrey.hex("#339933")("") + + chalk.bgGrey( + chalk.cyanBright(" \uf5ef " + new Date().toISOString()) + // Timestamp + "  " + + chalk.yellow( + res.locals.user + ? `\uf007 ${res.locals.user.name} (${res.locals.user.email})` // Authenticated user + : /* c8 ignore next */ // We do not test Cron jobs + res.locals.userIsCron + ? "\ufba7 cron" // Cron job + : "\uf21b", // Unauthenticated user + ) + + "  " + + chalk.magentaBright("\uf98c" + req.ip) + // IP address + " ", + ) + + chalk.grey("") + + "\n ├─" + + highlightMethod(req.method) + // HTTP request method + chalk.bgGrey( + ` ${req.originalUrl // URL path + .slice(1, req.originalUrl.indexOf("?") == -1 ? undefined : req.originalUrl.indexOf("?")) + .replaceAll("/", "  ")} `, + ) + + chalk.grey("") + + Object.entries(req.cookies) // Cookies + .map( + ([key, value]) => + "\n ├─" + chalk.bgGrey(chalk.yellow(" \uf697") + `  ${key} `) + chalk.grey("") + " " + value, + ) + .join(" ") + + Object.entries(req.query) // Query parameters + .map( + ([key, value]) => + "\n ├─" + chalk.bgGrey(chalk.cyan(" \uf002") + `  ${key} `) + chalk.grey("") + " " + value, + ) + .join(" ") + + "\n ╰─" + + statusCodeDescription(res.statusCode) + // HTTP response status code + ` ${ + res.hasHeader("Content-Length") && res.hasHeader("Content-Type") + ? `sent ${res.getHeader("Content-Length")} bytes of type “${ + res.getHeader("Content-Type").toString().split(";")[0] + }” ` + : "" + }after ${Math.round(time)} ms +`, // Response time + ) + .split("\n") + // Log internal requests or requests for resources such as logos as debug messages – those are far too many and + // typically only mildly interesting + .forEach((line) => + // Show newlines in the log in a pretty way + logger[req.originalUrl.startsWith(`/api${stockLogoEndpointPath}`) || req.ip === "::1" ? "trace" : "info"](line), + ); export default logger; diff --git a/packages/commons/package.json b/packages/commons/package.json index 10119245a..4b8ef824a 100644 --- a/packages/commons/package.json +++ b/packages/commons/package.json @@ -12,7 +12,7 @@ "types": "./src/index.ts", "scripts": { "dev:build": "yarn build --watch", - "build": "esbuild --color=true src/index.ts --bundle --minify --platform=node --format=esm --legal-comments=none --outfile=dist/index.js", + "build": "esbuild --color=true src/index.ts --bundle --minify --keep-names --platform=node --format=esm --legal-comments=none --outfile=dist/index.js", "test:watch": "vitest", "test": "vitest run --color", "lint": "eslint --cache --ext .ts src/", diff --git a/yarn.lock b/yarn.lock index f0aba782e..fd9ef9124 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1216,6 +1216,7 @@ __metadata: "@types/node": 18.17.5 "@types/selenium-webdriver": 4.1.15 "@types/supertest": 2.0.12 + "@types/swagger-ui-express": 4.1.3 "@typescript-eslint/eslint-plugin": 6.3.0 "@typescript-eslint/parser": 6.3.0 "@vitest/coverage-v8": 0.34.1 @@ -1247,6 +1248,7 @@ __metadata: response-time: 2.3.2 selenium-webdriver: 4.11.1 supertest: 6.3.3 + swagger-ui-dist: 5.3.1 swagger-ui-express: 5.0.0 typescript: 5.1.6 vite: 4.4.9 @@ -1334,6 +1336,9 @@ __metadata: eslint: 8.47.0 prettier: 3.0.1 typescript: 5.1.6 + dependenciesMeta: + swagger-ui-dist@5.3.1: + unplugged: true languageName: unknown linkType: soft @@ -1948,6 +1953,16 @@ __metadata: languageName: node linkType: hard +"@types/swagger-ui-express@npm:4.1.3": + version: 4.1.3 + resolution: "@types/swagger-ui-express@npm:4.1.3" + dependencies: + "@types/express": "*" + "@types/serve-static": "*" + checksum: 1c990fa8c158f699c5443245383daef2c4b47efce715c105e67f9b88447cf23663774393fdeea7d5a6e97a83d6959b09129f19c4abca64abdb7db74517162295 + languageName: node + linkType: hard + "@types/treeify@npm:^1.0.0": version: 1.0.0 resolution: "@types/treeify@npm:1.0.0" @@ -8148,10 +8163,10 @@ __metadata: languageName: node linkType: hard -"swagger-ui-dist@npm:>=5.0.0": - version: 5.1.0 - resolution: "swagger-ui-dist@npm:5.1.0" - checksum: 41b91708e757852423a4fddfc07d0e87ef38c4ad9fad5757fbc27b23c9e71593a1c48b53661fa2d9bb241043b6800cd9a57d980943a17c9eb388704e30156120 +"swagger-ui-dist@npm:5.3.1, swagger-ui-dist@npm:>=5.0.0": + version: 5.3.1 + resolution: "swagger-ui-dist@npm:5.3.1" + checksum: babf26f3a40a932db22248a620dbb0c5c68721c4165b173537a949c6ec3df3a89982db484651e1f5fe31f3cdb1037eeef13fb6b746efbb72902d7410291070c8 languageName: node linkType: hard From f6a335aed5e19923358513502e11d92893383e66 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 20:43:27 +0200 Subject: [PATCH 02/17] Optimize Docker caching related to Prisma Signed-off-by: Marvin A. Ruder --- README.md | 5 +++-- docker/Dockerfile-yarn | 9 +++++---- package.json | 1 - packages/backend/package.json | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ee1884de5..779ca56be 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,8 @@ An environment with services for development purposes can quickly be created usi - Run `yarn build:wasm` to build the WebAssembly utilities and create the `wasm` package. This requires [Rust](https://www.rust-lang.org) and [wasm-pack](https://rustwasm.github.io/wasm-pack/) to be installed. - Run `yarn dev:tools` to start NGINX, PostgreSQL, Redis, Selenium and the Signal REST API. SSL Certificates and the Redis ACL file must be provided beforehand, and a Signal account must be created before starting the server (see [section Setup steps](#setup-steps) for details). The NGINX configuration might require adjustment to your situation. -- Run `yarn prisma:migrate:dev` to initialize the PostgreSQL database and generate the Prisma client. +- Run `yarn workspace @rating-tracker/backend prisma:migrate:generate` to generate the Prisma client. +- Run `yarn workspace @rating-tracker/backend prisma:migrate:deploy` to initialize the PostgreSQL database. - Run `yarn dev:server` to start the backend server as well as the Vite frontend development server. NB: The `dev:tools` command sometimes fails because multiple Docker networks with the same name are created concurrently. In that case, manually delete all but one using `docker network ls` and `docker network rm …` and try again. @@ -370,7 +371,7 @@ A test environment with separate PostgreSQL and Redis instances can be created u - Run `yarn build:wasm` to build the WebAssembly utilities and create the `wasm` package. This requires [Rust](https://www.rust-lang.org) and [wasm-pack](https://rustwasm.github.io/wasm-pack/) to be installed. - Manually create a `main` entry in the `package.json` of the `wasm` package pointing to the same file as the `module` entry. - Run `yarn test:tools` to start PostgreSQL and Redis. -- Run `yarn test:prisma:migrate:init` to initialize the PostgreSQL database. +- Run `yarn test:prisma:migrate:deploy` to initialize the PostgreSQL database. - Run `yarn test` to run all tests from all packages. Additionally, the packages’ `package.json` configurations contain a `test:watch` script to run tests in watch mode. NB: The `test:tools` command sometimes fails because multiple Docker networks with the same name are created concurrently. In that case, manually delete all but one using `docker network ls` and `docker network rm …` and try again. diff --git a/docker/Dockerfile-yarn b/docker/Dockerfile-yarn index adf123cd4..f55177c70 100644 --- a/docker/Dockerfile-yarn +++ b/docker/Dockerfile-yarn @@ -15,11 +15,12 @@ COPY packages/wasm/package.json ./packages/wasm/ # Install dependencies RUN yarn --immutable -# Copy files required for Prisma +# Generate Prisma client COPY packages/backend/prisma ./packages/backend/prisma -COPY packages/backend/test/.env ./packages/backend/test/ +RUN yarn workspace @rating-tracker/backend prisma:generate -# Initialize Prisma for tests and generate client -RUN yarn workspace @rating-tracker/backend test:prisma:migrate:init +# Initialize Prisma for tests +COPY packages/backend/test/.env ./packages/backend/test/ +RUN yarn workspace @rating-tracker/backend test:prisma:migrate:deploy ENTRYPOINT [ "/bin/sh" ] diff --git a/package.json b/package.json index 6be68ee8f..70096b8dd 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "prisma:studio": "yarn workspace @rating-tracker/backend prisma:studio", "test": "yarn workspaces foreach -pt run test", "test:tools": "conc --kill-others \"yarn workspace @rating-tracker/backend test:tools\" -n \"\" -p \"{name}\" -c grey --timings", - "test:prisma:migrate:init": "yarn workspace @rating-tracker/backend test:prisma:migrate:init", "build": "yarn workspaces foreach -pt run build", "build:wasm": "wasm-pack build -s rating-tracker -d ../packages/wasm --release wasm && sed -E -i.bak 's/\"module\": \"([A-Za-z0-9\\-\\.]+)\",/\"main\": \"\\1\",\\\n \"module\": \"\\1\",/g ; s/^}$/}\\\n/' packages/wasm/package.json && rm packages/wasm/package.json.bak", "lint": "yarn workspaces foreach -pt run lint", diff --git a/packages/backend/package.json b/packages/backend/package.json index f080ea6f4..b01ad53d1 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -17,12 +17,12 @@ "dev:selenium": "docker compose -f dev/docker-compose.yml up selenium", "dev:signal": "docker compose -f dev/docker-compose.yml up signal", "dev:tools": "conc --kill-others \"yarn dev:nginx\" \"yarn dev:postgres\" \"yarn dev:redis\" \"yarn dev:selenium\" \"yarn dev:signal\" -n \",,,ﰍ,\" -p \"{name}\" -c #43972A,#336791,#D82C20,#43B02A,#4975E8 --timings", - "test:watch": "yarn test:prisma:migrate:init && vitest", - "test": "vitest run --color", + "test:watch": "yarn test:prisma:migrate:deploy && vitest", + "test": "yarn test:prisma:migrate:deploy && vitest run --color", "test:postgres": "PGPORT=54321 docker compose -f test/docker-compose.yml up --force-recreate -V postgres-test", "test:redis": "REDISPORT=63791 docker compose -f test/docker-compose.yml up --force-recreate -V redis-test", "test:tools": "conc --kill-others \"yarn test:postgres\" \"yarn test:redis\" -n \",\" -p \"{name}\" -c #336791,#D82C20 --timings", - "test:prisma:migrate:init": "env $(cat test/.env) pnpify prisma migrate dev --name init", + "test:prisma:migrate:deploy": "env $(cat test/.env) pnpify prisma migrate deploy", "prisma:studio": "pnpify prisma studio", "prisma:generate": "pnpify prisma generate", "prisma:migrate:dev": "pnpify prisma migrate dev", From 543378050f9527b4c2229ec49a361f090eb5d41e Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 21:40:20 +0200 Subject: [PATCH 03/17] Optimize initialization of database in tests Signed-off-by: Marvin A. Ruder --- .gitignore | 3 +++ Jenkinsfile | 23 ++++++++++++----------- README.md | 3 +-- docker/Dockerfile-yarn | 4 ---- package.json | 2 +- packages/backend/package.json | 5 ++--- packages/backend/test/.env | 2 +- packages/backend/test/docker-compose.yml | 2 ++ packages/backend/test/seeds/postgres.ts | 5 +---- 9 files changed, 23 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 322e5e78c..8e44ad710 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,9 @@ packages/backend/dev/signal-cli # Prisma packages/backend/prisma/client +# local test environment +packages/backend/test/*.sql + # Rust packages/wasm/* !packages/wasm/package.json diff --git a/Jenkinsfile b/Jenkinsfile index 2b64545d2..fbb768db9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,14 +18,6 @@ node('rating-tracker-build') { JOB_ID = sh (script: "#!/bin/bash\nprintf \"%04d\" \$((1 + RANDOM % 8192))", returnStdout: true) PGPORT = sh (script: "#!/bin/bash\necho -n \$((49151 + 10#$JOB_ID))", returnStdout: true) REDISPORT = sh (script: "#!/bin/bash\necho -n \$((57343 + 10#$JOB_ID))", returnStdout: true) - - // Change config files for use in CI - sh """ - echo \"globalFolder: /workdir/global\" >> .yarnrc.yml - echo \"preferAggregateCacheInfo: true\" >> .yarnrc.yml - echo \"enableGlobalCache: true\" >> .yarnrc.yml - sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env - """ } }, docker_env: { @@ -42,7 +34,12 @@ node('rating-tracker-build') { parallel( testenv: { stage('Start test environment') { - sh("PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose.yml up --force-recreate -V -d") + // Create migration script from all migrations and inject IP and ports into test environment + sh """ + cat packages/backend/prisma/migrations/*/migration.sql > packages/backend/test/all_migrations.sql + PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose.yml up --force-recreate -V -d + sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env + """ } }, wasm: { @@ -56,8 +53,12 @@ node('rating-tracker-build') { }, dep: { stage ('Install dependencies') { - // Copy global cache to workspace - sh("mkdir -p /home/jenkins/.cache/yarn/global && cp -arn /home/jenkins/.cache/yarn/global .") + // Change config files for use in CI and copy global cache to workspace + sh """ + echo \"globalFolder: /workdir/global\npreferAggregateCacheInfo: true\nenableGlobalCache: true\" >> .yarnrc.yml + mkdir -p /home/jenkins/.cache/yarn/global + cp -arn /home/jenkins/.cache/yarn/global . + """ // Install dependencies docker.build("$imagename:job$JOB_ID-yarn", "-f docker/Dockerfile-yarn .") diff --git a/README.md b/README.md index 779ca56be..a5b619988 100644 --- a/README.md +++ b/README.md @@ -369,9 +369,8 @@ LOG_LEVEL=trace A test environment with separate PostgreSQL and Redis instances can be created using the Docker Compose file in the [`test`](/packages/backend/test) folder. The `scripts` section in the [`package.json`](/package.json) provides helpful commands: - Run `yarn build:wasm` to build the WebAssembly utilities and create the `wasm` package. This requires [Rust](https://www.rust-lang.org) and [wasm-pack](https://rustwasm.github.io/wasm-pack/) to be installed. -- Manually create a `main` entry in the `package.json` of the `wasm` package pointing to the same file as the `module` entry. - Run `yarn test:tools` to start PostgreSQL and Redis. -- Run `yarn test:prisma:migrate:deploy` to initialize the PostgreSQL database. +- Run `yarn workspace @rating-tracker/backend prisma:migrate:generate` to generate the Prisma client. - Run `yarn test` to run all tests from all packages. Additionally, the packages’ `package.json` configurations contain a `test:watch` script to run tests in watch mode. NB: The `test:tools` command sometimes fails because multiple Docker networks with the same name are created concurrently. In that case, manually delete all but one using `docker network ls` and `docker network rm …` and try again. diff --git a/docker/Dockerfile-yarn b/docker/Dockerfile-yarn index f55177c70..b106609b1 100644 --- a/docker/Dockerfile-yarn +++ b/docker/Dockerfile-yarn @@ -19,8 +19,4 @@ RUN yarn --immutable COPY packages/backend/prisma ./packages/backend/prisma RUN yarn workspace @rating-tracker/backend prisma:generate -# Initialize Prisma for tests -COPY packages/backend/test/.env ./packages/backend/test/ -RUN yarn workspace @rating-tracker/backend test:prisma:migrate:deploy - ENTRYPOINT [ "/bin/sh" ] diff --git a/package.json b/package.json index 70096b8dd..bae7bd046 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "prisma:migrate:dev": "yarn workspace @rating-tracker/backend prisma:migrate:dev", "prisma:studio": "yarn workspace @rating-tracker/backend prisma:studio", "test": "yarn workspaces foreach -pt run test", - "test:tools": "conc --kill-others \"yarn workspace @rating-tracker/backend test:tools\" -n \"\" -p \"{name}\" -c grey --timings", + "test:tools": "cat packages/backend/prisma/migrations/*/migration.sql > packages/backend/test/all_migrations.sql && conc --kill-others \"yarn workspace @rating-tracker/backend test:tools\" -n \"\" -p \"{name}\" -c grey --timings", "build": "yarn workspaces foreach -pt run build", "build:wasm": "wasm-pack build -s rating-tracker -d ../packages/wasm --release wasm && sed -E -i.bak 's/\"module\": \"([A-Za-z0-9\\-\\.]+)\",/\"main\": \"\\1\",\\\n \"module\": \"\\1\",/g ; s/^}$/}\\\n/' packages/wasm/package.json && rm packages/wasm/package.json.bak", "lint": "yarn workspaces foreach -pt run lint", diff --git a/packages/backend/package.json b/packages/backend/package.json index b01ad53d1..ae4c6a3c1 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -17,12 +17,11 @@ "dev:selenium": "docker compose -f dev/docker-compose.yml up selenium", "dev:signal": "docker compose -f dev/docker-compose.yml up signal", "dev:tools": "conc --kill-others \"yarn dev:nginx\" \"yarn dev:postgres\" \"yarn dev:redis\" \"yarn dev:selenium\" \"yarn dev:signal\" -n \",,,ﰍ,\" -p \"{name}\" -c #43972A,#336791,#D82C20,#43B02A,#4975E8 --timings", - "test:watch": "yarn test:prisma:migrate:deploy && vitest", - "test": "yarn test:prisma:migrate:deploy && vitest run --color", + "test:watch": "vitest", + "test": "vitest run --color", "test:postgres": "PGPORT=54321 docker compose -f test/docker-compose.yml up --force-recreate -V postgres-test", "test:redis": "REDISPORT=63791 docker compose -f test/docker-compose.yml up --force-recreate -V redis-test", "test:tools": "conc --kill-others \"yarn test:postgres\" \"yarn test:redis\" -n \",\" -p \"{name}\" -c #336791,#D82C20 --timings", - "test:prisma:migrate:deploy": "env $(cat test/.env) pnpify prisma migrate deploy", "prisma:studio": "pnpify prisma studio", "prisma:generate": "pnpify prisma generate", "prisma:migrate:dev": "pnpify prisma migrate dev", diff --git a/packages/backend/test/.env b/packages/backend/test/.env index 5c7809806..31778d75b 100644 --- a/packages/backend/test/.env +++ b/packages/backend/test/.env @@ -1,4 +1,4 @@ -DATABASE_URL=postgresql://rating-tracker-test:rating-tracker-test@127.0.0.1:54321/rating-tracker-test?schema=rating-tracker-test&sslmode=disable +DATABASE_URL=postgresql://rating-tracker-test:rating-tracker-test@127.0.0.1:54321/rating-tracker-test?sslmode=disable NODE_ENV=test DOMAIN=example.com diff --git a/packages/backend/test/docker-compose.yml b/packages/backend/test/docker-compose.yml index 840df061a..40064f8f4 100644 --- a/packages/backend/test/docker-compose.yml +++ b/packages/backend/test/docker-compose.yml @@ -27,4 +27,6 @@ services: POSTGRES_DB: "rating-tracker-test" POSTGRES_USER: "rating-tracker-test" POSTGRES_PASSWORD: "rating-tracker-test" + volumes: + - ./all_migrations.sql:/docker-entrypoint-initdb.d/all_migrations.sql shm_size: '256mb' diff --git a/packages/backend/test/seeds/postgres.ts b/packages/backend/test/seeds/postgres.ts index 84ad3930e..c41d17a80 100644 --- a/packages/backend/test/seeds/postgres.ts +++ b/packages/backend/test/seeds/postgres.ts @@ -486,10 +486,7 @@ const applyPostgresSeeds = async (): Promise => { throw new Error("Refusing to apply seed when not in a test environment"); } - await client.$queryRaw`TRUNCATE TABLE "rating-tracker-test"."Stock" RESTART IDENTITY CASCADE`; - await client.$queryRaw`TRUNCATE TABLE "rating-tracker-test"."User" RESTART IDENTITY CASCADE`; - await client.$queryRaw`TRUNCATE TABLE "rating-tracker-test"."Watchlist" RESTART IDENTITY CASCADE`; - await client.$queryRaw`TRUNCATE TABLE "rating-tracker-test"."_StockToWatchlist" RESTART IDENTITY CASCADE`; + await client.$queryRaw`TRUNCATE TABLE "Stock", "User", "Watchlist", "_StockToWatchlist" RESTART IDENTITY CASCADE`; await applyStockSeed(); await applyUserSeed(); From c4c805bd460dacfd4674f889cf67f496bc22fab4 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 22:46:27 +0200 Subject: [PATCH 04/17] Fix initialization of test database in docker-in-docker environment Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fbb768db9..4395b9325 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,8 +36,10 @@ node('rating-tracker-build') { stage('Start test environment') { // Create migration script from all migrations and inject IP and ports into test environment sh """ - cat packages/backend/prisma/migrations/*/migration.sql > packages/backend/test/all_migrations.sql - PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose.yml up --force-recreate -V -d + PG_MIGRATIONS=$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") + cat packages/backend/test/docker-compose.yml | grep -v all_migrations > packages/backend/test/docker-compose-dind.yml + PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec psql -c \"\$PG_MIGRATIONS\" sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env """ } @@ -150,7 +152,7 @@ node('rating-tracker-build') { // Remove credentials and build artifacts sh """ docker logout - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose.yml down -t 0 + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml down -t 0 docker rmi $imagename:job$JOB_ID $imagename:job$JOB_ID-build $imagename:job$JOB_ID-test $imagename:job$JOB_ID-yarn || true docker builder prune -f --keep-storage 2G docker builder prune --builder rating-tracker -f --keep-storage 2G From 8090baaac0c298dd4ac3efd8a1f82de1bf4a5d39 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 22:47:46 +0200 Subject: [PATCH 05/17] Escape $ Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4395b9325..8785a6c6d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ node('rating-tracker-build') { stage('Start test environment') { // Create migration script from all migrations and inject IP and ports into test environment sh """ - PG_MIGRATIONS=$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") + PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec psql -c \"\$PG_MIGRATIONS\" From 361bef3ac7fb688f54b94bec2c04d7870d472e2b Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 22:49:54 +0200 Subject: [PATCH 06/17] Fix docker-compose-dind.yml Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8785a6c6d..5727efca1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ node('rating-tracker-build') { // Create migration script from all migrations and inject IP and ports into test environment sh """ PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") - cat packages/backend/test/docker-compose.yml | grep -v all_migrations > packages/backend/test/docker-compose-dind.yml + cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec psql -c \"\$PG_MIGRATIONS\" sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env From 1c4f1ba59e83695a16f7ccba7607411268983782 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 22:52:44 +0200 Subject: [PATCH 07/17] Fix Docker Compose Exec command Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5727efca1..9095d8000 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec psql -c \"\$PG_MIGRATIONS\" + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql -c \"\$PG_MIGRATIONS\" sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env """ } From 88f7b176f412d32962cb1e3212a29b74110435ee Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 22:58:46 +0200 Subject: [PATCH 08/17] Wait for open PostgreSQL port Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 9095d8000..9225d16d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d + while ! nc -z 172.17.0.1 $PGPORT ; do sleep 1 ; done docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql -c \"\$PG_MIGRATIONS\" sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env """ From 291803a1f8cce7b5b9366052e30bb4a16c86f9d4 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:06:13 +0200 Subject: [PATCH 09/17] Use database URL to connect to test database Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9225d16d1..d2acaa44a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,12 +36,12 @@ node('rating-tracker-build') { stage('Start test environment') { // Create migration script from all migrations and inject IP and ports into test environment sh """ + sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env + eval \$(cat packages/backend/test/.env | grep DATABASE_URL) PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - while ! nc -z 172.17.0.1 $PGPORT ; do sleep 1 ; done - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql -c \"\$PG_MIGRATIONS\" - sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\" """ } }, From 3c6570d9ca81d03b2813b3ed88db8ab2575f4ed5 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:16:10 +0200 Subject: [PATCH 10/17] Wait for PostgreSQL to accept initialization script Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d2acaa44a..5a4a5827a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\" + while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } }, From 32488538b6e3017a73b48690fc9a830b2d92edb9 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:24:20 +0200 Subject: [PATCH 11/17] Use a different approach Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5a4a5827a..95e22debf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,13 +35,15 @@ node('rating-tracker-build') { testenv: { stage('Start test environment') { // Create migration script from all migrations and inject IP and ports into test environment + // Since the Jenkins agent is running in a Docker container itself, we cannot mount the script directly sh """ sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env eval \$(cat packages/backend/test/.env | grep DATABASE_URL) PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql + # while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } }, From f127d91cc94d85ac86a3eaeb65180d21773b94eb Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:27:13 +0200 Subject: [PATCH 12/17] Fix command Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 95e22debf..9b0e58e4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test eval 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' # while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } From fc8487e028bcd55fbc4b533565986a4908ebb0b5 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:28:45 +0200 Subject: [PATCH 13/17] Fix command Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9b0e58e4f..0922ff82e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test eval 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test /bin/sh -c 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' # while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } From 8737f22eea385c707cd44d00de7b8d259734f4e9 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:30:07 +0200 Subject: [PATCH 14/17] Run command as root Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0922ff82e..9adf98d69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test /bin/sh -c 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' + docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec -u root postgres-test /bin/sh -c 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' # while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } From a62ffe1ec604d45a51e19ebbf3a003c6ef7f558a Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:32:24 +0200 Subject: [PATCH 15/17] Revert idea Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9adf98d69..58e12ae19 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,8 +42,7 @@ node('rating-tracker-build') { PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d - docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec -u root postgres-test /bin/sh -c 'echo \$PG_MIGRATIONS > /docker-entrypoint-initdb.d/all_migrations.sql' - # while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done + while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done """ } }, From 25752db40f54b93858fe5a4b3486ec6ade9b6cf6 Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:33:58 +0200 Subject: [PATCH 16/17] Remove newlines from migration script Signed-off-by: Marvin A. Ruder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 58e12ae19..25d74f0c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ node('rating-tracker-build') { sh """ sed -i \"s/127.0.0.1/172.17.0.1/ ; s/54321/$PGPORT/ ; s/63791/$REDISPORT/\" packages/backend/test/.env eval \$(cat packages/backend/test/.env | grep DATABASE_URL) - PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\") + PG_MIGRATIONS=\$(cat packages/backend/prisma/migrations/*/migration.sql | grep -v \"^--\" | tr -d '\\\n') cat packages/backend/test/docker-compose.yml | grep -v all_migrations | grep -v volumes > packages/backend/test/docker-compose-dind.yml PGPORT=$PGPORT REDISPORT=$REDISPORT docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml up --force-recreate -V -d while ! docker compose -p rating-tracker-test-job$JOB_ID -f packages/backend/test/docker-compose-dind.yml exec postgres-test psql \$DATABASE_URL -c \"\$PG_MIGRATIONS\"; do sleep 1 ; done From 962dc9b461eb1a73c7ccce4f0cacd074f7058e8f Mon Sep 17 00:00:00 2001 From: "Marvin A. Ruder" Date: Fri, 11 Aug 2023 23:51:41 +0200 Subject: [PATCH 17/17] Fix path Signed-off-by: Marvin A. Ruder --- docker/Dockerfile-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile-build b/docker/Dockerfile-build index 2a911e525..0091b4aa5 100644 --- a/docker/Dockerfile-build +++ b/docker/Dockerfile-build @@ -7,7 +7,7 @@ WORKDIR /workdir # Copy project files as well as Swagger UI files and WebAssembly package COPY --from=marvinruder/rating-tracker:wasm /workdir/pkg packages/wasm -COPY .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui.css .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-bundle.js .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-standalone-preset.js /workdir/app/packages/backend/dist/public/api-docs/ +COPY .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui.css .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-bundle.js .yarn/unplugged/swagger-ui-dist-*/node_modules/swagger-ui-dist/swagger-ui-standalone-preset.js /workdir/app/public/api-docs/ COPY . . RUN \