From b5cc872e9e803998ba23457b10054d0c4d7caa00 Mon Sep 17 00:00:00 2001 From: Diogo Viana Date: Wed, 3 Jan 2024 01:44:06 +0000 Subject: [PATCH] Task#33 (#39) * Add use cases for quantile type. Add vegteta endpoint for parsing * add properties for success and test execution count * Remove logging * add changesets for next release * wip --- .changeset/fair-pigs-care.md | 5 +++ .changeset/pre.json | 8 ++++ .github/PULL_REQUEST_TEMPLATE.md | 5 +-- src/middlewares/slurp.middleware.ts | 5 +++ src/models/schema.ts | 3 +- src/schemas/vegeta-schema.json | 27 +++++++++++++ src/services/index.ts | 9 ++++- src/services/parsers/index.ts | 5 ++- src/services/parsers/slurper.ts | 40 +++++++++++++++++-- src/services/parsers/vegeta.slurper.ts | 15 +++++++ .../serializers/prometheus.serializer.ts | 25 ++++++------ tsconfig.json | 2 +- 12 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .changeset/fair-pigs-care.md create mode 100644 .changeset/pre.json create mode 100644 src/schemas/vegeta-schema.json create mode 100644 src/services/parsers/vegeta.slurper.ts diff --git a/.changeset/fair-pigs-care.md b/.changeset/fair-pigs-care.md new file mode 100644 index 0000000..54acfc7 --- /dev/null +++ b/.changeset/fair-pigs-care.md @@ -0,0 +1,5 @@ +--- +"@deadcow-enterprises/junit-prometheus-exporter": minor +--- + +add quantile type and vegeta endpoints diff --git a/.changeset/pre.json b/.changeset/pre.json new file mode 100644 index 0000000..cef1564 --- /dev/null +++ b/.changeset/pre.json @@ -0,0 +1,8 @@ +{ + "mode": "pre", + "tag": "next", + "initialVersions": { + "@deadcow-enterprises/junit-prometheus-exporter": "0.2.0" + }, + "changesets": [] +} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f6c381b..44ecb2b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -32,6 +32,5 @@ ### Reviewers - -@reviewer1 -@reviewer2 + + diff --git a/src/middlewares/slurp.middleware.ts b/src/middlewares/slurp.middleware.ts index a8f175d..8f25385 100644 --- a/src/middlewares/slurp.middleware.ts +++ b/src/middlewares/slurp.middleware.ts @@ -6,4 +6,9 @@ export default function registerSlurpMiddleware(instance: Express) { facadeService.parseJunit(body, q); res.status(202).send(); }); + instance.post("/slurp/vegeta", ({ body, query }, res) => { + const q = query as Record; + facadeService.parseVegeta(body, q); + res.status(202).send(); + }); } diff --git a/src/models/schema.ts b/src/models/schema.ts index 9069bcf..2de648f 100644 --- a/src/models/schema.ts +++ b/src/models/schema.ts @@ -3,7 +3,8 @@ export interface ParsingSchema { } export interface PropertySchema { - type: "variable" | "counter"; + type: "variable" | "counter" | "quantile"; + quantiles?: string[]; description: string; value: string; validLabels?: string[]; diff --git a/src/schemas/vegeta-schema.json b/src/schemas/vegeta-schema.json new file mode 100644 index 0000000..df2b45a --- /dev/null +++ b/src/schemas/vegeta-schema.json @@ -0,0 +1,27 @@ +{ + "vegeta_load_test_latency_ms": { + "type": "quantile", + "description": "Load test latency in milliseconds", + "value": "{latencies.[quantile]} / 1000000", + "quantiles": ["mean", "max", "50th", "95th", "99th"], + "validLabels": ["project", "team", "version"], + "labelEquality": "{currentLabels.project} == {newLabels.project} && {newLabels.team} == {currentLabels.team}", + "labelEqualityResolution": "replace" + }, + "vegeta_load_test_success_percent_ms": { + "type": "variable", + "description": "Total number of tests passing", + "value": "{success}", + "defaultValue": 0, + "validLabels": ["project", "team", "version"], + "labelEquality": "{currentLabels.project} == {newLabels.project} && {newLabels.team} == {currentLabels.team}", + "labelEqualityResolution": "replace" + }, + "vegeta_load_test_executions": { + "type": "counter", + "description": "Total tests runs", + "validLabels": ["project", "team", "version"], + "labelEquality": "{currentLabels.project} == {newLabels.project} && {newLabels.team} == {currentLabels.team}", + "labelEqualityResolution": "replace" + } +} diff --git a/src/services/index.ts b/src/services/index.ts index 9aab9dd..55e8544 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,21 +1,25 @@ -import { Property } from "./parsers"; +import { Property, VegetaSlurper } from "./parsers"; import { JunitSlurper } from "./parsers/junit.slurper"; import { JsonSerializer, PrometheusSerializer } from "./serializers"; export class FacadeService { constructor( private readonly junit: JunitSlurper, + private readonly vegeta: VegetaSlurper, private readonly prometheus: PrometheusSerializer, private readonly json: JsonSerializer ) {} private get properties(): Property[] { - return [...this.junit.properties]; + return [...this.junit.properties, ...this.vegeta.properties]; } parseJunit(content: any, labels: Record) { this.junit.parse(content, labels); } + parseVegeta(content: any, labels: Record) { + this.vegeta.parse(content, labels); + } toPrometheus(): string { return this.prometheus.serialize(...this.properties); @@ -33,6 +37,7 @@ export const facadeService = globalForServices.facadeService ?? new FacadeService( new JunitSlurper(), + new VegetaSlurper(), new PrometheusSerializer(), new JsonSerializer() ); diff --git a/src/services/parsers/index.ts b/src/services/parsers/index.ts index 1d1a423..909d10a 100644 --- a/src/services/parsers/index.ts +++ b/src/services/parsers/index.ts @@ -1,2 +1,3 @@ -export * from './junit.slurper'; -export { type Property } from './slurper'; +export * from "./junit.slurper"; +export { type Property } from "./slurper"; +export * from "./vegeta.slurper"; diff --git a/src/services/parsers/slurper.ts b/src/services/parsers/slurper.ts index 659908d..b4bdcec 100644 --- a/src/services/parsers/slurper.ts +++ b/src/services/parsers/slurper.ts @@ -14,6 +14,7 @@ type PropertyParser = { type: PropertySchema["type"]; name: string; pattern: Array any)>; + extraLabels?: Labels; validLabels?: PropertySchema["validLabels"]; equalityResolution?: PropertySchema["labelEqualityResolution"]; equality: Array any)>; @@ -44,6 +45,29 @@ export abstract class Slurper { // if property parser already exists then we don't need to generate it again if (this.parsers.findIndex((g) => g.name === name) >= 0) return; const curr = this.schema[name]; + if (curr.type === "quantile") { + if (!curr.quantiles) + throw new Error(`Property ${name} is missing quantiles`); + for (const quantile of curr.quantiles) { + this.generatePropertyParserForSingle( + name, + { + ...curr, + value: curr.value.replaceAll("[quantile]", quantile), + }, + { quantile } + ); + } + } else { + this.generatePropertyParserForSingle(name, curr); + } + } + + private generatePropertyParserForSingle( + name: string, + curr: PropertySchema, + extraLabels?: Labels + ) { const pattern = this.getValuePattern(curr); const labelPattern = this.getLabelPattern(curr); @@ -53,6 +77,7 @@ export abstract class Slurper { pattern, validLabels: curr.validLabels, equality: labelPattern, + extraLabels, equalityResolution: curr.labelEqualityResolution, }); } @@ -67,6 +92,7 @@ export abstract class Slurper { ); break; case "variable": + case "quantile": // normalize instructions by splitting them by space and removing empty strings const instructions = curr.value .split(" ") @@ -128,9 +154,10 @@ export abstract class Slurper { return labelPattern; } - parse(content: T, labels: Labels): void { + parse(content: T, _labels: Labels): void { const values: Labels = {}; this.parsers.forEach((g) => { + const labels = { ..._labels, ...g.extraLabels }; const curr = this._properties.find((p) => p.name === g.name)!; const valueIndex = this.getLabelIndex(curr, g, labels); @@ -150,7 +177,9 @@ export abstract class Slurper { if (g.equalityResolution === "replace") { curr.values[valueIndex].labels = this.getValidLabels( labels, - g.validLabels + g.validLabels?.concat( + g.extraLabels ? Object.keys(g.extraLabels) : [] + ) ); curr.values[valueIndex].value = result; return; @@ -158,7 +187,10 @@ export abstract class Slurper { } curr.values.push({ - labels: this.getValidLabels(labels, g.validLabels), + labels: this.getValidLabels( + labels, + g.validLabels?.concat(g.extraLabels ? Object.keys(g.extraLabels) : []) + ), value: result, }); }); @@ -186,6 +218,8 @@ export abstract class Slurper { if (!equal) { continue; } + if (p.type === "quantile" && oldLabels.quantile !== labels.quantile) + continue; return index; } return -1; diff --git a/src/services/parsers/vegeta.slurper.ts b/src/services/parsers/vegeta.slurper.ts new file mode 100644 index 0000000..2d7692a --- /dev/null +++ b/src/services/parsers/vegeta.slurper.ts @@ -0,0 +1,15 @@ +import * as fs from "fs"; +import * as path from "path"; +import { ParsingSchema } from "../../models/schema"; +import { Slurper } from "./slurper"; + +export class VegetaSlurper extends Slurper { + constructor(filePath?: string) { + const fileLocation = + filePath || path.join(__dirname, "../../schemas/vegeta-schema.json"); + const fileStr = fs.readFileSync(fileLocation); + + const schema = JSON.parse(fileStr.toString()); + super(schema as ParsingSchema); + } +} diff --git a/src/services/serializers/prometheus.serializer.ts b/src/services/serializers/prometheus.serializer.ts index 1eb4121..748938d 100644 --- a/src/services/serializers/prometheus.serializer.ts +++ b/src/services/serializers/prometheus.serializer.ts @@ -1,5 +1,5 @@ -import { Property } from '../parsers'; -import { Serializer } from './serializer'; +import { Property } from "../parsers"; +import { Serializer } from "./serializer"; export class PrometheusSerializer extends Serializer { serialize(...data: Property[]): string { @@ -12,28 +12,29 @@ export class PrometheusSerializer extends Serializer { } ${type}\n${p.values .map( ({ labels, value }) => - `${p.name}${this.parseLabels(labels)} ${value}`, + `${p.name}${this.parseLabels(labels)} ${value}` ) - .join('\n')}`; + .join("\n")}`; }) - .join('\n\n'); + .join("\n\n"); } parseLabels(labels: Record) { if (Object.keys(labels).length === 0) { - return ''; + return ""; } return `{${Object.keys(labels) .map((l) => `${l}="${labels[l]}"`) - .join(', ')}}`; + .join(", ")}}`; } - propertyMapToPrometheusType(p: Property['type']): string { + propertyMapToPrometheusType(p: Property["type"]): string { switch (p) { - case 'counter': - return 'counter'; - case 'variable': - return 'gauge'; + case "counter": + return "counter"; + case "variable": + case "quantile": + return "gauge"; } } } diff --git a/tsconfig.json b/tsconfig.json index f655b4f..0c405f0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "require": ["tsconfig-paths/register"] }, "compilerOptions": { - "lib": ["es5", "es6", "es7"], + "lib": ["es5", "es6", "es7", "es2021"], "target": "es2017", "module": "commonjs", "moduleResolution": "node",