Feat: support skipping tests (#419)

* feat: support skipping tests

Skipping tests can be useful in a CI environment where specific tests
can be disabled viam moonwall config

* update biome config

* test: add tests to skip smoke tests

* fix lint errors

* add test execution to ci

* fix: regexp and execution

* test: add test cases

* fix lint errors

* remove unused import

.github/workflows/main.yml

@@ -110,7 +110,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: ["dev_test", "dev_multi", "dev_seq"]
+        suite: ["dev_test", "dev_multi", "dev_seq", "dev_smoke"]
     steps:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v1
@@ -244,3 +244,4 @@ jobs:
         run: |
           cd test
           bun moonwall test ${{matrix.suite}}
+

biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/1.6.1/schema.json",
+  "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
   "files": {
     "include": ["*.js", "*.ts", "*.json", "*.yml", "*.md"],
     "ignore": ["./dist/*", "./node_modules/*", "package.json"]
@@ -13,7 +13,7 @@
   },
   "javascript": {
     "formatter": {
-      "trailingComma": "es5",
+      "trailingCommas": "es5",
       "semicolons": "always",
       "indentStyle": "space",
       "lineWidth": 100

packages/cli/src/cmds/runTests.ts

@@ -106,6 +106,12 @@ export async function executeTests(env: Environment, additionalArgs?: any) {
       },
     } satisfies UserConfig;
 
+    // transform  in regexp pattern
+    if (env.skipTests && env.skipTests.length > 0) {
+      // the final pattern will look like this: "^((?!SO00T02|SM00T01|SM00T03).)*$"
+      additionalArgs.testNamePattern = `^((?!${env.skipTests?.map((test) => `${test.name}`).join("|")}).)*$`;
+    }
+
     // TODO: Create options builder class
     const options = addThreadConfig(baseOptions, env.multiThreads);
 

packages/types/config_schema.json

@@ -631,6 +631,27 @@
             },
             "type": "array"
           },
+          "skipTests": {
+            "description": "A list of test to skip.",
+            "items": {
+              "properties": {
+                "name": {
+                  "description": "The name of the test to skip. Eg. S22C500",
+                  "type": "string"
+                },
+                "reason": {
+                  "description": "The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123.",
+                  "type": "string"
+                },
+                "since": {
+                  "description": "The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z",
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            },
+            "type": "array"
+          },
           "testFileDir": {
             "description": "An array of directories with test files.",
             "items": {

packages/types/src/config.ts

@@ -127,6 +127,28 @@ export type Environment = {
    * Toggle whether createBlock() will finalize blocks by default or not.
    */
   defaultFinalization?: boolean;
+
+  /**
+   * A list of test to skip.
+   */
+  skipTests?: SkipTestSpec[];
+};
+
+export type SkipTestSpec = {
+  /**
+   * The name of the test to skip. Eg. S22C500
+   */
+  name: string;
+
+  /**
+   * The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123.
+   */
+  reason: string;
+
+  /**
+   * The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z
+   */
+  since: string;
 };
 
 /**

test/moonwall.config.json

@@ -794,6 +794,40 @@
       },
       "connections": []
     },
+    {
+      "name": "dev_smoke",
+      "foundation": {
+        "launchSpec": {
+          "disableRuntimeVersionCheck": true
+        },
+        "type": "read_only"
+      },
+      "connections": [
+        {
+          "name": "para",
+          "type": "polkadotJs",
+          "endpoints": ["wss://moonbeam-rpc.dwellir.com"]
+        }
+      ],
+      "testFileDir": ["/suites/smoke"],
+      "skipTests": [
+        {
+          "name": "SO00T02",
+          "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/1",
+          "since": "2024-01-28T00:00:00Z"
+        },
+        {
+          "name": "SM00T01",
+          "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/2",
+          "since": "2024-01-28T00:00:00Z"
+        },
+        {
+          "name": "SM00T03",
+          "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/3",
+          "since": "2024-01-28T00:00:00Z"
+        }
+      ]
+    },
     {
       "name": "multi_chopsticks",
       "testFileDir": ["suites/multi_chopsticks"],

test/suites/smoke/test_skipMany.ts

@@ -0,0 +1,32 @@
+import { describeSuite, expect } from "@moonwall/cli";
+
+describeSuite({
+  id: "SM00",
+  title: "New Test Suite",
+  foundationMethods: "read_only",
+  testCases: ({ it }) => {
+    it({
+      id: "T01",
+      title: "Skipped Test",
+      test: () => {
+        expect(false).to.be.true;
+      },
+    });
+
+    it({
+      id: "T02",
+      title: "Passing test",
+      test: () => {
+        expect(true).to.be.true;
+      },
+    });
+
+    it({
+      id: "T03",
+      title: "Skipped test",
+      test: () => {
+        expect(false).to.be.true;
+      },
+    });
+  },
+});

test/suites/smoke/test_skipNone.ts

@@ -0,0 +1,24 @@
+import { describeSuite, expect } from "@moonwall/cli";
+
+describeSuite({
+  id: "SN00",
+  title: "New Test Suite",
+  foundationMethods: "read_only",
+  testCases: ({ it }) => {
+    it({
+      id: "T01",
+      title: "Passing Test",
+      test: () => {
+        expect(true).to.be.true;
+      },
+    });
+
+    it({
+      id: "T02",
+      title: "Passing test",
+      test: () => {
+        expect(true).to.be.true;
+      },
+    });
+  },
+});

test/suites/smoke/test_skipOne.ts

@@ -0,0 +1,24 @@
+import { describeSuite, expect } from "@moonwall/cli";
+
+describeSuite({
+  id: "SO00",
+  title: "New Test Suite",
+  foundationMethods: "read_only",
+  testCases: ({ it }) => {
+    it({
+      id: "T01",
+      title: "Passing Test",
+      test: () => {
+        expect(true).to.be.true;
+      },
+    });
+
+    it({
+      id: "T02",
+      title: "Skipped test",
+      test: () => {
+        expect(false).to.be.true;
+      },
+    });
+  },
+});