feat: improved build pipeline

.github/workflows/main.yml

@@ -43,6 +43,7 @@ jobs:
 
   browser-tests:
     strategy:
+      fail-fast: false
       matrix:
         command: [development, production, fastboot]
         launcher: [Chrome, IE, Firefox, Safari]

package.json

@@ -24,7 +24,8 @@
       "**/@skyrocketjs/schema",
       "**/@skyrocketjs/compiler",
       "**/@skyrocketjs/service",
-      "**/@skyrocketjs/ember"
+      "**/@skyrocketjs/ember",
+      "**/regenerator-runtime"
     ]
   },
   "engines": {

packages/compiler/package.json

@@ -15,13 +15,16 @@
     "@babel/plugin-proposal-class-properties": "^7.7.0",
     "@babel/plugin-proposal-decorators": "^7.7.0",
     "@babel/plugin-syntax-decorators": "^7.2.0",
+    "@rollup/plugin-babel": "^5.0.2",
     "@rollup/plugin-node-resolve": "^8.0.0",
     "@skyrocketjs/schema": "0.0.1-alpha.3",
     "@skyrocketjs/worker": "0.0.1-alpha.3",
     "broccoli-babel-transpiler": "^7.3.0",
     "broccoli-debug": "^0.6.5",
     "broccoli-funnel": "^3.0.0",
+    "broccoli-merge-trees": "^4.2.0",
     "broccoli-rollup": "^4.1.1",
+    "regenerator-runtime": "^0.13.5",
     "tmp": "^0.2.1"
   },
   "devDependencies": {

packages/compiler/src/plugin/index.ts

@@ -19,24 +19,105 @@ const tmp = require('tmp');
 
 tmp.setGracefulCleanup();
 
-module.exports = function compile(node: any, options: { projectRoot: string }) {
+interface BroccoliSchemaCompilerOptions {
+  parentRoot: string;
+  babelOptions?: any;
+  isProduction?: boolean;
+  compilerOptions: {
+    directoryToCompile: string;
+    node_modules_path: string;
+    importPaths: string[];
+    enableChunks?: boolean;
+  };
+}
+
+function configurePlugins(babelOptions: any, plugins: any[]): any {
+  let babelPlugins = (babelOptions.plugins = babelOptions.plugins || []);
+  plugins.forEach(plugin => {
+    let pluginPath = plugin[0];
+    let pluginOptions = plugin[1] || {};
+
+    for (let i = 0; i < babelPlugins.length; i++) {
+      let existingPlugin = babelPlugins[i];
+      if (typeof existingPlugin === 'string') {
+        if (existingPlugin === pluginPath) {
+          babelPlugins[i] = plugin;
+          return;
+        }
+      }
+      if (Array.isArray(existingPlugin) && existingPlugin[0] === pluginPath) {
+        existingPlugin[1] = Object.assign(existingPlugin[1] || {}, pluginOptions);
+        return;
+      }
+    }
+
+    // plugin not found
+    babelPlugins.push(plugin);
+  });
+
+  let disallowedPlugins = [
+    '@babel/plugin-transform-modules-amd',
+    'babel-plugin-module-resolver',
+    'babel-plugin-ember-modules-api-polyfill',
+  ];
+
+  // remove the AMD plugin
+  for (let i = 0; i < babelPlugins.length; i++) {
+    let existingPlugin = babelPlugins[i];
+    let pluginName =
+      typeof existingPlugin === 'string' ? existingPlugin : Array.isArray(existingPlugin) ? existingPlugin[0] : '';
+
+    for (let name of disallowedPlugins) {
+      if (pluginName.indexOf(name) !== -1) {
+        babelPlugins.splice(i, 1);
+        i -= 1;
+        break;
+      }
+    }
+  }
+
+  return babelOptions;
+}
+
+function enforceTrailingSlash(str: string): string {
+  if (str.charAt(str.length - 1) !== '/') {
+    return str + '/';
+  }
+  return str;
+}
+
+module.exports = function compile(options: BroccoliSchemaCompilerOptions) {
   const tmpobj = tmp.dirSync();
   /*
-    Grab just the files in workers/ for schema parsing
+    Grab all the other sources for code for workers (other than node_modules)
+  */
+  let baseTree;
+  if (options.compilerOptions.importPaths.length) {
+    baseTree = debug(
+      merge(
+        options.compilerOptions.importPaths.map(dir => {
+          return new Funnel(path.join(options.parentRoot, dir), { destDir: dir, allowEmpty: true });
+        })
+      ),
+      `compiler-input-trees`
+    );
+  }
+
+  /*
+    Grab just the files in options.directoryToCompile for schema parsing
   */
-  const schemaTree = debug(
-    new Funnel(node, {
-      srcDir: 'workers',
-      destDir: 'workers',
+  const workerTree = debug(
+    new Funnel(path.join(options.parentRoot, options.compilerOptions.directoryToCompile), {
+      destDir: options.compilerOptions.directoryToCompile,
     }),
-    'schema-tree'
+    'worker-files-for-schema-compilation'
   );
 
   /*
    Parse out the schema info into tmpobj.name directory
   */
   const parsedSchemas = debug(
-    babel(schemaTree, {
+    babel(workerTree, {
       throwUnlessParallelizable: true,
       plugins: [
         [
@@ -45,15 +126,15 @@ module.exports = function compile(node: any, options: { projectRoot: string }) {
             schemaSourceFiles: {
               '@skyrocketjs/worker': true,
             },
-            filePrefix: 'workers/',
+            filePrefix: enforceTrailingSlash(options.compilerOptions.directoryToCompile),
             outputPath: tmpobj.name,
             removeDecorators: true,
           },
         ],
-        [ParseDecorators, { decoratorsBeforeExport: false }],
+        [ParseDecorators, { legacy: true }],
       ],
     }),
-    'babel-schemas'
+    'files-post-parsing-for-schema'
   );
 
   /*
@@ -62,9 +143,9 @@ module.exports = function compile(node: any, options: { projectRoot: string }) {
   const withSchemas = debug(
     new Formatter({
       schemaPath: tmpobj.name,
-      schemaOutputDirectory: 'workers',
+      schemaOutputDirectory: options.compilerOptions.directoryToCompile,
     }),
-    'schemas'
+    'generated-schema-modules'
   );
 
   /*
@@ -74,36 +155,46 @@ module.exports = function compile(node: any, options: { projectRoot: string }) {
     new Launchers({
       schemaPath: tmpobj.name,
     }),
-    'launchers'
+    'generated-launcher-modules'
   );
 
   /*
     Our schema scan has to be "pulled" during broccoli compilation
     in order to run, so we merge it back in.
   */
-  const pullTree = debug(merge([node, parsedSchemas, withSchemas, withLaunchers], { overwrite: true }), 'pull-tree');
+  const pullTree = debug(
+    merge([baseTree, workerTree, parsedSchemas, withSchemas, withLaunchers].filter(Boolean), { overwrite: true }),
+    'pull-tree'
+  );
 
   /*
     Process our JS from app and workers in advance of rollup
     because rollup cannot handle decorators or class properties
+
+    TODO use the passed in babel config
   */
-  const parsedForRollup = debug(
-    babel(pullTree, {
-      throwUnlessParallelizable: true,
-      plugins: [[Decorators, { decoratorsBeforeExport: false }], [ClassProps]],
-    }),
-    'babel-rollup'
-  );
+  const babelConfig = configurePlugins(Object.assign(options.babelOptions || {}, { throwUnlessParallelizable: true }), [
+    [Decorators, { legacy: true }],
+    [ClassProps, { loose: true }],
+  ]);
+  console.log(babelConfig.presets[0][1].targets.browsers);
+  const parsedForRollup = debug(babel(pullTree, babelConfig), 'babel-processed-files-for-worker-rollup');
 
   /*
     Convert our worker files into stand-alone executables with the proper
     shell to communicate with the @skyrocketjs/service
+
+    TODO enable chunks. Chunks are more efficient for the build pipeline AND
+    more efficient at runtime; however, they will require rewriting `import` to `importScript`
+    including accounting for SRI fingerprints in production
+
+    TODO minify rollup output in production
   */
   const rollupTree = debug(
     rollupLaunchers(parsedForRollup, {
       cache: true, // likely need to make sure this is keyed to the output of the schemas
       schemaPath: tmpobj.name,
-      nodeModulesPath: path.join(options.projectRoot, 'node_modules'),
+      nodeModulesPath: options.compilerOptions.node_modules_path,
       rollup: {
         // these options assigned later
         input: '',
@@ -116,16 +207,27 @@ module.exports = function compile(node: any, options: { projectRoot: string }) {
         ],
       },
     }),
-    'rollup'
+    'rollup-built-workers'
   );
 
+  /*
+    Workers do not have integrity features (we could maybe build our own SRI solution)
+    but we do the minimum good thing here by allowing our assets to be revisioned with
+    a checksum fingerprint.
+
+    This means our worker service on the main thread will need to know these checksums,
+    so we insert them into the compiled schemas as the first argument.
+  */
   const togetherAgainTree = merge([rollupTree, withSchemas]);
-  const finalTree = debug(
+  const finalSchemas = debug(
     new SchemaSRIGenerator(togetherAgainTree, {
       schemaPath: tmpobj.name,
     }),
     'schemas-with-sri'
   );
 
-  return merge([rollupTree, finalTree]);
+  return {
+    workers: rollupTree,
+    schemas: finalSchemas,
+  };
 };

packages/compiler/src/tsconfig.json

@@ -10,18 +10,13 @@
     "noImplicitAny": true,
     "noEmitOnError": false,
     "strictNullChecks": true,
-    "experimentalDecorators": false,
+    "experimentalDecorators": true,
     "noEmit": false,
     "skipLibCheck": true,
     "inlineSourceMap": true,
     "inlineSources": true,
     "outDir": "../build"
   },
-  "files": [
-    "plugin/index.ts",
-    "plugin/rollup.ts",
-    "plugin/formatter.ts",
-    "plugin/utils/gather-schemas.ts"
-  ],
+  "files": ["plugin/index.ts", "plugin/rollup.ts", "plugin/formatter.ts", "plugin/utils/gather-schemas.ts"],
   "exclude": ["../node_modules/**", "../../../node_modules/**"]
 }

packages/compiler/tests/fixtures/output/schemas/workers/example-one.js

@@ -1 +1 @@
-export default '["b6d064159b6e0ad721b130300a208e6b",1,"fetchUsers",0,2,"reload",0]';
+export default '["9cc2916bda494a2271dcb24c40ec50bc",1,"fetchUsers",0,2,"reload",0]';

packages/compiler/tests/fixtures/output/schemas/workers/example-two.js

@@ -1 +1 @@
-export default '["df825ac81ebcff71433ab0ca3a14a59a",1,"fetchUsers",0,2,"reload",0]';
+export default '["e2ba2d56f43c5ebcb459467640692c87",1,"fetchUsers",0,2,"reload",0]';

packages/compiler/tests/fixtures/output/workers/example_one__launcher.js

@@ -1,12 +1,3 @@
-var __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 const SkyrocketMessageIdentifier = '-srwm';
 const SkyrocketErrorIdentifier = '-srwme';
 function createShell(Global, schema, WorkerMain) {
@@ -19,36 +10,34 @@ function createShell(Global, schema, WorkerMain) {
     const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope;
     const worker = new WorkerMain(channel, { isWorker });
     let DEF_CACHE = Object.create(null);
-    function recieve(event) {
-        return __awaiter(this, arguments, void 0, function* () {
-            const data = event.data;
-            if (Array.isArray(data) && data[0] === SkyrocketMessageIdentifier) {
-                const fieldDef = getFieldDef(data[1]);
-                switch (fieldDef[0]) {
-                    case 1: // method
-                        try {
-                            const result = yield exec(worker, data[1], data[2]);
-                            return send(result, data[3]);
-                        }
-                        catch (e) {
-                            const result = {
-                                message: e.message,
-                                stack: e.stack,
-                            };
-                            return send(result, data[3]);
-                        }
-                    case 2: // event
-                        return exec(worker, data[1], data[2]);
-                    case 3: // signal
-                        return exec(worker, data[1], data[2]);
-                    default:
-                        throw new Error(`Unknown field type`);
-                }
-            }
-            else {
-                channel.onmessage(...arguments);
+    async function recieve(event) {
+        const data = event.data;
+        if (Array.isArray(data) && data[0] === SkyrocketMessageIdentifier) {
+            const fieldDef = getFieldDef(data[1]);
+            switch (fieldDef[0]) {
+                case 1: // method
+                    try {
+                        const result = await exec(worker, data[1], data[2]);
+                        return send(result, data[3]);
+                    }
+                    catch (e) {
+                        const result = {
+                            message: e.message,
+                            stack: e.stack,
+                        };
+                        return send(result, data[3]);
+                    }
+                case 2: // event
+                    return exec(worker, data[1], data[2]);
+                case 3: // signal
+                    return exec(worker, data[1], data[2]);
+                default:
+                    throw new Error(`Unknown field type`);
             }
-        });
+        }
+        else {
+            channel.onmessage(...arguments);
+        }
     }
     function getFieldDef(name) {
         let def = DEF_CACHE[name];