Use Regex to optimize node_module transpiling (codesandbox#794)

* Use Regex to optimize node_module transpiling

* IE11 support

Author: CompuIves

packages/app/src/sandbox/eval/manager.js

@@ -22,6 +22,7 @@ import DependencyNotFoundError from '../errors/dependency-not-found-error';
 import ModuleNotFoundError from '../errors/module-not-found-error';
 import TestRunner from './tests/jest-lite';
 import dependenciesToQuery from '../npm/dependencies-to-query';
+import isESModule from './utils/is-es-module';
 
 type Externals = {
   [name: string]: string,
@@ -186,7 +187,7 @@ export default class Manager {
 
       // Check if module syntax, only transpile when that's NOT the case
       // TODO move this check to the packager
-      if (!/^(import|export)\s/gm.test(module.code)) {
+      if (!isESModule(module.code)) {
         module.requires = this.manifest.contents[path].requires;
       }
 
@@ -734,7 +735,15 @@ export default class Manager {
    */
   async save() {
     try {
-      await localforage.setItem(this.id, this.serialize());
+      const serialized = this.serialize();
+      if (process.env.NODE_ENV === 'development') {
+        debug(
+          'Saving cache of ' +
+            (JSON.stringify(serialized).length / 1024).toFixed(2) +
+            ' kb'
+        );
+      }
+      await localforage.setItem(this.id, serialized);
     } catch (e) {
       if (process.env.NODE_ENV === 'development') {
         console.error(e);

packages/app/src/sandbox/eval/transpilers/babel/worker/__snapshots__/get-require-statements.test.js.snap

@@ -35,6 +35,15 @@ Array [
 ]
 `;
 
+exports[`get-require-statements can find reexports 1`] = `
+Array [
+  Object {
+    "path": "./Hello",
+    "type": "direct",
+  },
+]
+`;
+
 exports[`get-require-statements can find simple requires 1`] = `
 Array [
   Object {

packages/app/src/sandbox/eval/transpilers/babel/worker/babel-worker.js

@@ -9,6 +9,7 @@ import infiniteLoops from './plugins/babel-plugin-transform-prevent-infinite-loo
 import dynamicCSSModules from './plugins/babel-plugin-dynamic-css-modules';
 
 import { buildWorkerError } from '../../utils/worker-error-handler';
+import isESModule from '../../../utils/is-es-module';
 import getDependencies from './get-require-statements';
 
 import { evaluateFromPath, resetCache } from './evaluate';
@@ -17,6 +18,8 @@ import {
   getPrefixedPresetName,
 } from './get-prefixed-name';
 
+import regexGetRequireStatements from './simple-get-require-statements';
+
 let fsInitialized = false;
 let lastConfig = null;
 
@@ -226,6 +229,27 @@ self.addEventListener('message', async event => {
     return;
   }
 
+  // When we find a node_module that already is commonjs we will just get the
+  // dependencies from the file and return the same code. We get the dependencies
+  // with a regex since commonjs modules just have `require` and regex is MUCH
+  // faster than generating an AST from the code.
+  if (path.startsWith('/node_modules') && !isESModule(code)) {
+    regexGetRequireStatements(code).forEach(dependency => {
+      self.postMessage({
+        type: 'add-dependency',
+        path: dependency.path,
+        isGlob: dependency.type === 'glob',
+      });
+    });
+
+    self.postMessage({
+      type: 'result',
+      transpiledCode: code,
+    });
+
+    return;
+  }
+
   const { disableCodeSandboxPlugins } = loaderOptions;
 
   const babelUrl = babelTranspilerOptions && babelTranspilerOptions.babelURL;

packages/app/src/sandbox/eval/transpilers/babel/worker/get-require-statements.test.js

@@ -58,6 +58,14 @@ describe('get-require-statements', () => {
     testAst(code);
   });
 
+  it('can find reexports', () => {
+    const code = `
+      export * from './Hello';
+    `;
+
+    testAst(code);
+  });
+
   it('can work with real life code', () => {
     const code = `
     import React from 'react'

packages/app/src/sandbox/eval/transpilers/babel/worker/index.js

@@ -1,3 +1,7 @@
+require('core-js/fn/string/starts-with');
+require('core-js/fn/string/ends-with');
+require('core-js/fn/array/find');
+
 self.importScripts(
   `${process.env.CODESANDBOX_HOST}/static/browserfs/browserfs.min.js`
 );

packages/app/src/sandbox/eval/transpilers/babel/worker/simple-get-require-statements.js

@@ -0,0 +1,31 @@
+/**
+ * This is the regex version of getting all require statements, it makes the assumption
+ * that the file is commonjs and only has `require()` statements.
+ */
+export default function getRequireStatements(code: string) {
+  const results = [];
+  code.split('\n').forEach(line => {
+    const regex = /require\(['|"|`]([^"|'|`]*)['|"|`]\)|require\((.*)\)/;
+    const match = line.match(regex);
+
+    if (match) {
+      if (match[1]) {
+        if (!results.find(r => r.type === 'direct' && r.path === match[1])) {
+          results.push({
+            type: 'direct',
+            path: match[1],
+          });
+        }
+      } else if (match[2]) {
+        if (!results.find(r => r.type === 'glob' && r.path === match[2])) {
+          results.push({
+            type: 'glob',
+            path: match[2],
+          });
+        }
+      }
+    }
+  });
+
+  return results;
+}

packages/app/src/sandbox/eval/transpilers/typescript/__snapshots__/get-require-statements.test.js.snap

@@ -35,6 +35,15 @@ Array [
 ]
 `;
 
+exports[`get-require-statements can find reexports 1`] = `
+Array [
+  Object {
+    "path": "./Hello",
+    "type": "direct",
+  },
+]
+`;
+
 exports[`get-require-statements can find relative imports 1`] = `
 Array [
   Object {

packages/app/src/sandbox/eval/transpilers/typescript/get-require-statements.test.js

@@ -14,6 +14,21 @@ describe('get-require-statements', () => {
     expect(getRequireStatements(sourceFile, ts)).toMatchSnapshot();
   }
 
+  function testRegex(code) {
+    const results = [];
+
+    code.split('\n').forEach(line => {
+      const regex = /import\s.*['|"|`](.*)['|"|`]|require\((.*)\)|import\((.*)\)/;
+
+      const [, ...results] = line.match(regex);
+
+      const path = results.find(Boolean);
+      results.push({ type: 'direct', path });
+    });
+
+    return results;
+  }
+
   it('can find simple requires', () => {
     const code = `
       import React from 'react';
@@ -49,6 +64,14 @@ describe('get-require-statements', () => {
     testAst(code);
   });
 
+  it('can find reexports', () => {
+    const code = `
+      export * from './Hello';
+    `;
+
+    testAst(code);
+  });
+
   it('can find relative imports', () => {
     const code = `
     import Hello from './Hello';

packages/app/src/sandbox/eval/utils/is-es-module.js

@@ -0,0 +1,3 @@
+export default function isESModule(code: string) {
+  return /^(import|export)\s/gm.test(code);
+}

packages/common/version.js

@@ -14,7 +14,11 @@ const versionNumber = Math.floor(preval`module.exports = Date.now();` / 1000);
 
 const shortCommitSha = preval(`
 var execSync = require('child_process').execSync;
-module.exports = execSync('git rev-parse --short HEAD').toString().trim();
+try {
+  module.exports = execSync('git rev-parse --short HEAD').toString().trim();
+} catch (e) {
+  module.exports = 'unknown';
+}
 `);
 
 export default preval(