Improve esmodules loading by writing a custom converter (codesandbox#3370)

* Improve esmodules loading by writing a custom converter

* Polishment

* Fix case

* Fix issue

* Big perf improvements

* Improve support of metrics

* Fix types

* Fix linting issues

* Fix default interop

* Cleanup

* Fix edge case, add benchmarking

* Fix typings

* Force esmodules

* Fix recursive loops

* Fix reexport with as

* Tweaks to performance and benchmarking

* Fix babel listing

* Fix dynamically downloading modules

* Cleanup reading tmodule and fixing resolving async files

* Update measurement keys

* Fix resolver

* Force prod packager for development as well

* Revert "Force prod packager for development as well"

This reverts commit b95e40b.

* Use new packger

* Fix urls

* Fix showing transpiler errors

* Fix vue

* Remove indirection of metrics

Author: CompuIves

packages/app/package.json

@@ -86,6 +86,7 @@
     "apollo-boost": "^0.4.4",
     "apollo-link-batch-http": "^1.2.12",
     "apollo-link-context": "^1.0.18",
+    "astring": "^1.4.3",
     "axios": "^0.19.0",
     "babel-code-frame": "^6.26.0",
     "babel-macros": "^2.0.0",
@@ -115,6 +116,7 @@
     "downshift": "^1.0.0-rc.14",
     "eslint-plugin-react-hooks": "1.6.0",
     "eslint-plugin-vue": "^4.2.2",
+    "estree-walker": "^1.0.1",
     "file-saver": "^1.3.3",
     "fontfaceobserver": "^2.0.13",
     "fuse.js": "^3.2.1",
@@ -146,6 +148,7 @@
     "lodash-es": "^4.17.2",
     "lru-cache": "^4.1.3",
     "match-sorter": "^1.8.1",
+    "meriyah": "^1.9.7",
     "minimatch": "^3.0.4",
     "mobx": "^5.11.0",
     "mobx-react": "^6.1.1",

packages/app/src/sandbox/compile.ts

@@ -495,7 +495,7 @@ async function compile({
         showFullScreen: firstLoad,
       }
     );
-    metrics.endMeasure('dependencies', 'Dependencies');
+    metrics.endMeasure('dependencies', { displayName: 'Dependencies' });
 
     const shouldReloadManager =
       (isNewCombination && !firstLoad) || manager.id !== sandboxId;
@@ -547,7 +547,7 @@ async function compile({
     await manager.verifyTreeTranspiled();
     await manager.transpileModules(managerModuleToTranspile);
 
-    metrics.endMeasure('transpilation', 'Transpilation');
+    metrics.endMeasure('transpilation', { displayName: 'Transpilation' });
 
     dispatch({ type: 'status', status: 'evaluating' });
     manager.setStage('evaluation');
@@ -612,14 +612,16 @@ async function compile({
 
       metrics.measure('external-resources');
       await handleExternalResources(externalResources);
-      metrics.endMeasure('external-resources', 'External Resources');
+      metrics.endMeasure('external-resources', {
+        displayName: 'External Resources',
+      });
 
       const oldHTML = document.body.innerHTML;
       metrics.measure('evaluation');
       const evalled = manager.evaluateModule(managerModuleToTranspile, {
         force: isModuleView,
       });
-      metrics.endMeasure('evaluation', 'Evaluation');
+      metrics.endMeasure('evaluation', { displayName: 'Evaluation' });
 
       const domChanged =
         !manager.preset.htmlDisabled && oldHTML !== document.body.innerHTML;
@@ -669,8 +671,8 @@ async function compile({
 
     debug(`Total time: ${Date.now() - startTime}ms`);
 
-    metrics.endMeasure('compilation', 'Compilation');
-    metrics.endMeasure('total', 'Total', { lastTime: 0 });
+    metrics.endMeasure('compilation', { displayName: 'Compilation' });
+    metrics.endMeasure('total', { displayName: 'Total', lastTime: 0 });
     dispatch({
       type: 'success',
     });

packages/app/src/sandbox/eval/manager.ts

@@ -9,6 +9,7 @@ import { ParsedConfigurationFiles } from '@codesandbox/common/lib/templates/temp
 import DependencyNotFoundError from 'sandbox-hooks/errors/dependency-not-found-error';
 import ModuleNotFoundError from 'sandbox-hooks/errors/module-not-found-error';
 
+import { generateBenchmarkInterface } from '../utils/benchmark';
 import { Module } from './types/module';
 import TranspiledModule, {
   ChildModule,
@@ -28,6 +29,7 @@ import { packageFilter } from './utils/resolve-utils';
 import { ignoreNextCache, deleteAPICache, clearIndexedDBCache } from './cache';
 import { shouldTranspile } from './transpilers/babel/check';
 import { splitQueryFromPath } from './utils/query-path';
+import { measure, endMeasure } from '../utils/metrics';
 
 declare const BrowserFS: any;
 
@@ -122,7 +124,10 @@ export default class Manager {
   fileResolver: IFileResolver | undefined;
 
   // List of modules that are being transpiled, to prevent duplicate jobs.
-  transpileJobs: { [transpiledModuleId: string]: true };
+  transpileJobs: {
+    [transpiledModuleId: string]: true | Promise<TranspiledModule>;
+  };
+
   transpiledModulesByHash: { [hash: string]: TranspiledModule };
 
   // All paths are resolved at least twice: during transpilation and evaluation.
@@ -161,6 +166,9 @@ export default class Manager {
     if (process.env.NODE_ENV === 'development') {
       // eslint-disable-next-line no-console
       console.log(this);
+
+      // Initialize benchmark logic
+      getGlobal().Benchmark = generateBenchmarkInterface(this);
     }
 
     BrowserFS.configure(
@@ -266,7 +274,7 @@ export default class Manager {
       return this.fileResolver.readFile(p).then(code => {
         this.addModule({ code, path: p });
 
-        callback(code);
+        callback(null, code);
       });
     }
 
@@ -499,6 +507,10 @@ export default class Manager {
     this.getTranspiledModules().map(tModule => tModule.resetCompilation());
   }
 
+  clearTranspilationCache() {
+    this.getTranspiledModules().map(tModule => tModule.resetTranspilation());
+  }
+
   getModules(): Array<Module | ChildModule> {
     return values(this.transpiledModules).map(t => t.module);
   }
@@ -587,102 +599,111 @@ export default class Manager {
     currentPath: string,
     defaultExtensions = ['js', 'jsx', 'json']
   ): Promise<Module> {
-    return new Promise(promiseResolve => {
-      const dirredPath = pathUtils.dirname(currentPath);
-      if (this.cachedPaths[dirredPath] === undefined) {
-        this.cachedPaths[dirredPath] = {};
-      }
+    const dirredPath = pathUtils.dirname(currentPath);
+    if (this.cachedPaths[dirredPath] === undefined) {
+      this.cachedPaths[dirredPath] = {};
+    }
 
-      const cachedPath = this.cachedPaths[dirredPath][path];
+    const cachedPath = this.cachedPaths[dirredPath][path];
 
-      let resolvedPath;
+    if (cachedPath) {
+      return Promise.resolve(this.transpiledModules[cachedPath].module);
+    }
 
-      if (cachedPath) {
-        resolvedPath = cachedPath;
-      } else {
-        const presetAliasedPath = this.getPresetAliasedPath(path);
+    const measureKey = `resolve-async:${path}:${currentPath}`;
+    return new Promise((promiseResolve, promiseReject) => {
+      measure(measureKey);
+      const presetAliasedPath = this.getPresetAliasedPath(path);
 
-        const aliasedPath = this.getAliasedDependencyPath(
-          presetAliasedPath,
-          currentPath
-        );
-        const shimmedPath = coreLibraries[aliasedPath] || aliasedPath;
+      const aliasedPath = this.getAliasedDependencyPath(
+        presetAliasedPath,
+        currentPath
+      );
+      const shimmedPath = coreLibraries[aliasedPath] || aliasedPath;
 
-        if (NODE_LIBS.indexOf(shimmedPath) > -1) {
-          this.cachedPaths[dirredPath][path] = shimmedPath;
-          promiseResolve(getShimmedModuleFromPath(currentPath, path));
-          return;
-        }
+      if (NODE_LIBS.indexOf(shimmedPath) > -1) {
+        this.cachedPaths[dirredPath][path] = shimmedPath;
+        promiseResolve(getShimmedModuleFromPath(currentPath, path));
+        return;
+      }
 
-        try {
-          resolve(
-            shimmedPath,
-            {
-              filename: currentPath,
-              extensions: defaultExtensions.map(ext => '.' + ext),
-              isFile: this.isFile,
-              readFileSync: this.readFileSync,
-              packageFilter,
-              moduleDirectory: this.getModuleDirectories(),
-            },
-            (err, foundPath) => {
-              if (err) {
-                throw err;
-              }
+      resolve(
+        shimmedPath,
+        {
+          filename: currentPath,
+          extensions: defaultExtensions.map(ext => '.' + ext),
+          isFile: this.isFile,
+          readFileSync: this.readFileSync,
+          packageFilter,
+          moduleDirectory: this.getModuleDirectories(),
+        },
+        (err, foundPath) => {
+          endMeasure(measureKey, { silent: true });
+          if (err) {
+            if (
+              this.cachedPaths[dirredPath] &&
+              this.cachedPaths[dirredPath][path]
+            ) {
+              delete this.cachedPaths[dirredPath][path];
+            }
 
-              this.cachedPaths[dirredPath][path] = foundPath;
+            let connectedPath = shimmedPath;
+            if (connectedPath.indexOf('/node_modules') !== 0) {
+              connectedPath = /^(\w|@\w)/.test(shimmedPath)
+                ? pathUtils.join('/node_modules', shimmedPath)
+                : pathUtils.join(pathUtils.dirname(currentPath), shimmedPath);
+            }
 
-              if (foundPath === '//empty.js') {
-                promiseResolve(getShimmedModuleFromPath(currentPath, path));
-                return;
-              }
+            const isDependency = connectedPath.includes('/node_modules/');
 
-              if (!this.transpiledModules[foundPath]) {
-                this.readFileSync(foundPath, code => {
-                  this.addModule({ path: foundPath, code });
-                  promiseResolve(this.transpiledModules[foundPath].module);
-                });
-              } else {
-                promiseResolve(this.transpiledModules[foundPath].module);
-              }
+            connectedPath = connectedPath.replace('/node_modules/', '');
+
+            if (!isDependency) {
+              promiseReject(
+                new ModuleNotFoundError(shimmedPath, false, currentPath)
+              );
             }
-          );
-        } catch (e) {
-          if (
-            this.cachedPaths[dirredPath] &&
-            this.cachedPaths[dirredPath][path]
-          ) {
-            delete this.cachedPaths[dirredPath][path];
-          }
 
-          let connectedPath = shimmedPath;
-          if (connectedPath.indexOf('/node_modules') !== 0) {
-            connectedPath = /^(\w|@\w)/.test(shimmedPath)
-              ? pathUtils.join('/node_modules', shimmedPath)
-              : pathUtils.join(pathUtils.dirname(currentPath), shimmedPath);
-          }
+            const dependencyName = getDependencyName(connectedPath);
+
+            if (
+              this.manifest.dependencies.find(d => d.name === dependencyName) ||
+              this.manifest.dependencyDependencies[dependencyName]
+            ) {
+              promiseReject(
+                new ModuleNotFoundError(connectedPath, true, currentPath)
+              );
+            } else {
+              promiseReject(
+                new DependencyNotFoundError(connectedPath, currentPath)
+              );
+            }
 
-          const isDependency = connectedPath.includes('/node_modules/');
+            return;
+          }
 
-          connectedPath = connectedPath.replace('/node_modules/', '');
+          this.cachedPaths[dirredPath][path] = foundPath;
 
-          if (!isDependency) {
-            throw new ModuleNotFoundError(shimmedPath, false, currentPath);
+          if (foundPath === '//empty.js') {
+            promiseResolve(getShimmedModuleFromPath(currentPath, path));
+            return;
           }
 
-          const dependencyName = getDependencyName(connectedPath);
+          if (!this.transpiledModules[foundPath]) {
+            this.readFileSync(foundPath, (error, code) => {
+              if (error) {
+                promiseReject(error);
+                return;
+              }
 
-          if (
-            this.manifest.dependencies.find(d => d.name === dependencyName) ||
-            this.manifest.dependencyDependencies[dependencyName]
-          ) {
-            throw new ModuleNotFoundError(connectedPath, true, currentPath);
+              this.addModule({ path: foundPath, code });
+              promiseResolve(this.transpiledModules[foundPath].module);
+            });
           } else {
-            throw new DependencyNotFoundError(connectedPath, currentPath);
+            promiseResolve(this.transpiledModules[foundPath].module);
           }
         }
-      }
-      promiseResolve(this.transpiledModules[resolvedPath].module);
+      );
     });
   }
 
@@ -704,6 +725,8 @@ export default class Manager {
     if (cachedPath && this.transpiledModules[cachedPath]) {
       resolvedPath = cachedPath;
     } else {
+      const measureKey = `resolve-sync:${path}:${currentPath}`;
+      measure(measureKey);
       const presetAliasedPath = this.getPresetAliasedPath(path);
 
       const aliasedPath = this.getAliasedDependencyPath(
@@ -726,6 +749,7 @@ export default class Manager {
           packageFilter,
           moduleDirectory: this.getModuleDirectories(),
         });
+        endMeasure(measureKey, { silent: true });
 
         this.cachedPaths[dirredPath][path] = resolvedPath;
 
@@ -812,12 +836,10 @@ export default class Manager {
     const tModule =
       currentTModule || this.getTranspiledModule(this.modules['/package.json']); // Get arbitrary file from root
     try {
-      return Promise.resolve(
-        this.resolveTranspiledModule(
-          path,
-          tModule.module.path,
-          ignoredExtensions
-        )
+      return this.resolveTranspiledModule(
+        path,
+        tModule.module.path,
+        ignoredExtensions
       );
     } catch (e) {
       if (e.type === 'module-not-found' && e.isDependency) {

packages/app/src/sandbox/eval/presets/svelte/index.js

@@ -19,7 +19,7 @@ const babelOptions = {
 export default function initialize() {
   const sveltePreset = new Preset('svelte', ['js', 'jsx', 'svelte'], {});
 
-  sveltePreset.registerTranspiler(module => /\.jsx?$/.test(module.path), [
+  sveltePreset.registerTranspiler(module => /\.m?jsx?$/.test(module.path), [
     { transpiler: babelTranspiler, options: babelOptions },
   ]);