Packager fixes (codesandbox#253)

* Packager fixes

* Enable packager by default

* Flip more switches

* Fix svelte

* Fix react-ts

* Use custom threshold for apollo sandbox

* Remove flow typed changes

* Allow resolving of versioned dependencies

* Set experimentalPackager defaults to false

* Make dynamic fetching robust & provide dynamic polyfills for node libraries

* Revert "Set experimentalPackager defaults to false"

This reverts commit 1d078ea.

* Fix file resolving for main dependencies

* Revert "Revert "Set experimentalPackager defaults to false""

This reverts commit dcbb1a4.

* Use production search on local server

Author: CompuIves

integration-tests/tests/sandboxes.test.js

@@ -7,7 +7,7 @@ const SANDBOXES = [
   'svelte',
   'react-ts',
   'github/reactjs/redux/tree/master/examples/todomvc',
-  'jvlrl98xw3',
+  { id: 'jvlrl98xw3', threshold: 0.05 },
   'vVoQVk78',
   'github/faceyspacey/redux-first-router-codesandbox/tree/master',
   'mZRjw05yp',
@@ -20,6 +20,7 @@ const SANDBOXES = [
 SANDBOXES.forEach(sandbox => {
   const id = sandbox.id || sandbox;
   const root = sandbox.root || '#root';
+  const threshold = sandbox.threshold || 0.01;
   let browser = puppeteer.launch({
     args: ['--no-sandbox', '--disable-setuid-sandbox'],
   });
@@ -45,7 +46,7 @@ SANDBOXES.forEach(sandbox => {
 
         expect(screenshot).toMatchImageSnapshot({
           customDiffConfig: {
-            threshold: 0.01, // 1% threshold
+            threshold,
           },
           customSnapshotIdentifier: id.split('/').join('-'),
         });

package.json

@@ -46,7 +46,7 @@
     "extract-text-webpack-plugin": "^2.1.0",
     "file-loader": "^0.11.1",
     "filesize": "^3.5.6",
-    "flow-bin": "^0.48.0",
+    "flow-bin": "^0.57.3",
     "fs-extra": "^2.1.2",
     "gulp-replace": "^0.5.4",
     "gzip-size": "3.0.0",
@@ -101,10 +101,10 @@
     "babel-plugin-transform-remove-strict-mode": "^0.0.2",
     "babel-plugin-transform-vue-jsx": "^3.5.0",
     "babel-polyfill": "^6.23.0",
-    "babel-preset-env": "^1.6.0",
     "babel-preset-stage-0": "^6.24.1",
     "babel-standalone": "^6.25.0",
     "base64-loader": "^1.0.0",
+    "browser-resolve": "CompuIves/node-browser-resolve",
     "codemirror": "^5.27.4",
     "codesandbox-api": "^0.0.12",
     "color": "^0.11.4",

src/app/utils/config.js

@@ -1,7 +1,9 @@
 export const ALGOLIA_API_KEY = '7675b9e87ed7dd5bbf18f3e5310f92d6';
 export const ALGOLIA_APPLICATION_ID = 'ZACZHDBO7S';
 export const ALGOLIA_DEFAULT_INDEX =
-  process.env.NODE_ENV === 'production' ? 'prod_sandboxes' : 'dev_sandboxes';
+  process.env.NODE_ENV === 'production' || process.env.LOCAL_SERVER
+    ? 'prod_sandboxes'
+    : 'dev_sandboxes';
 
 export const STRIPE_API_KEY =
   process.env.NODE_ENV === 'production'

src/sandbox/compile.js

@@ -120,8 +120,7 @@ async function compile({
       updateManager(sandboxId, template, managerModules, experimentalPackager),
     ]);
 
-    const { externals = {} } = manifest;
-
+    const { externals = {} } = manifest || {};
     if (experimentalPackager) {
       manager.setManifest(manifest);
     } else {

src/sandbox/eval/loaders/eval.js

@@ -1,10 +1,11 @@
 // @flow
+import buildProcess from './utils/process';
 
 /* eslint-disable no-unused-vars */
 export default function(code: string, require: Function, exports) {
   const module = { exports: {} };
   const global = window;
-  const process = { env: { NODE_ENV: 'development' } };
+  const process = buildProcess();
 
   try {
     eval(code); // eslint-disable-line no-eval

src/sandbox/eval/loaders/utils/process.js

@@ -0,0 +1,41 @@
+const process = {};
+process.title = 'browser';
+process.browser = true;
+process.env = { NODE_ENV: 'development' };
+process.argv = [];
+process.version = ''; // empty string to avoid regexp issues
+process.versions = {};
+
+function noop() {}
+
+process.on = noop;
+process.addListener = noop;
+process.once = noop;
+process.off = noop;
+process.removeListener = noop;
+process.removeAllListeners = noop;
+process.emit = noop;
+process.prependListener = noop;
+process.prependOnceListener = noop;
+
+process.listeners = function(name) {
+  return [];
+};
+
+process.binding = function(name) {
+  throw new Error('process.binding is not supported');
+};
+
+process.cwd = function() {
+  return '/';
+};
+process.chdir = function(dir) {
+  throw new Error('process.chdir is not supported');
+};
+process.umask = function() {
+  return 0;
+};
+
+export default function build() {
+  return process;
+}

src/sandbox/eval/manager.js

@@ -1,13 +1,15 @@
 // @flow
 import { flattenDeep, uniq, values } from 'lodash';
+import resolve from 'browser-resolve';
 
 import * as pathUtils from 'common/utils/path';
 
 import type { Module } from './entities/module';
 import TranspiledModule from './transpiled-module';
 import Preset from './presets';
-import nodeResolvePath from './utils/node-resolve-path';
-import fetchModule from './npm/fetch-npm-module';
+import fetchModule, { getCombinedMetas } from './npm/fetch-npm-module';
+import coreLibraries from './npm/get-core-libraries';
+import getDependencyName from './utils/get-dependency-name';
 import DependencyNotFoundError from '../errors/dependency-not-found-error';
 import ModuleNotFoundError from '../errors/module-not-found-error';
 
@@ -20,15 +22,21 @@ type ModuleObject = {
 };
 
 export type Manifest = {
-  aliases: {
-    [path: string]: string | false,
-  },
   contents: {
     [path: string]: { content: string, requires: Array<string> },
   },
   dependencies: Array<{ name: string, version: string }>,
   dependencyDependencies: {
-    [name: string]: string,
+    [name: string]: {
+      semver: string,
+      resolved: string,
+      parents: string[],
+    },
+  },
+  dependencyAliases: {
+    [name: string]: {
+      [depName: string]: string,
+    },
   },
 };
 
@@ -73,8 +81,21 @@ export default class Manager {
     this.externals = externals;
   }
 
-  setManifest(manifest: Manifest) {
-    this.manifest = manifest;
+  setManifest(manifest: ?Manifest) {
+    this.manifest = manifest || {
+      contents: {},
+      dependencies: [],
+      dependencyDependencies: {},
+      dependencyAliases: {},
+    };
+
+    Object.keys(this.manifest.contents).forEach(path => {
+      this.addModule({
+        path,
+        code: this.manifest.contents[path].content,
+        requires: this.manifest.contents[path].requires,
+      });
+    });
   }
 
   evaluateModule(module: Module) {
@@ -187,94 +208,102 @@ export default class Manager {
     return values(this.transpiledModules).map(t => t.module);
   }
 
-  resolveModule(
-    path: string,
-    defaultExtensions: Array<string> = ['js', 'jsx', 'json']
-  ): Module {
-    const moduleObject = this.transpiledModules;
-    const foundPath = nodeResolvePath(path, moduleObject, defaultExtensions);
+  getAliasedDependencyPath(path: string, currentPath: string) {
+    const isDependency = /^(\w|@\w)/.test(path);
+
+    if (!isDependency) {
+      return path;
+    }
 
-    if (foundPath && moduleObject[foundPath]) {
-      return moduleObject[foundPath].module;
+    const isCurrentPathDependency = currentPath.startsWith('/node_modules');
+    if (!isCurrentPathDependency) {
+      return path;
     }
 
-    throw new ModuleNotFoundError(path, false);
+    const dependencyName = getDependencyName(path);
+    const previousDependencyName = getDependencyName(currentPath);
+
+    if (
+      this.manifest.dependencyAliases[previousDependencyName] &&
+      this.manifest.dependencyAliases[previousDependencyName][dependencyName]
+    ) {
+      const aliasedDependencyName = this.manifest.dependencyAliases[
+        previousDependencyName
+      ][dependencyName];
+
+      return path.replace(dependencyName, aliasedDependencyName);
+    }
+
+    return path;
   }
 
-  resolveDependency(
+  resolveModule(
     path: string,
+    currentPath: string,
     defaultExtensions: Array<string> = ['js', 'jsx', 'json']
   ): Module {
-    let depPath = path.replace('/node_modules/', '');
+    const aliasedPath = this.getAliasedDependencyPath(path, currentPath);
+    const shimmedPath = coreLibraries[aliasedPath] || aliasedPath;
+    try {
+      const resolvedPath = resolve.sync(shimmedPath, {
+        filename: currentPath,
+        extensions: defaultExtensions.map(ext => '.' + ext),
+        isFile: p => !!this.transpiledModules[p] || !!getCombinedMetas()[p],
+        readFileSync: p => {
+          if (this.transpiledModules[p]) {
+            return this.transpiledModules[p].module.code;
+          }
+
+          const err = new Error('Could not find ' + p);
+          err.code = 'ENOENT';
+
+          throw err;
+        },
+      });
+
+      if (NODE_LIBS.includes(resolvedPath)) {
+        return {
+          path: pathUtils.join('/node_modules', resolvedPath),
+          code: `// empty`,
+          requires: [],
+        };
+      }
 
-    const aliasedPath = nodeResolvePath(
-      depPath,
-      this.manifest.aliases,
-      defaultExtensions
-    );
+      return this.transpiledModules[resolvedPath].module;
+    } catch (e) {
+      let connectedPath = /^(\w|@\w)/.test(shimmedPath)
+        ? pathUtils.join('/node_modules', shimmedPath)
+        : pathUtils.join(pathUtils.dirname(currentPath), shimmedPath);
 
-    depPath = aliasedPath == null ? depPath : aliasedPath;
+      const isDependency = connectedPath.includes('/node_modules/');
 
-    const alias = this.manifest.aliases[depPath];
-    // So aliased to not be included in the bundle, decided by browser object on pkg
-    if (alias === false) {
-      return {
-        path: pathUtils.join('/node_modules', depPath),
-        code: 'module.exports = null;',
-        requires: [],
-      };
-    }
+      connectedPath = connectedPath.replace('/node_modules/', '');
 
-    // Check if the module is already there
-    const foundPath = nodeResolvePath(
-      path,
-      this.transpiledModules,
-      defaultExtensions
-    );
-    if (foundPath && this.transpiledModules[foundPath]) {
-      return this.transpiledModules[foundPath].module;
-    }
+      if (!isDependency) {
+        throw new ModuleNotFoundError(shimmedPath, false);
+      }
 
-    const resolvedPath = nodeResolvePath(
-      alias || depPath,
-      this.manifest.contents,
-      defaultExtensions
-    );
-    if (resolvedPath && this.manifest.contents[resolvedPath]) {
-      return {
-        path: pathUtils.join('/node_modules', resolvedPath),
-        code: this.manifest.contents[resolvedPath].content,
-        requires: this.manifest.contents[resolvedPath].requires,
-      };
-    }
+      const dependencyName = getDependencyName(connectedPath);
 
-    const dependencyParts = depPath.split('/');
-    const dependencyName = depPath.startsWith('@')
-      ? `${dependencyParts[0]}/${dependencyParts[1]}`
-      : dependencyParts[0];
-
-    if (NODE_LIBS.includes(dependencyName)) {
-      return {
-        path: pathUtils.join('/node_modules', depPath),
-        code: `// empty`,
-        requires: [],
-      };
-    }
-
-    if (
-      this.manifest.dependencies.find(d => d.name === dependencyName) ||
-      this.manifest.dependencyDependencies[dependencyName]
-    ) {
-      throw new ModuleNotFoundError(alias || depPath, true);
-    } else {
-      throw new DependencyNotFoundError(alias || depPath);
+      if (
+        this.manifest.dependencies.find(d => d.name === dependencyName) ||
+        this.manifest.dependencyDependencies[dependencyName]
+      ) {
+        throw new ModuleNotFoundError(connectedPath, true);
+      } else {
+        throw new DependencyNotFoundError(connectedPath);
+      }
     }
   }
 
-  async downloadDependency(path: string): Promise<TranspiledModule> {
+  async downloadDependency(
+    path: string,
+    currentPath: string
+  ): Promise<TranspiledModule> {
     return fetchModule(
       path,
-      this.manifest,
+      currentPath,
+      this,
       this.preset.ignoredExtensions
     ).then(module => this.getTranspiledModule(module));
   }
@@ -294,28 +323,15 @@ export default class Manager {
     // pop() mutates queryPath, queryPath is now just the loaders
     const modulePath = queryPath.pop();
 
-    let module;
-
-    let newPath = pathUtils
-      .join(
-        pathUtils.dirname(currentPath),
-        this.preset.getAliasedPath(modulePath)
-      )
+    const newPath = this.preset
+      .getAliasedPath(modulePath)
       .replace(/.*\{\{sandboxRoot\}\}/, '');
 
-    if (/^(\w|@\w)/.test(modulePath) && !modulePath.includes('!')) {
-      // Prepend node_modules and go to root if it is a dependency
-      newPath = pathUtils.join(
-        '/node_modules',
-        this.preset.getAliasedPath(modulePath)
-      );
-    }
-
-    if (newPath.startsWith('/node_modules')) {
-      module = this.resolveDependency(newPath, this.preset.ignoredExtensions);
-    } else {
-      module = this.resolveModule(newPath, this.preset.ignoredExtensions);
-    }
+    const module = this.resolveModule(
+      newPath,
+      currentPath,
+      this.preset.ignoredExtensions
+    );
 
     return this.getTranspiledModule(module, queryPath.join('!'));
   }