Save downloaded JSON files to cache (codesandbox#1069)

Right now we don't download dynamically downloaded `package.json` files to cache, which causes errors when a sandbox is loaded from cache and a dependency uses a dynamic `package.json` to resolve another dynamically downloaded `js` file.

This fixes that by explicitly marking all dynamic `package.json` files as a dependency when they are used for resolving.

Fixes codesandbox#1058

Author: CompuIves

packages/app/src/sandbox/eval/entities/module.js

@@ -4,4 +4,5 @@ export type Module = {
   path: string,
   code: string,
   requires?: Array<string>,
+  downloaded?: boolean,
 };

packages/app/src/sandbox/eval/manager.js

@@ -495,6 +495,14 @@ export default class Manager {
         return SHIMMED_MODULE;
       }
 
+      // Quick try
+      const directPath = pathUtils.join(currentPath, shimmedPath);
+      const directModulePath = this.transpiledModules[directPath];
+      if (directModulePath) {
+        this.cachedPaths[dirredPath][path] = directPath;
+        return directModulePath.module;
+      }
+
       try {
         resolvedPath = resolve.sync(shimmedPath, {
           filename: currentPath,
@@ -558,10 +566,10 @@ export default class Manager {
 
   downloadDependency(
     path: string,
-    currentPath: string,
+    currentTModule: TranspiledModule,
     ignoredExtensions: Array<string> = this.preset.ignoredExtensions
   ): Promise<TranspiledModule> {
-    return fetchModule(path, currentPath, this, ignoredExtensions).then(
+    return fetchModule(path, currentTModule, this, ignoredExtensions).then(
       module => this.getTranspiledModule(module)
     );
   }
@@ -788,7 +796,8 @@ export default class Manager {
         if (
           !this.manifest.contents[tModule.module.path] ||
           (tModule.module.path.endsWith('.js') &&
-            tModule.module.requires == null)
+            tModule.module.requires == null) ||
+          tModule.module.downloaded
         ) {
           // Only save modules that are not precomputed
           serializedTModules[tModule.getId()] = tModule.serialize();

packages/app/src/sandbox/eval/npm/fetch-npm-module.js

@@ -9,6 +9,7 @@ import type { Manifest } from '../manager';
 import DependencyNotFoundError from '../../errors/dependency-not-found-error';
 import getDependencyName from '../utils/get-dependency-name';
 import { packageFilter } from '../utils/resolve-utils';
+import type { default as TranspiledModule } from '../transpiled-module';
 
 type Meta = {
   [path: string]: any,
@@ -128,18 +129,21 @@ function downloadDependency(depName: string, depVersion: string, path: string) {
     .then(x => ({
       path,
       code: x,
+      downloaded: true,
     }));
 
   return packages[path];
 }
 
 function resolvePath(
   path: string,
-  currentPath: string,
+  currentTModule: TranspiledModule,
   manager: Manager,
   defaultExtensions: Array<string> = ['js', 'jsx', 'json'],
   meta = {}
 ): Promise<string> {
+  const currentPath = currentTModule.module.path;
+
   return new Promise((res, reject) => {
     resolve(
       path,
@@ -160,7 +164,10 @@ function resolvePath(
           const callback = cb || c;
 
           try {
-            return callback(null, manager.readFileSync(p));
+            const tModule = manager.resolveTranspiledModule(p, '/');
+            tModule.initiators.add(currentTModule);
+            currentTModule.dependencies.add(tModule);
+            return callback(null, tModule.module.code);
           } catch (e) {
             const depPath = p.replace('/node_modules/', '');
             const depName = getDependencyName(depPath);
@@ -192,6 +199,10 @@ function resolvePath(
 
                 if (module) {
                   manager.addModule(module);
+                  const tModule = manager.addTranspiledModule(module, '');
+
+                  tModule.initiators.add(currentTModule);
+                  currentTModule.dependencies.add(tModule);
 
                   callback(null, module.code);
                   return null;
@@ -217,7 +228,7 @@ function resolvePath(
 }
 
 async function findDependencyVersion(
-  currentPath: string,
+  currentTModule: TranspiledModule,
   manager: Manager,
   defaultExtensions: Array<string> = ['js', 'jsx', 'json'],
   dependencyName: string
@@ -227,7 +238,7 @@ async function findDependencyVersion(
   try {
     const foundPackageJSONPath = await resolvePath(
       pathUtils.join(dependencyName, 'package.json'),
-      currentPath,
+      currentTModule,
       manager,
       defaultExtensions
     );
@@ -265,10 +276,11 @@ async function findDependencyVersion(
 
 export default async function fetchModule(
   path: string,
-  currentPath: string,
+  currentTModule: TranspiledModule,
   manager: Manager,
   defaultExtensions: Array<string> = ['js', 'jsx', 'json']
 ): Promise<Module> {
+  const currentPath = currentTModule.module.path;
   // Get the last part of the path as dependency name for paths like
   // instantsearch.js/node_modules/lodash/sum.js
   // In this case we want to get the lodash dependency info
@@ -277,7 +289,7 @@ export default async function fetchModule(
   );
 
   const versionInfo = await findDependencyVersion(
-    currentPath,
+    currentTModule,
     manager,
     defaultExtensions,
     dependencyName
@@ -304,7 +316,7 @@ export default async function fetchModule(
 
   const foundPath = await resolvePath(
     path,
-    currentPath,
+    currentTModule,
     manager,
     defaultExtensions,
     normalizedMeta

packages/app/src/sandbox/eval/presets/angular-cli/index.js

@@ -4,7 +4,6 @@ import Preset from '../';
 
 import angular2Transpiler from '../../transpilers/angular2-template';
 import typescriptTranspiler from '../../transpilers/typescript';
-import babelTranspiler from '../../transpilers/babel';
 import jsonTranspiler from '../../transpilers/json';
 import stylesTranspiler from '../../transpilers/style';
 import sassTranspiler from '../../transpilers/sass';
@@ -214,7 +213,7 @@ export default function initialize() {
   ]);
 
   preset.registerTranspiler(module => /\.js$/.test(module.path), [
-    { transpiler: babelTranspiler },
+    { transpiler: typescriptTranspiler },
   ]);
 
   preset.registerTranspiler(module => /\.json$/.test(module.path), [

packages/app/src/sandbox/eval/transpiled-module.js

@@ -422,7 +422,7 @@ export default class TranspiledModule {
         } catch (e) {
           if (e.type === 'module-not-found' && e.isDependency) {
             this.asyncDependencies.push(
-              manager.downloadDependency(e.path, this.module.path)
+              manager.downloadDependency(e.path, this)
             );
           } else {
             // Don't throw the error, we want to throw this error during evaluation
@@ -675,7 +675,11 @@ export default class TranspiledModule {
         !isESModule(this.module.code)
       ) {
         if (process.env.NODE_ENV === 'development') {
-          console.warn('[WARN] Sandpack: loading an untranspiled module');
+          console.warn(
+            `[WARN] Sandpack: loading an untranspiled module: ${
+              this.module.path
+            }`
+          );
         }
         // This code is probably required as a dynamic require. Since we can
         // assume that node_modules dynamic requires are only done for node
@@ -891,7 +895,8 @@ export default class TranspiledModule {
     if (
       this.initiators.size === 0 &&
       this.transpilationInitiators.size === 0 &&
-      !this.isEntry
+      !this.isEntry &&
+      !manager.isFirstLoad
     ) {
       // Remove the module from the transpiler if it's not used anymore
       debug(`Removing '${this.getId()}' from manager.`);
@@ -977,11 +982,7 @@ export default class TranspiledModule {
       this.source = data.source;
     }
 
-    const loadModule = (
-      depId: string,
-      initiator = false,
-      transpilation = false
-    ) => {
+    const getModule = (depId: string) => {
       if (state[depId]) {
         return state[depId];
       }
@@ -990,7 +991,15 @@ export default class TranspiledModule {
       const query = queryParts.join(':');
 
       const module = manager.transpiledModules[path].module;
-      const tModule = manager.getTranspiledModule(module, query);
+      return manager.getTranspiledModule(module, query);
+    };
+
+    const loadModule = (
+      depId: string,
+      initiator = false,
+      transpilation = false
+    ) => {
+      const tModule = getModule(depId);
 
       if (initiator) {
         if (transpilation) {

packages/app/src/sandbox/eval/transpilers/babel/index.js

@@ -17,7 +17,7 @@ class BabelTranspiler extends WorkerTranspiler {
   worker: Worker;
 
   constructor() {
-    super('babel-loader', BabelWorker, 2, { hasFS: true });
+    super('babel-loader', BabelWorker, 3, { hasFS: true });
   }
 
   startupWorkersInitialized = false;

packages/app/src/sandbox/eval/transpilers/typescript/index.js

@@ -8,7 +8,7 @@ class TypeScriptTranspiler extends WorkerTranspiler {
   worker: Worker;
 
   constructor() {
-    super('ts-loader', TypeScriptWorker, 2);
+    super('ts-loader', TypeScriptWorker, 3);
   }
 
   doTranspilation(code: string, loaderContext: LoaderContext) {

packages/app/src/sandbox/startup.js

@@ -1,6 +1,6 @@
 import BabelWorker from 'worker-loader?publicPath=/&name=babel-transpiler.[hash:8].worker.js!./eval/transpilers/babel/worker/index.js';
 
 window.babelworkers = [];
-for (let i = 0; i < 2; i++) {
+for (let i = 0; i < 3; i++) {
   window.babelworkers.push(new BabelWorker());
 }