Manager caching (codesandbox#420)

* Manager caching

* Fix error caching and cache resolve paths

* Start of HMR and bugfixes

* HMR

* Fix HMR

* Small cleanup

* Cleanup

* Cleanup HMR

* Experimental vue hot reloading

* Got it working!!!!

* Fix css modules

* Cleaning

* Fix loaders

* Changes

* Progress

* Fix

* Fix Vue recompiling

* Changes

* Fix hard reload of vue file changes

* Add ts-loader

* Fix tests

* Update dependency handling

* Trigger CI

* Update cache to v3

* CI test commit

* Fix scss dependencies

* Fix loading screen being in front of error

* Fix inconsistent states

* Use CircularJSON

Author: CompuIves

.babelrc

@@ -1,10 +1,7 @@
 {
   "env": {
     "test": {
-      "presets": [
-        "env",
-        "react"
-      ],
+      "presets": ["env", "react"],
       "plugins": [
         "babel-plugin-transform-async-to-generator",
         "babel-plugin-transform-object-rest-spread",
@@ -19,7 +16,8 @@
             "webpack": true
           }
         ],
-        "babel-plugin-system-import-transformer"
+        "babel-plugin-system-import-transformer",
+        "babel-macros"
       ]
     }
   }

.circleci/config.yml

@@ -8,24 +8,22 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: v2-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
+          keys:
+            - v3-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
+            - v3-dependency-cache-{{ .Branch }}
+            - v3-dependency-cache
       - run:
           name: Install Dependencies
           command: yarn install
       - save_cache:
-          key: v2-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
+          key: v3-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
           paths:
-            - ./node_modules
-            - ./packages/app/node_modules
-            - ./packages/codesandbox-api/node_modules
-            - ./packages/common/node_modules
-            - ./packages/homepage/node_modules
-            - ~/.yarn-cache
+            - node_modules
       - run:
           name: Build
           command: yarn build:dependents
       - save_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - ~/codesandbox-client
 
@@ -35,19 +33,19 @@ jobs:
     working_directory: ~/codesandbox-client
     steps:
       - restore_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
       # We do this to compare sizes from these builds with master
       - restore_cache:
-          key: v2-prod-app-build-cache-master
+          key: v3-prod-app-build-cache-master
       - run:
           name: Build Application
           command: yarn build:prod
       - save_cache:
-          key: v2-prod-app-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-prod-app-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - ./packages/app/www
       - save_cache:
-          key: v2-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
           paths:
             - ./www
       - store_artifacts:
@@ -60,7 +58,7 @@ jobs:
     working_directory: ~/codesandbox-client
     steps:
       - restore_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Start Test Server
           command: yarn start:test
@@ -84,7 +82,7 @@ jobs:
     working_directory: ~/codesandbox-client
     steps:
       - restore_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Test
           command: yarn test --ci --testResultsProcessor="jest-junit"
@@ -97,7 +95,7 @@ jobs:
     working_directory: ~/codesandbox-client
     steps:
       - restore_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
       - run:
           name: Lint
           command: yarn lint
@@ -108,9 +106,9 @@ jobs:
     working_directory: ~/codesandbox-client
     steps:
       - restore_cache:
-          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-repo-{{ .Environment.CIRCLE_SHA1 }}
       - restore_cache:
-          key: v2-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
+          key: v3-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{ .Environment.CIRCLE_SHA1 }}
       - add_ssh_keys:
           fingerprints:
             - "f7:f1:e6:60:96:24:d9:cd:1b:8b:c0:34:e7:ee:fa:82"
@@ -153,6 +151,3 @@ workflows:
               only: master
           requires:
             - hold
-
-
-

lerna.json

@@ -1,5 +1,6 @@
 {
   "lerna": "2.4.0",
+  "packages": ["packages/*"],
   "version": "independent",
   "npmClient": "yarn",
   "useWorkspaces": true

packages/app/package.json

@@ -131,6 +131,7 @@
     "humps": "CompuIves/humps",
     "image-extensions": "^1.1.0",
     "jszip": "^3.1.3",
+    "localforage": "^1.5.5",
     "lodash": "^4.17.2",
     "match-sorter": "^1.8.1",
     "moment": "^2.18.1",
@@ -141,6 +142,7 @@
     "postcss": "^6.0.9",
     "postcss-selector-parser": "^2.2.3",
     "qs": "^6.5.0",
+    "querystring": "^0.2.0",
     "rc-slider": "^8.2.0",
     "react": "16.2.0",
     "react-addons-css-transition-group": "^15.6.0",
@@ -174,6 +176,7 @@
     "svg-react-loader": "^0.4.4",
     "tern": "^0.21.0",
     "vue": "^2.5.2",
+    "vue-hot-reload-api": "^2.2.4",
     "vue-template-compiler": "^2.5.2",
     "vue-template-es2015-compiler": "^1.6.0"
   },

packages/app/src/app/components/sandbox/Preview/utils/should-update.js

@@ -2,13 +2,7 @@
 import type { Module, Directory } from 'common/types';
 
 // Fields to compare
-const MODULE_FIELDS = [
-  'directoryShortid',
-  'code',
-  'title',
-  'isNotSynced',
-  'isBinary',
-];
+const MODULE_FIELDS = ['directoryShortid', 'code', 'title', 'isBinary'];
 
 const DIRECTORY_FIELDS = ['title', 'directoryShortid', 'shortid'];
 

packages/app/src/app/store/entities/reducer.js

@@ -10,7 +10,7 @@ import moduleReducer from './sandboxes/modules/reducer';
 import directoryReducer from './sandboxes/directories/reducer';
 import userReducer from './users/reducer';
 
-const d = _debug('cw:app:store:reducers:entities');
+const d = _debug('cs:app:store:reducers:entities');
 
 const entityReducers = {
   sandboxes: sandboxReducer,

packages/app/src/sandbox/compile.js

@@ -35,13 +35,31 @@ export function getCurrentManager(): ?Manager {
   return manager;
 }
 
-async function updateManager(sandboxId, template, managerModules) {
+let firstLoad = true;
+
+async function updateManager(
+  sandboxId,
+  template,
+  managerModules,
+  manifest,
+  isNewCombination
+) {
+  let newManager = false;
   if (!manager || manager.id !== sandboxId) {
-    manager = new Manager(sandboxId, managerModules, getPreset(template));
-  } else {
-    await manager.updateData(managerModules);
+    newManager = true;
+    manager = new Manager(sandboxId, getPreset(template), managerModules);
+    if (firstLoad) {
+      // We save the state of transpiled modules, and load it here again. Gives
+      // faster initial loads.
+
+      await manager.load();
+    }
   }
 
+  if (isNewCombination || newManager) {
+    manager.setManifest(manifest);
+  }
+  await manager.updateData(managerModules);
   return manager;
 }
 
@@ -69,10 +87,11 @@ async function compile({
   isModuleView = false,
   template,
 }) {
+  const startTime = Date.now();
   try {
     clearErrorTransformers();
     initializeErrorTransformers();
-    unmount();
+    unmount(manager && manager.webpackHMR);
   } catch (e) {
     console.error(e);
   }
@@ -81,52 +100,57 @@ async function compile({
   handleExternalResources(externalResources);
 
   try {
-    const [{ manifest, isNewCombination }] = await Promise.all([
-      loadDependencies(dependencies),
-      updateManager(sandboxId, template, modules),
-    ]);
+    const { manifest, isNewCombination } = await loadDependencies(dependencies);
 
-    // Just reset the whole packager if it's a new combination
-    if (isNewCombination) {
+    if (isNewCombination && !firstLoad) {
+      // Just reset the whole manager if it's a new combination
       manager = null;
-      await updateManager(sandboxId, template, modules);
     }
+    const t = Date.now();
 
-    manager.setManifest(manifest);
+    await updateManager(
+      sandboxId,
+      template,
+      modules,
+      manifest,
+      isNewCombination
+    );
 
     const managerModuleToTranspile = modules.find(m => m.path === entry);
 
-    const t = Date.now();
     await manager.transpileModules(managerModuleToTranspile);
+
     debug(`Transpilation time ${Date.now() - t}ms`);
 
     resetScreen();
 
-    try {
-      const children = document.body.children;
-      // Do unmounting for react
-      if (manifest.dependencies.find(n => n.name === 'react-dom')) {
-        const reactDOMModule = manager.resolveModule('react-dom', '');
-        const reactDOM = manager.evaluateModule(reactDOMModule);
+    if (!manager.webpackHMR) {
+      try {
+        const children = document.body.children;
+        // Do unmounting for react
+        if (manifest.dependencies.find(n => n.name === 'react-dom')) {
+          const reactDOMModule = manager.resolveModule('react-dom', '');
+          const reactDOM = manager.evaluateModule(reactDOMModule);
 
-        reactDOM.unmountComponentAtNode(document.body);
+          reactDOM.unmountComponentAtNode(document.body);
 
-        for (let i = 0; i < children.length; i += 1) {
-          if (children[i].tagName === 'DIV') {
-            reactDOM.unmountComponentAtNode(children[i]);
+          for (let i = 0; i < children.length; i += 1) {
+            if (children[i].tagName === 'DIV') {
+              reactDOM.unmountComponentAtNode(children[i]);
+            }
           }
         }
+      } catch (e) {
+        /* don't do anything with this error */
       }
-    } catch (e) {
-      /* don't do anything with this error */
     }
-
-    const htmlModule = modules.find(
-      m => m.path === '/public/index.html' || m.path === '/index.html'
-    );
-    const html = htmlModule ? htmlModule.code : '<div id="root"></div>';
-
-    document.body.innerHTML = html;
+    if (!manager.webpackHMR || firstLoad) {
+      const htmlModule = modules.find(
+        m => m.path === '/public/index.html' || m.path === '/index.html'
+      );
+      const html = htmlModule ? htmlModule.code : '<div id="root"></div>';
+      document.body.innerHTML = html;
+    }
 
     const tt = Date.now();
     const oldHTML = document.body.innerHTML;
@@ -162,13 +186,22 @@ async function compile({
       initializeResizeListener();
     }
 
+    debug(`Total time: ${Date.now() - startTime}ms`);
+
     dispatch({
       type: 'success',
     });
+
+    firstLoad = false;
+    manager.save();
   } catch (e) {
     console.log('Error in sandbox:');
     console.error(e);
 
+    if (manager) {
+      manager.clearCache();
+    }
+
     e.module = e.module;
     e.fileName = e.fileName || entry;