Add support for version resolution (codesandbox#2924)

Author: CompuIves

packages/app/package.json

@@ -143,6 +143,7 @@
     "lru-cache": "^4.1.3",
     "match-sorter": "^1.8.1",
     "memoize-one": "^4.0.0",
+    "minimatch": "^3.0.4",
     "mobx": "^5.11.0",
     "mobx-react": "^6.1.1",
     "mobx-react-lite": "^1.4.1",

packages/app/src/sandbox/compile.ts

@@ -485,7 +485,10 @@ async function compile({
 
     const { manifest, isNewCombination } = await loadDependencies(
       dependencies,
-      disableDependencyPreprocessing
+      {
+        disableExternalConnection: disableDependencyPreprocessing,
+        resolutions: parsedPackageJSON.resolutions,
+      }
     );
 
     if (isNewCombination && !firstLoad) {

packages/app/src/sandbox/eval/npm/fetch-npm-module.ts

@@ -3,6 +3,7 @@ import { CSB_PKG_PROTOCOL } from '@codesandbox/common/lib/utils/ci';
 import resolve from 'browser-resolve';
 import DependencyNotFoundError from 'sandbox-hooks/errors/dependency-not-found-error';
 
+import delay from 'sandbox/utils/delay';
 import { Module } from '../entities/module';
 import Manager from '../manager';
 
@@ -114,7 +115,7 @@ const urlProtocols = {
   },
 };
 
-async function fetchWithRetries(url: string, retries = 3): Promise<Response> {
+async function fetchWithRetries(url: string, retries = 6): Promise<Response> {
   const doFetch = () =>
     window.fetch(url).then(x => {
       if (x.ok) {
@@ -124,8 +125,15 @@ async function fetchWithRetries(url: string, retries = 3): Promise<Response> {
       throw new Error(`Could not fetch ${url}`);
     });
 
+  let lastTryTime = 0;
   for (let i = 0; i < retries; i++) {
+    if (Date.now() - lastTryTime < 3000) {
+      // Ensure that we at least wait 3s before retrying a request to prevent rate limits
+      // eslint-disable-next-line
+      await delay(3000 - (Date.now() - lastTryTime));
+    }
     try {
+      lastTryTime = Date.now();
       // eslint-disable-next-line
       return await doFetch();
     } catch (e) {
@@ -174,9 +182,11 @@ async function getMeta(
   }
 
   const protocol = getFetchProtocol(version);
-  const metaUrl = await protocol.meta(nameWithoutAlias, version);
 
-  metas[id] = fetchWithRetries(metaUrl).then(x => x.json());
+  metas[id] = protocol
+    .meta(nameWithoutAlias, version)
+    .then(fetchWithRetries)
+    .then(x => x.json());
 
   return metas[id];
 }
@@ -202,9 +212,10 @@ export async function downloadDependency(
 
   const nameWithoutAlias = depName.replace(ALIAS_REGEX, '');
   const protocol = getFetchProtocol(depVersion);
-  const url = await protocol.file(nameWithoutAlias, depVersion, relativePath);
 
-  packages[id] = fetchWithRetries(url)
+  packages[id] = protocol
+    .file(nameWithoutAlias, depVersion, relativePath)
+    .then(fetchWithRetries)
     .then(x => x.text())
     .catch(async () => {
       const fallbackProtocol = getFetchProtocol(depVersion, true);

packages/app/src/sandbox/npm/fetch-dependencies.ts

@@ -145,7 +145,10 @@ async function getDependencies(dependencies: Object) {
   }
 }
 
-export async function fetchDependencies(npmDependencies: Dependencies) {
+export async function fetchDependencies(
+  npmDependencies: Dependencies,
+  resolutions?: { [key: string]: string }
+) {
   if (Object.keys(npmDependencies).length !== 0) {
     // New Packager flow
 

packages/app/src/sandbox/npm/index.ts

@@ -14,7 +14,7 @@ type NPMDependencies = {
 };
 
 /**
- * If there is a URL to a file we need to fetch the dependenices dynamically, at least
+ * If there is a URL to a file we need to fetch the dependencies dynamically, at least
  * for the first version. In the future we might want to consider a hybrid version where
  * we only fetch the dynamic files for dependencies with a url as version. But this is a good
  * start.
@@ -31,7 +31,7 @@ function shouldFetchDynamically(dependencies: NPMDependencies) {
  */
 export async function loadDependencies(
   dependencies: NPMDependencies,
-  disableExternalConnection = false
+  { disableExternalConnection = false, resolutions = undefined } = {}
 ) {
   let isNewCombination = false;
   if (Object.keys(dependencies).length !== 0) {
@@ -54,7 +54,7 @@ export async function loadDependencies(
         ? getDependencyVersions
         : fetchDependencies;
 
-      const data = await fetchFunction(dependenciesWithoutTypings);
+      const data = await fetchFunction(dependenciesWithoutTypings, resolutions);
 
       // Mark that the last requested url is this
       loadedDependencyCombination = depQuery;

packages/app/src/sandbox/version-resolving/index.ts

@@ -1,12 +1,19 @@
 import { resolveDependencyInfo } from './resolve-dependency';
 import { mergeDependencies } from './merge-dependency';
 
-export async function getDependencyVersions(dependencies: {
-  [depName: string]: string;
-}) {
+import { parseResolutions } from './resolutions';
+
+export async function getDependencyVersions(
+  dependencies: {
+    [depName: string]: string;
+  },
+  resolutions?: { [startGlob: string]: string }
+) {
+  const parsedResolutions = parseResolutions(resolutions);
+
   const depInfos = await Promise.all(
     Object.keys(dependencies).map(depName =>
-      resolveDependencyInfo(depName, dependencies[depName])
+      resolveDependencyInfo(depName, dependencies[depName], parsedResolutions)
     )
   );
 

packages/app/src/sandbox/version-resolving/merge-dependency.ts

@@ -163,6 +163,7 @@ export function mergeDependencies(responses: ILambdaResponse[]) {
       if (
         rootDependency &&
         !intersects(rootDependency.version, newDepDep.semver) &&
+        rootDependency.version !== newDepDep.resolved &&
         rootDependency.name !== r.dependency.name // and this dependency doesn't require an older version of itself
       ) {
         // If a root dependency is in conflict with a child dependency, we always

packages/app/src/sandbox/version-resolving/resolutions.ts

@@ -0,0 +1,57 @@
+/**
+ * Parse input strings like `package-1/package-2` to an array of packages
+ */
+function parsePackagePath(input: string) {
+  return input.match(/(@[^/]+\/)?([^/]+)/g) || [];
+}
+
+const WRONG_PATTERNS = /\/$|\/{2,}|\*+$/;
+const GLOBAL_NESTED_DEP_PATTERN = '**/';
+
+function isValidPackagePath(input: string) {
+  return !WRONG_PATTERNS.test(input);
+}
+
+export interface IParsedResolution {
+  name: string;
+  range: string;
+  globPattern: string;
+  pattern: string;
+}
+
+export function parsePatternInfo(
+  globPattern: string,
+  range: string
+): IParsedResolution {
+  if (!isValidPackagePath(globPattern)) {
+    console.warn('invalidResolutionName');
+    return null;
+  }
+
+  const directories = parsePackagePath(globPattern);
+  const name = directories.pop();
+
+  // For legacy support of resolutions, replace `name` with `**/name`
+  if (name === globPattern) {
+    // eslint-disable-next-line
+    globPattern = `${GLOBAL_NESTED_DEP_PATTERN}${name}`;
+  }
+
+  return {
+    name,
+    range,
+    globPattern,
+    pattern: `${name}@${range}`,
+  };
+}
+
+export function parseResolutions(resolutions?: {
+  [name: string]: string;
+}): IParsedResolution[] {
+  if (!resolutions) {
+    return [];
+  }
+
+  const keys = Object.keys(resolutions);
+  return keys.map(key => parsePatternInfo(key, resolutions[key]));
+}

packages/app/src/sandbox/version-resolving/resolve-dependency.ts

@@ -1,5 +1,9 @@
+import minimatch from 'minimatch';
+import * as semver from 'semver';
+
 import { ILambdaResponse } from './merge-dependency';
 import { downloadDependency } from '../eval/npm/fetch-npm-module';
+import { IParsedResolution } from './resolutions';
 
 function getPackageJSON(dep: string, version: string) {
   return downloadDependency(dep, version, '/package.json').then(m => m.code);
@@ -18,9 +22,39 @@ interface IPeerDependencyResult {
   };
 }
 
+function getAbsoluteVersion(
+  originalDep: string,
+  depName: string,
+  depVersion: string,
+  parsedResolutions: { [name: string]: IParsedResolution[] }
+) {
+  // Try getting it from the resolutions field first, if that doesn't work
+  // we try to get the latest version from the semver.
+  const applicableResolutions = parsedResolutions[depName];
+  if (applicableResolutions) {
+    const modulePath = [originalDep, depName].join('/');
+
+    const { range } =
+      applicableResolutions.find(({ globPattern }) =>
+        minimatch(modulePath, globPattern)
+      ) || {};
+
+    if (range) {
+      if (semver.valid(range)) {
+        return getLatestVersionForSemver(depName, range);
+      }
+
+      return range;
+    }
+  }
+
+  return getLatestVersionForSemver(depName, depVersion);
+}
+
 async function getDependencyDependencies(
   dep: string,
   version: string,
+  parsedResolutions: { [name: string]: IParsedResolution[] },
   peerDependencyResult: IPeerDependencyResult = {}
 ): Promise<IPeerDependencyResult> {
   const packageJSONCode = await getPackageJSON(dep, version);
@@ -35,9 +69,11 @@ async function getDependencyDependencies(
         return;
       }
 
-      const absoluteVersion = await getLatestVersionForSemver(
+      const absoluteVersion = await getAbsoluteVersion(
+        dep,
         depName,
-        depVersion
+        depVersion,
+        parsedResolutions
       );
 
       // eslint-disable-next-line
@@ -51,6 +87,7 @@ async function getDependencyDependencies(
       await getDependencyDependencies(
         depName,
         depVersion,
+        parsedResolutions,
         peerDependencyResult
       );
     })
@@ -59,7 +96,11 @@ async function getDependencyDependencies(
   return peerDependencyResult;
 }
 
-export async function resolveDependencyInfo(dep: string, version: string) {
+export async function resolveDependencyInfo(
+  dep: string,
+  version: string,
+  parsedResolutions: IParsedResolution[]
+) {
   const packageJSONCode = await getPackageJSON(dep, version);
   const packageJSON = JSON.parse(packageJSONCode);
   const response: ILambdaResponse = {
@@ -73,10 +114,17 @@ export async function resolveDependencyInfo(dep: string, version: string) {
     dependencyAliases: {},
   };
 
+  const resolutionsByPackage = {};
+  parsedResolutions.forEach(res => {
+    resolutionsByPackage[res.name] = resolutionsByPackage[res.name] || [];
+    resolutionsByPackage[res.name].push(res);
+  });
+
   response.peerDependencies = packageJSON.peerDependencies || {};
   response.dependencyDependencies = await getDependencyDependencies(
     dep,
-    version
+    version,
+    resolutionsByPackage
   );
 
   response.contents = {

packages/common/src/templates/template.ts

@@ -33,6 +33,9 @@ export type ParsedConfigurationFiles = {
     main: string;
     dependencies?: Dependencies;
     devDependencies: Dependencies;
+    resolutions?: {
+      [source: string]: string;
+    };
     [otherProperties: string]: any | undefined;
   }>;
   [path: string]: ParsedConfigurationFile<any> | undefined;