Skip to content

Commit 3817199

Browse files
CompuIvesCompuIves
authored
Dynamic dep download (codesandbox#2364)
* Dynamic dependency downloading * Fix caches * Set default peerDependencies * Support different url * Add description * Automatically detect whether to use dynamic fetching * Fix lint
1 parent 06132be commit 3817199

File tree

6 files changed

+139
-97
lines changed

6 files changed

+139
-97
lines changed

‎packages/app/src/sandbox/compile.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import {
2727
findBoilerplate,
2828
} from './boilerplates';
2929

30-
import loadDependencies from './npm';
30+
import { loadDependencies } from './npm';
3131
import { consumeCache, saveCache, deleteAPICache } from './eval/cache';
3232

3333
import { showRunOnClick } from './status-screen/run-on-click';
@@ -484,6 +484,7 @@ async function compile({
484484
templateDefinition,
485485
configurations
486486
);
487+
487488
const { manifest, isNewCombination } = await loadDependencies(
488489
dependencies,
489490
disableDependencyPreprocessing

‎packages/app/src/sandbox/eval/npm/fetch-npm-module.ts‎

Lines changed: 99 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,69 @@ export let combinedMetas: Meta = {}; // eslint-disable-line
3131
const normalizedMetas: { [key: string]: Meta } = {};
3232
const packages: Packages = {};
3333

34+
function normalize(files: MetaFiles, fileObject: Meta = {}, rootPath: string) {
35+
for (let i = 0; i < files.length; i += 1) {
36+
if (files[i].type === 'file') {
37+
const absolutePath = rootPath + files[i].path;
38+
fileObject[absolutePath] = true; // eslint-disable-line no-param-reassign
39+
}
40+
41+
if (files[i].files) {
42+
normalize(files[i].files, fileObject, rootPath);
43+
}
44+
}
45+
46+
return fileObject;
47+
}
48+
49+
function normalizeJSDelivr(files: any, fileObject: Meta = {}, rootPath) {
50+
for (let i = 0; i < files.length; i += 1) {
51+
const absolutePath = pathUtils.join(rootPath, files[i].name);
52+
fileObject[absolutePath] = true; // eslint-disable-line no-param-reassign
53+
}
54+
55+
return fileObject;
56+
}
57+
58+
const urlProtocols = {
59+
csbGH: {
60+
file: async (name: string, version: string, path: string) =>
61+
`${version.replace(/\/_pkg.tgz$/, '')}${path}`,
62+
meta: async (name: string, version: string) =>
63+
`${version.replace(/\/\.pkg.tgz$/, '')}/_csb-meta.json`,
64+
normalizeMeta: normalize,
65+
},
66+
unpkg: {
67+
file: async (name: string, version: string, path: string) =>
68+
`https://unpkg.com/${name}@${version}/${path}`,
69+
meta: async (name: string, version: string) =>
70+
`https://unpkg.com/${name}@${version}/?meta`,
71+
normalizeMeta: normalize,
72+
},
73+
jsDelivrNPM: {
74+
file: async (name: string, version: string, path: string) =>
75+
`https://cdn.jsdelivr.net/npm/${name}@${version}${path}`,
76+
meta: async (name: string, version: string) =>
77+
`https://data.jsdelivr.com/v1/package/npm/${name}@${version}/flat`,
78+
normalizeMeta: normalizeJSDelivr,
79+
},
80+
jsDelivrGH: {
81+
file: async (name: string, version: string, path: string) =>
82+
`https://cdn.jsdelivr.net/gh/${version}${path}`,
83+
meta: async (name: string, version: string) => {
84+
// First get latest sha from GitHub API
85+
const sha = await fetch(
86+
`https://api.github.com/repos/${version}/commits/master`
87+
)
88+
.then(x => x.json())
89+
.then(x => x.sha);
90+
91+
return `https://data.jsdelivr.com/v1/package/gh/${version}@${sha}/flat`;
92+
},
93+
normalizeMeta: normalizeJSDelivr,
94+
},
95+
};
96+
3497
async function fetchWithRetries(url: string, retries = 3): Promise<string> {
3598
const doFetch = () =>
3699
window.fetch(url).then(x => {
@@ -60,53 +123,28 @@ export function setCombinedMetas(givenCombinedMetas: Meta) {
60123
combinedMetas = givenCombinedMetas;
61124
}
62125

63-
function normalize(
64-
depName: string,
65-
files: MetaFiles,
66-
fileObject: Meta = {},
67-
rootPath: string
68-
) {
69-
for (let i = 0; i < files.length; i += 1) {
70-
if (files[i].type === 'file') {
71-
const absolutePath = rootPath + files[i].path;
72-
fileObject[absolutePath] = true; // eslint-disable-line no-param-reassign
73-
}
126+
const CSB_DRAFT_PROTOCOL = /https:\/\/pkg(-staging)?\.csb.dev/;
74127

75-
if (files[i].files) {
76-
normalize(depName, files[i].files, fileObject, rootPath);
77-
}
128+
const getFetchProtocol = (depVersion: string, useFallback = false) => {
129+
const isDraftProtocol = CSB_DRAFT_PROTOCOL.test(depVersion);
130+
131+
if (isDraftProtocol) {
132+
return urlProtocols.csbGH;
78133
}
79134

80-
return fileObject;
81-
}
135+
const isGitHub = /\//.test(depVersion);
82136

83-
function normalizeJSDelivr(
84-
depName: string,
85-
files: any,
86-
fileObject: Meta = {},
87-
rootPath
88-
) {
89-
for (let i = 0; i < files.length; i += 1) {
90-
const absolutePath = pathUtils.join(rootPath, files[i].name);
91-
fileObject[absolutePath] = true; // eslint-disable-line no-param-reassign
137+
if (isGitHub) {
138+
return urlProtocols.jsDelivrGH;
92139
}
93140

94-
return fileObject;
95-
}
141+
return useFallback ? urlProtocols.jsDelivrNPM : urlProtocols.unpkg;
142+
};
96143

97-
const TEMP_USE_JSDELIVR = false;
98144
// Strips the version of a path, eg. test/1.3.0 -> test
99145
const ALIAS_REGEX = /\/\d*\.\d*\.\d*.*?(\/|$)/;
100146

101-
function getUnpkgUrl(name: string, version: string, forceJsDelivr?: boolean) {
102-
const nameWithoutAlias = name.replace(ALIAS_REGEX, '');
103-
104-
return TEMP_USE_JSDELIVR || forceJsDelivr
105-
? `https://cdn.jsdelivr.net/npm/${nameWithoutAlias}@${version}`
106-
: `https://unpkg.com/${nameWithoutAlias}@${version}`;
107-
}
108-
109-
function getMeta(
147+
async function getMeta(
110148
name: string,
111149
packageJSONPath: string | null,
112150
version: string
@@ -117,24 +155,22 @@ function getMeta(
117155
return metas[id];
118156
}
119157

120-
metas[id] = window
121-
.fetch(
122-
TEMP_USE_JSDELIVR
123-
? `https://data.jsdelivr.com/v1/package/npm/${nameWithoutAlias}@${version}/flat`
124-
: `https://unpkg.com/${nameWithoutAlias}@${version}/?meta`
125-
)
126-
.then(x => x.json());
158+
const protocol = getFetchProtocol(version);
159+
const metaUrl = await protocol.meta(nameWithoutAlias, version);
160+
161+
metas[id] = window.fetch(metaUrl).then(x => x.json());
127162

128163
return metas[id];
129164
}
130165

131-
function downloadDependency(
166+
export async function downloadDependency(
132167
depName: string,
133168
depVersion: string,
134169
path: string
135170
): Promise<Module> {
136-
if (packages[path]) {
137-
return packages[path];
171+
const id = depName + depVersion + path;
172+
if (packages[id]) {
173+
return packages[id];
138174
}
139175

140176
const relativePath = path
@@ -145,30 +181,29 @@ function downloadDependency(
145181
''
146182
)
147183
.replace(/#/g, '%23');
148-
const isGitHub = /\//.test(depVersion);
149184

150-
const url = isGitHub
151-
? `https://cdn.jsdelivr.net/gh/${depVersion}${relativePath}`
152-
: `${getUnpkgUrl(depName, depVersion)}${relativePath}`;
153-
154-
packages[path] = fetchWithRetries(url)
155-
.catch(err => {
156-
if (!isGitHub) {
157-
// Fallback to jsdelivr
158-
return fetchWithRetries(
159-
`${getUnpkgUrl(depName, depVersion, true)}${relativePath}`
160-
);
161-
}
185+
const nameWithoutAlias = depName.replace(ALIAS_REGEX, '');
186+
const protocol = getFetchProtocol(depVersion);
187+
const url = await protocol.file(nameWithoutAlias, depVersion, relativePath);
188+
189+
packages[id] = fetchWithRetries(url)
190+
.catch(async () => {
191+
const fallbackProtocol = getFetchProtocol(depVersion, true);
192+
const fallbackUrl = await fallbackProtocol.file(
193+
nameWithoutAlias,
194+
depVersion,
195+
relativePath
196+
);
162197

163-
throw err;
198+
return fetchWithRetries(fallbackUrl);
164199
})
165200
.then(x => ({
166201
path,
167202
code: x,
168203
downloaded: true,
169204
}));
170205

171-
return packages[path];
206+
return packages[id];
172207
}
173208

174209
function resolvePath(
@@ -357,14 +392,14 @@ export default async function fetchModule(
357392

358393
const meta = await getMeta(dependencyName, packageJSONPath, version);
359394

360-
const normalizeFunction = TEMP_USE_JSDELIVR ? normalizeJSDelivr : normalize;
395+
const normalizeFunction = getFetchProtocol(version).normalizeMeta;
361396
const rootPath = packageJSONPath
362397
? pathUtils.dirname(packageJSONPath)
363398
: pathUtils.join('/node_modules', dependencyName);
364399
const normalizedCacheKey = dependencyName + rootPath;
365400
const normalizedMeta =
366401
normalizedMetas[normalizedCacheKey] ||
367-
normalizeFunction(dependencyName, meta.files, {}, rootPath);
402+
normalizeFunction(meta.files, {}, rootPath);
368403
normalizedMetas[normalizedCacheKey] = normalizedMeta;
369404
combinedMetas = { ...combinedMetas, ...normalizedMeta };
370405

packages/app/src/sandbox/npm/dependencies-to-query.js renamed to packages/app/src/sandbox/npm/dependencies-to-query.ts

File renamed without changes.

packages/app/src/sandbox/npm/fetch-dependencies.js renamed to packages/app/src/sandbox/npm/fetch-dependencies.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import delay from '../utils/delay';
88
import setScreen from '../status-screen';
99

1010
type Dependencies = {
11-
[dependency: string]: string,
11+
[dependency: string]: string;
1212
};
1313

1414
const RETRY_COUNT = 60;
@@ -43,11 +43,13 @@ function callApi(url: string, method = 'GET') {
4343
})
4444
.then(async response => {
4545
if (!response.ok) {
46-
const error = new Error(response.statusText || response.status);
46+
const error = new Error(response.statusText || '' + response.status);
4747

4848
const message = await response.json();
4949

50+
// @ts-ignore
5051
error.response = message;
52+
// @ts-ignore
5153
error.statusCode = response.status;
5254
return Promise.reject(error);
5355
}
@@ -143,7 +145,7 @@ async function getDependencies(dependencies: Object) {
143145
}
144146
}
145147

146-
export default async function fetchDependencies(npmDependencies: Dependencies) {
148+
export async function fetchDependencies(npmDependencies: Dependencies) {
147149
if (Object.keys(npmDependencies).length !== 0) {
148150
// New Packager flow
149151

packages/app/src/sandbox/npm/index.js renamed to packages/app/src/sandbox/npm/index.ts

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,35 @@
11
import { pickBy } from 'lodash-es';
22

3-
import fetchDependencies from './fetch-dependencies';
3+
import { fetchDependencies } from './fetch-dependencies';
44
import { getDependencyVersions } from '../version-resolving';
55
import dependenciesToQuery from './dependencies-to-query';
66

77
import setScreen from '../status-screen';
88

9-
let loadedDependencyCombination: ?string = null;
9+
let loadedDependencyCombination: string | null = null;
1010
let manifest = null;
1111

1212
type NPMDependencies = {
13-
[dependency: string]: string,
13+
[dependency: string]: string;
1414
};
1515

16+
/**
17+
* If there is a URL to a file we need to fetch the dependenices dynamically, at least
18+
* for the first version. In the future we might want to consider a hybrid version where
19+
* we only fetch the dynamic files for dependencies with a url as version. But this is a good
20+
* start.
21+
*/
22+
function shouldFetchDynamically(dependencies: NPMDependencies) {
23+
return Object.keys(dependencies).some(depName =>
24+
dependencies[depName].includes('http')
25+
);
26+
}
27+
1628
/**
1729
* This fetches the manifest and dependencies from the
1830
* @param {*} dependencies
1931
*/
20-
export default async function loadDependencies(
32+
export async function loadDependencies(
2133
dependencies: NPMDependencies,
2234
disableExternalConnection = false
2335
) {
@@ -34,9 +46,15 @@ export default async function loadDependencies(
3446
if (loadedDependencyCombination !== depQuery) {
3547
isNewCombination = true;
3648

37-
const data = await (disableExternalConnection
49+
const fetchDynamically =
50+
disableExternalConnection ||
51+
shouldFetchDynamically(dependenciesWithoutTypings);
52+
53+
const fetchFunction = fetchDynamically
3854
? getDependencyVersions
39-
: fetchDependencies)(dependenciesWithoutTypings);
55+
: fetchDependencies;
56+
57+
const data = await fetchFunction(dependenciesWithoutTypings);
4058

4159
// Mark that the last requested url is this
4260
loadedDependencyCombination = depQuery;

0 commit comments

Comments
 (0)