Skip to content

Commit ad604d2

Browse files
authored
Fix the dynamic downloading of babel getting stuck (codesandbox#2474)
* Fix the dynamic downloading of babel getting stuck * Make id generation sequential * Upgrade cache
1 parent 4c72474 commit ad604d2

File tree

3 files changed

+99
-86
lines changed

3 files changed

+99
-86
lines changed

.circleci/config.yml

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@ aliases:
2525

2626
- &restore_deps_cache
2727
keys:
28-
- v27-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
29-
- v27-dependency-cache-{{ .Branch }}
30-
- v27-dependency-cache
28+
- v28-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
29+
- v28-dependency-cache-{{ .Branch }}
30+
- v28-dependency-cache
3131

3232
- &save_deps_cache
33-
key: v27-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
33+
key: v28-dependency-cache-{{ .Branch }}-{{ checksum "yarn.lock" }}
3434
paths:
3535
- node_modules
3636
- packages/app/node_modules
@@ -49,51 +49,51 @@ aliases:
4949

5050
- &restore_standalone_deps_cache
5151
keys:
52-
- v27-standalone-dependency-cache-{{ .Branch }}-{{ checksum
52+
- v28-standalone-dependency-cache-{{ .Branch }}-{{ checksum
5353
"standalone-packages/codesandbox-browserfs/yarn.lock" }}
54-
- v27-standalone-dependency-cache-{{ .Branch }}
55-
- v27-standalone-dependency-cache
54+
- v28-standalone-dependency-cache-{{ .Branch }}
55+
- v28-standalone-dependency-cache
5656

5757
- &save_standalone_deps_cache
5858
key:
59-
v27-standalone-dependency-cache-{{ .Branch }}-{{ checksum
59+
v28-standalone-dependency-cache-{{ .Branch }}-{{ checksum
6060
"standalone-packages/codesandbox-browserfs/yarn.lock" }}
6161
paths:
6262
- standalone-packages/codesandbox-browserfs/node_modules
6363

6464
- &restore_prod_homepage_cache
65-
key: v27-prod-homepage-build-cache-master
65+
key: v28-prod-homepage-build-cache-master
6666

6767
- &restore_prod_result
6868
key:
69-
v27-prod-build-result-{{ .Environment.CIRCLE_BRANCH
69+
v28-prod-build-result-{{ .Environment.CIRCLE_BRANCH
7070
}}-{{.Environment.CIRCLE_SHA1 }}
7171

7272
- &save_prod_build_cache
7373
key:
74-
v27-prod-build-cache-{{ .Environment.CIRCLE_BRANCH
74+
v28-prod-build-cache-{{ .Environment.CIRCLE_BRANCH
7575
}}-{{.Environment.CIRCLE_SHA1 }}
7676
paths:
7777
- packages/app/node_modules/.cache
7878

7979
- &restore_prod_build_cache
8080
keys:
81-
- v27-prod-build-cache-{{ .Environment.CIRCLE_BRANCH
81+
- v28-prod-build-cache-{{ .Environment.CIRCLE_BRANCH
8282
}}-{{.Environment.CIRCLE_SHA1 }}
83-
- v27-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-
84-
- v27-prod-build-cache-master-
83+
- v28-prod-build-cache-{{ .Environment.CIRCLE_BRANCH }}-
84+
- v28-prod-build-cache-master-
8585

8686
# To persist the images built by sharp
8787
- &save_prod_homepage_cache
8888
key:
89-
v27-prod-homepage-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{
89+
v28-prod-homepage-build-cache-{{ .Environment.CIRCLE_BRANCH }}-{{
9090
.Environment.CIRCLE_SHA1 }}
9191
paths:
9292
- ./packages/homepage/public
9393

9494
- &save_prod_result
9595
key:
96-
v27-prod-build-result-{{ .Environment.CIRCLE_BRANCH }}-{{
96+
v28-prod-build-result-{{ .Environment.CIRCLE_BRANCH }}-{{
9797
.Environment.CIRCLE_SHA1 }}
9898
paths:
9999
- ./www
@@ -134,14 +134,14 @@ commands:
134134
steps:
135135
- restore_cache:
136136
keys:
137-
- v27-source-cache-{{ .Branch }}-{{ .Revision }}
138-
- v27-source-cache-{{ .Branch }}-
139-
- v27-source-cache-
137+
- v28-source-cache-{{ .Branch }}-{{ .Revision }}
138+
- v28-source-cache-{{ .Branch }}-
139+
- v28-source-cache-
140140

141141
- checkout
142142

143143
- save_cache:
144-
key: v27-source-cache-{{ .Branch }}-{{ .Revision }}
144+
key: v28-source-cache-{{ .Branch }}-{{ .Revision }}
145145
paths:
146146
- '.git'
147147
build_deps:

packages/app/src/sandbox/eval/npm/fetch-npm-module.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,11 @@ const urlProtocols = {
114114
},
115115
};
116116

117-
async function fetchWithRetries(url: string, retries = 3): Promise<string> {
117+
async function fetchWithRetries(url: string, retries = 3): Promise<Response> {
118118
const doFetch = () =>
119119
window.fetch(url).then(x => {
120120
if (x.ok) {
121-
return x.text();
121+
return x;
122122
}
123123

124124
throw new Error(`Could not fetch ${url}`);
@@ -176,7 +176,7 @@ async function getMeta(
176176
const protocol = getFetchProtocol(version);
177177
const metaUrl = await protocol.meta(nameWithoutAlias, version);
178178

179-
metas[id] = window.fetch(metaUrl).then(x => x.json());
179+
metas[id] = fetchWithRetries(metaUrl).then(x => x.json());
180180

181181
return metas[id];
182182
}
@@ -205,6 +205,7 @@ export async function downloadDependency(
205205
const url = await protocol.file(nameWithoutAlias, depVersion, relativePath);
206206

207207
packages[id] = fetchWithRetries(url)
208+
.then(x => x.text())
208209
.catch(async () => {
209210
const fallbackProtocol = getFetchProtocol(depVersion, true);
210211
const fallbackUrl = await fallbackProtocol.file(
@@ -213,7 +214,7 @@ export async function downloadDependency(
213214
relativePath
214215
);
215216

216-
return fetchWithRetries(fallbackUrl);
217+
return fetchWithRetries(fallbackUrl).then(x => x.text());
217218
})
218219
.then(x => ({
219220
path,

packages/app/src/sandbox/eval/transpilers/babel/worker/dynamic-download.js renamed to packages/app/src/sandbox/eval/transpilers/babel/worker/dynamic-download.ts

Lines changed: 74 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
import resolve from 'browser-resolve';
2+
import { getGlobal } from '@codesandbox/common/lib/utils/global';
23
import getRequireStatements from './simple-get-require-statements';
34

4-
const path = self.BrowserFS.BFSRequire('path');
5+
const global = getGlobal();
6+
const path = global.BrowserFS.BFSRequire('path');
57

6-
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
7-
const fs = self.BrowserFS.BFSRequire('fs');
8+
function mkDirByPathSync(
9+
targetDir: string,
10+
{ isRelativeToScript = false } = {}
11+
) {
12+
const fs = global.BrowserFS.BFSRequire('fs');
813

914
const { sep } = path;
1015
const initDir = path.isAbsolute(targetDir) ? sep : '';
@@ -36,54 +41,77 @@ function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
3641
}, initDir);
3742
}
3843

44+
interface IResolveResponse {
45+
found: boolean;
46+
type: 'resolve-async-transpiled-module-response';
47+
id: number;
48+
path: string;
49+
code: string;
50+
}
51+
52+
const downloadCache = new Map<string, Promise<IResolveResponse>>();
53+
let lastSentId = 0;
54+
3955
export const resolveAsyncModule = (
4056
modulePath: string,
41-
{ ignoredExtensions }?: { ignoredExtensions?: Array<string> }
42-
) =>
43-
new Promise((r, reject) => {
44-
const sendId = Math.floor(Math.random() * 10000);
45-
self.postMessage({
46-
type: 'resolve-async-transpiled-module',
47-
path: modulePath,
48-
id: sendId,
49-
options: { isAbsolute: true, ignoredExtensions },
50-
});
51-
52-
const resolveFunc = message => {
53-
const { type, id, found } = message.data;
54-
55-
if (
56-
type === 'resolve-async-transpiled-module-response' &&
57-
id === sendId
58-
) {
59-
if (found) {
60-
r(message.data);
61-
} else {
62-
reject(new Error("Could not find path: '" + modulePath + "'."));
57+
{ ignoredExtensions }: { ignoredExtensions?: Array<string> }
58+
): Promise<IResolveResponse> => {
59+
if (downloadCache.get(modulePath)) {
60+
return downloadCache.get(modulePath);
61+
}
62+
63+
downloadCache.set(
64+
modulePath,
65+
new Promise((r, reject) => {
66+
const sendId = lastSentId++;
67+
68+
global.postMessage({
69+
type: 'resolve-async-transpiled-module',
70+
path: modulePath,
71+
id: sendId,
72+
options: { isAbsolute: true, ignoredExtensions },
73+
});
74+
75+
const resolveFunc = (message: { data: IResolveResponse }) => {
76+
const { type, id, found } = message.data;
77+
78+
if (
79+
type === 'resolve-async-transpiled-module-response' &&
80+
id === sendId
81+
) {
82+
if (found) {
83+
r(message.data);
84+
} else {
85+
reject(new Error("Could not find path: '" + modulePath + "'."));
86+
}
87+
global.removeEventListener('message', resolveFunc);
6388
}
64-
self.removeEventListener('message', resolveFunc);
65-
}
66-
};
89+
};
6790

68-
self.addEventListener('message', resolveFunc);
69-
});
91+
global.addEventListener('message', resolveFunc);
92+
})
93+
);
7094

71-
const downloads = {};
72-
export async function downloadPath(absolutePath) {
95+
return downloadCache.get(modulePath);
96+
};
97+
98+
export async function downloadPath(
99+
absolutePath: string
100+
): Promise<{ code: string; path: string }> {
73101
const r = await resolveAsyncModule(absolutePath, {});
74102

75103
if (!r.found) {
76104
throw new Error(`${absolutePath} not found.`);
77105
}
78106

79-
self.postMessage({
107+
global.postMessage({
80108
type: 'add-transpilation-dependency',
81109
path: r.path,
82110
});
83111

84-
const fs = self.BrowserFS.BFSRequire('fs');
112+
const fs = global.BrowserFS.BFSRequire('fs');
85113

86-
let existingFile;
114+
let existingFile: string;
87115

88116
try {
89117
existingFile = fs.readFileSync(r.path);
@@ -108,27 +136,23 @@ export async function downloadPath(absolutePath) {
108136
/* ignore */
109137
}
110138

111-
return Promise.resolve({
139+
return {
112140
code: existingFile,
113141
path: r.path,
114-
});
142+
};
115143
}
116144

117145
mkDirByPathSync(path.dirname(r.path));
118-
119146
fs.writeFileSync(r.path, r.code);
120147

121148
const requires = getRequireStatements(r.code);
122149

150+
// Download all other needed files
123151
await Promise.all(
124-
requires.map(foundR => {
152+
requires.map(async foundR => {
125153
if (foundR.type === 'direct') {
126154
if (foundR.path === 'babel-plugin-macros') {
127-
return '';
128-
}
129-
130-
if (downloads[foundR.path]) {
131-
return downloads[foundR.path];
155+
return;
132156
}
133157

134158
try {
@@ -137,35 +161,23 @@ export async function downloadPath(absolutePath) {
137161
extensions: ['.js', '.json'],
138162
moduleDirectory: ['node_modules'],
139163
});
140-
return '';
141164
} catch (e) {
142-
// eslint-disable-next-line no-use-before-define
143-
downloads[foundR.path] = downloadFromError(e)
144-
.then(() => {
145-
delete downloads[foundR.path];
146-
})
147-
.catch(() => {
148-
delete downloads[foundR.path];
149-
});
150-
return downloads[foundR.path];
165+
await downloadFromError(e);
151166
}
152167
}
153-
return Promise.resolve();
154168
})
155169
);
156170

157171
return r;
158172
}
159173

160-
export async function downloadFromError(e) {
174+
export function downloadFromError(e: Error) {
161175
if (e.message.indexOf('Cannot find module') > -1) {
162-
return new Promise(res => {
163-
const dep = e.message.match(/Cannot find module '(.*?)'/)[1];
164-
const from = e.message.match(/from '(.*?)'/)[1];
165-
const absolutePath = dep.startsWith('.') ? path.join(from, dep) : dep;
176+
const dep = e.message.match(/Cannot find module '(.*?)'/)[1];
177+
const from = e.message.match(/from '(.*?)'/)[1];
178+
const absolutePath = dep.startsWith('.') ? path.join(from, dep) : dep;
166179

167-
res(downloadPath(absolutePath));
168-
});
180+
return downloadPath(absolutePath);
169181
}
170182

171183
return Promise.resolve();

0 commit comments

Comments
 (0)