Skip to content
This repository was archived by the owner on Aug 7, 2021. It is now read-only.

Commit a8138d2

Browse files
DimitarTachevDimitarTachev
committed
refactor: make the snapshot generation more readable
1 parent 6861d22 commit a8138d2

File tree

2 files changed

+142
-129
lines changed

2 files changed

+142
-129
lines changed

‎snapshot/android/snapshot-generator.js

Lines changed: 133 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
const fs = require("fs");
22
const { dirname, relative, join, EOL } = require("path");
33
const child_process = require("child_process");
4+
const { convertToUnixPath } = require("../../lib/utils");
45

56
const shelljs = require("shelljs");
67

7-
const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher } = require("./utils");
8+
const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher, isSubPath } = require("./utils");
89

910
const SNAPSHOTS_DOCKER_IMAGE = "nativescript/v8-snapshot:latest";
1011
const SNAPSHOT_TOOLS_DIR_NAME = "mksnapshot-tools";
@@ -75,33 +76,25 @@ SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputF
7576

7677
const snapshotToolsDownloads = {};
7778

78-
SnapshotGenerator.prototype.downloadMkSnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, useDocker) {
79+
SnapshotGenerator.prototype.downloadMksnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, snapshotInDocker) {
7980
var toolsOS = "";
8081
var toolsArch = "";
81-
if (typeof useDocker === "boolean") {
82-
if (useDocker) {
83-
toolsOS = DOCKER_IMAGE_OS;
84-
toolsArch = DOCKER_IMAGE_ARCH;
85-
} else {
86-
toolsOS = getHostOS();
87-
toolsArch = getHostOSArch();
88-
}
82+
if (snapshotInDocker) {
83+
toolsOS = DOCKER_IMAGE_OS;
84+
toolsArch = DOCKER_IMAGE_ARCH;
8985
} else {
9086
toolsOS = getHostOS();
9187
toolsArch = getHostOSArch();
92-
fallbackToDocker = true;
9388
}
9489

95-
96-
9790
return Promise.all(targetArchs.map((arch) => {
98-
return this.downloadMkSnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => {
91+
return this.downloadMksnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => {
9992
return { path, arch };
10093
});
10194
}));
10295
}
10396

104-
SnapshotGenerator.prototype.downloadMkSnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) {
97+
SnapshotGenerator.prototype.downloadMksnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) {
10598
const mksnapshotToolRelativePath = join(SNAPSHOT_TOOLS_DIR_NAME, "v8-v" + v8Version, hostOS + "-" + hostArch, "mksnapshot-" + targetArch);
10699
const mksnapshotToolPath = join(snapshotToolsPath, mksnapshotToolRelativePath);
107100
if (fs.existsSync(mksnapshotToolPath))
@@ -139,39 +132,37 @@ SnapshotGenerator.prototype.convertToAndroidArchName = function (archName) {
139132
}
140133
}
141134

142-
SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, useDocker) {
135+
SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, snapshotInDocker) {
143136
// Cleans the snapshot build folder
144-
debugger;
145137
shelljs.rm("-rf", join(this.buildPath, "snapshots"));
146-
return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, useDocker).then((localTools) => {
147-
var snapshotInDocker = !!useDocker;
138+
return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, snapshotInDocker).then((localTools) => {
148139
var shouldDownloadDockerTools = false;
149140
if (!snapshotInDocker) {
150141
snapshotInDocker = localTools.some(tool => !this.canUseSnapshotTool(tool.path));
151142
shouldDownloadDockerTools = snapshotInDocker;
152143
}
153144

154145
if (shouldDownloadDockerTools) {
155-
return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => {
146+
return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => {
156147
console.log(`Executing '${snapshotToolPath}' in a docker container.`);
157-
return this.runMKSnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
148+
return this.runMksnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
158149
});
159150
} else {
160-
return this.runMKSnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
151+
return this.runMksnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
161152
}
162153
});
163154
}
164155

165156

166-
SnapshotGenerator.prototype.runMKSnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) {
157+
SnapshotGenerator.prototype.runMksnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) {
167158
let currentSnapshotOperation = Promise.resolve();
168159
const canRunInParallel = snapshotTools.length <= 1 || !snapshotInDocker;
169160
return Promise.all(snapshotTools.map((tool) => {
170161
if (canRunInParallel) {
171-
return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
162+
return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
172163
} else {
173164
currentSnapshotOperation = currentSnapshotOperation.then(() => {
174-
return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
165+
return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
175166
});
176167

177168
return currentSnapshotOperation;
@@ -193,14 +184,11 @@ SnapshotGenerator.prototype.canUseSnapshotTool = function (snapshotToolPath) {
193184
}
194185

195186
SnapshotGenerator.prototype.setupDocker = function () {
196-
const installMessage = "Install Docker and add it to your PATH in order to build snapshots.";
197187
try {
198-
// e.g. Docker version 19.03.2, build 6a30dfc
199188
child_process.execSync(`docker --version`);
200-
// TODO: require a minimum version?
201189
}
202190
catch (error) {
203-
throw new Error(`Docker installation cannot be found. ${installMessage}`);
191+
throw new Error(`Docker installation cannot be found. Install Docker and add it to your PATH in order to build snapshots.`);
204192
}
205193

206194
child_process.execSync(`docker pull ${SNAPSHOTS_DOCKER_IMAGE}`);
@@ -257,116 +245,133 @@ SnapshotGenerator.prototype.generate = function (options) {
257245
});
258246
}
259247

260-
SnapshotGenerator.prototype.runMKSnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource) {
261-
const currentArchMksnapshotToolPath = tool.path;
262-
const arch = tool.arch;
263-
if (!fs.existsSync(currentArchMksnapshotToolPath)) {
264-
throw new Error("Can't find mksnapshot tool for " + arch + " at path " + currentArchMksnapshotToolPath);
248+
SnapshotGenerator.prototype.getSnapshotToolCommand = function (snapshotToolPath, inputFilePath, outputPath, toolParams) {
249+
return `${snapshotToolPath} ${inputFilePath} --startup_blob ${join(outputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${toolParams}`;
250+
}
251+
252+
SnapshotGenerator.prototype.getXxdCommand = function (srcOutputDir) {
253+
return `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`)}`;
254+
}
255+
256+
SnapshotGenerator.prototype.getPathInDocker = function (mappedLocalDir, mappedDockerDir, targetPath) {
257+
if (!isSubPath(mappedLocalDir, targetPath)) {
258+
throw new Error(`Cannot determine a docker path. '${targetPath}' should be inside '${mappedLocalDir}'`)
259+
}
260+
261+
const pathInDocker = join(mappedDockerDir, relative(mappedLocalDir, targetPath));
262+
263+
return convertToUnixPath(pathInDocker);
264+
}
265+
266+
SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, stderr, androidArch) {
267+
let toolError = null;
268+
const errorHeader = `Target architecture: ${androidArch}\n`;
269+
let errorFooter = ``;
270+
if (stderr.length || error) {
271+
try {
272+
require(inputFile);
273+
}
274+
catch (e) {
275+
errorFooter = `\nJavaScript execution error: ${e.stack}$`;
276+
}
277+
}
278+
279+
if (stderr.length) {
280+
const message = `${errorHeader}${stderr}${errorFooter}`;
281+
toolError = new Error(message);
282+
}
283+
else if (error) {
284+
error.message = `${errorHeader}${error.message}${errorFooter}`;
285+
toolError = error;
286+
} else {
287+
console.log(stdout);
288+
}
289+
290+
return toolError;
291+
}
292+
293+
SnapshotGenerator.prototype.copySnapshotTool = function (allToolsDir, targetTool, destinationDir) {
294+
// we cannot mount the source tools folder as its not shared by default:
295+
// docker: Error response from daemon: Mounts denied:
296+
// The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools
297+
// is not shared from OS X and is not known to Docker.
298+
const toolPathRelativeToAllToolsDir = relative(allToolsDir, targetTool);
299+
const toolDestinationPath = join(destinationDir, toolPathRelativeToAllToolsDir)
300+
createDirectory(dirname(toolDestinationPath));
301+
shelljs.cp(targetTool, toolDestinationPath);
302+
303+
return toolDestinationPath;
304+
}
305+
306+
SnapshotGenerator.prototype.buildCSource = function (androidArch, blobInputDir, snapshotInDocker) {
307+
const srcOutputDir = join(this.buildPath, "snapshots/src", androidArch);
308+
createDirectory(srcOutputDir);
309+
let command = "";
310+
if (snapshotInDocker) {
311+
const blobsInputInDocker = `/blobs/${androidArch}`
312+
const srcOutputDirInDocker = `/dist/src/${androidArch}`;
313+
const buildCSourceCommand = this.getXxdCommand(srcOutputDirInDocker);
314+
// add vim in order to get xxd
315+
command = `docker run -v "${blobInputDir}:${blobsInputInDocker}" -v "${srcOutputDir}:${srcOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${blobsInputInDocker} && apk add vim && ${buildCSourceCommand}"`;
316+
}
317+
else {
318+
command = this.getXxdCommand(srcOutputDir);
265319
}
320+
shellJsExecuteInDir(blobInputDir, function () {
321+
shelljs.exec(command);
322+
});
323+
}
266324

267-
const androidArch = this.convertToAndroidArchName(arch);
268-
console.log("***** Generating snapshot for " + androidArch + " *****");
269-
// Generate .blob file
270-
const currentArchBlobOutputPath = join(this.buildPath, "snapshots/blobs", androidArch);
271-
shelljs.mkdir("-p", currentArchBlobOutputPath);
272-
let dockerCurrentArchBlobOutputPath = "";
273-
var params = "--profile_deserialization";
274-
if (mksnapshotParams) {
275-
// Starting from android runtime 5.3.0, the parameters passed to mksnapshot are read from the settings.json file
276-
params = mksnapshotParams;
325+
SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, allToolsDir, buildCSource) {
326+
const toolPath = tool.path;
327+
const androidArch = this.convertToAndroidArchName(tool.arch);
328+
if (!fs.existsSync(toolPath)) {
329+
throw new Error("Can't find mksnapshot tool for " + androidArch + " at path " + toolPath);
277330
}
331+
332+
const tempFolders = [];
278333
return new Promise((resolve, reject) => {
279-
let snapshotToolPath = currentArchMksnapshotToolPath;
334+
console.log("***** Generating snapshot for " + androidArch + " *****");
280335
const inputFileDir = dirname(inputFile);
281-
let inputFilePath = inputFile;
282-
const appDirInDocker = "/app";
283-
let outputPath = currentArchBlobOutputPath;
336+
const blobOutputDir = join(this.buildPath, "snapshots/blobs", androidArch);
337+
createDirectory(blobOutputDir);
338+
const toolParams = mksnapshotParams || "--profile_deserialization";
339+
340+
let command = "";
284341
if (snapshotInDocker) {
285342
this.setupDocker();
286-
// create snapshots dir in docker
287-
dockerCurrentArchBlobOutputPath = join(inputFileDir, "snapshots", androidArch);
288-
shelljs.mkdir("-p", dockerCurrentArchBlobOutputPath);
289-
outputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchBlobOutputPath));
290-
// calculate input in docker
291-
inputFilePath = join(appDirInDocker, relative(inputFileDir, inputFile));
292-
// calculate snapshotTool path
293-
snapshotToolPath = join(appDirInDocker, relative(snapshotToolsBasePath, currentArchMksnapshotToolPath));
294-
}
295-
let command = `${snapshotToolPath} ${inputFilePath} --startup_blob ${join(outputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${params}`;
296-
if (snapshotInDocker) {
297-
// we cannot mount the source tools folder as its not shared by default:
298-
// docker: Error response from daemon: Mounts denied:
299-
// The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools
300-
// is not shared from OS X and is not known to Docker.
301-
const currentToolRelativeToSnapshotTools = relative(snapshotToolsBasePath, currentArchMksnapshotToolPath);
302-
const currentToolDestination = join(inputFileDir, currentToolRelativeToSnapshotTools)
303-
debugger;
304-
createDirectory(dirname(currentToolDestination));
305-
shelljs.cp(currentArchMksnapshotToolPath, join(inputFileDir, currentToolRelativeToSnapshotTools));
306-
command = `docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${command}"`;
343+
const appDirInDocker = "/app";
344+
const blobOutputDirInDocker = `/dist/blobs/${androidArch}`;
345+
const toolsTempFolder = join(inputFileDir, "tmp");
346+
tempFolders.push(toolsTempFolder);
347+
const toolPathInAppDir = this.copySnapshotTool(allToolsDir, toolPath, toolsTempFolder);
348+
const toolPathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, toolPathInAppDir);
349+
const inputFilePathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, inputFile);
350+
const outputPathInDocker = this.getPathInDocker(blobOutputDir, blobOutputDirInDocker, blobOutputDir);
351+
const toolCommandInDocker = this.getSnapshotToolCommand(toolPathInDocker, inputFilePathInDocker, outputPathInDocker, toolParams);
352+
command = `docker run -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`;
353+
} else {
354+
command = this.getSnapshotToolCommand(toolPath, inputFilePath, outputPath, toolParams);
307355
}
356+
357+
// Generate .blob file
308358
child_process.exec(command, { encoding: "utf8" }, (error, stdout, stderr) => {
309-
const errorHeader = `Target architecture: ${androidArch}\n`;
310-
let errorFooter = ``;
311-
if (stderr.length || error) {
312-
try {
313-
require(inputFile);
314-
}
315-
catch (e) {
316-
errorFooter = `\nJavaScript execution error: ${e.stack}$`;
317-
}
318-
}
319-
if (stderr.length) {
320-
const message = `${errorHeader}${stderr}${errorFooter}`;
321-
reject(new Error(message));
322-
}
323-
else if (error) {
324-
error.message = `${errorHeader}${error.message}${errorFooter}`;
325-
reject(error);
326-
}
327-
else {
328-
console.log(stdout);
329-
// Generate .c file
330-
/// TODO: test in docker
331-
if (buildCSource) {
332-
let srcOutputPath = "";
333-
let dockerCurrentArchSrcOutputPath = "";
334-
if (snapshotInDocker) {
335-
dockerCurrentArchSrcOutputPath = join(inputFileDir, "snapshots/src", androidArch);
336-
shelljs.mkdir("-p", dockerCurrentArchSrcOutputPath);
337-
srcOutputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchSrcOutputPath));
338-
} else {
339-
srcOutputPath = join(this.buildPath, "snapshots/src", androidArch);
340-
shelljs.mkdir("-p", srcOutputPath);
341-
}
342-
343-
const buildCSourceCommand = `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputPath, `${SNAPSHOT_BLOB_NAME}.c`)}`;
344-
if (snapshotInDocker) {
345-
// add vim in order to get xxd
346-
const commandInDocker =
347-
`docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${outputPath} && apk add vim && ${buildCSourceCommand}"`;
348-
child_process.execSync(commandInDocker);
349-
} else {
350-
shellJsExecuteInDir(currentArchBlobOutputPath, function () {
351-
shelljs.exec(buildCSourceCommand);
352-
});
353-
}
354-
355-
if (snapshotInDocker) {
356-
createDirectory(join(this.buildPath, "snapshots/src", androidArch));
357-
shelljs.cp("-R", dockerCurrentArchSrcOutputPath + "/", join(this.buildPath, "snapshots/src", androidArch));
358-
shelljs.rm("-rf", dockerCurrentArchSrcOutputPath);
359-
}
360-
}
361-
362-
// TODO: move cleanup to afterPrepare?
363-
if (snapshotInDocker) {
364-
shelljs.cp("-R", dockerCurrentArchBlobOutputPath + "/", currentArchBlobOutputPath);
365-
shelljs.rm("-rf", dockerCurrentArchBlobOutputPath);
366-
}
367-
resolve();
359+
tempFolders.forEach(tempFolder => {
360+
shelljs.rm("-rf", tempFolder);
361+
});
362+
363+
const snapshotError = this.handleSnapshotToolResult(error, stdout, stderr, androidArch);
364+
if (snapshotError) {
365+
return reject(error);
368366
}
367+
368+
return resolve(blobOutputDir);
369369
});
370+
}).then((blobOutputDir) => {
371+
// Generate .c file
372+
if (buildCSource) {
373+
this.buildCSource(androidArch, blobOutputDir, snapshotInDocker)
374+
}
370375
});
371376
}
372377

‎snapshot/android/utils.js

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
const { chmodSync, createWriteStream, existsSync } = require("fs");
22
const { tmpdir, EOL } = require("os");
3-
const { join } = require("path");
3+
const { join, relative, isAbsolute } = require("path");
44
const os = require("os");
55

66
const { mkdir } = require("shelljs");
@@ -40,6 +40,13 @@ function has32BitArch(targetArchs) {
4040
return Array.isArray(targetArchs) && targetArchs.some(arch => arch === "arm" || arch === "ia32")
4141
}
4242

43+
function isSubPath(parentPath, childPath) {
44+
const relativePath = relative(parentPath, childPath);
45+
46+
return relativePath === "" ||
47+
(relativePath && !relativePath.startsWith('..') && !isAbsolute(relativePath));
48+
}
49+
4350
function isMacOSCatalinaOrHigher() {
4451
const isCatalinaOrHigher = false;
4552
const catalinaVersion = "19.0.0";
@@ -113,4 +120,5 @@ module.exports = {
113120
isMacOSCatalinaOrHigher,
114121
downloadFile,
115122
getJsonFile,
123+
isSubPath
116124
};

0 commit comments

Comments
 (0)