Merge remote-tracking branch 'upstream/master'

* upstream/master:
  Fix error when shared process exits with null
  Add flags for customizing user data dir and extensions dir (coder#420)
  Initialize backup service (coder#419)
  Add port in use message (coder#418)
  Improve CI caching (coder#416)
  Update notes title

Author: andreimc

.gitignore

@@ -4,3 +4,4 @@ dist
 out
 .DS_Store
 release
+.cache

.travis.yml

@@ -9,11 +9,12 @@ env:
 matrix:
   include:
   - os: linux
-    dist: ubuntu
+    dist: trusty
   - os: osx
 before_install:
 - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
   libsecret-1-dev; fi
+- npm install -g yarn@1.12.3
 script:
 - scripts/build.sh
 before_deploy:
@@ -39,4 +40,8 @@ deploy:
     repo: devonlineco/code-server
     all_branches: true
     condition: $TRAVIS_BRANCH =~ ^master|testing|stable$
-cache: yarn
+cache:
+  yarn: true
+  timeout: 1000
+  directories:
+  - .cache

README.md

@@ -63,7 +63,7 @@ How to [secure your setup](/doc/security/ssl.md).
 - Electron and Chrome OS applications to bridge the gap between local<->remote.
 - Run VS Code unit tests against our builds to ensure features work as expected.
 
-### Notes
+### Extensions
 
 At the moment we can't use the official VSCode Marketplace. We've created a custom extension marketplace focused around open-sourced extensions. However, if you have access to the `.vsix` file, you can manually install the extension.
 

build/tasks.ts

@@ -4,21 +4,22 @@ import * as fse from "fs-extra";
 import * as os from "os";
 import * as path from "path";
 import * as zlib from "zlib";
+import * as https from "https";
+import * as tar from "tar";
 
 const isWin = os.platform() === "win32";
 const libPath = path.join(__dirname, "../lib");
 const vscodePath = path.join(libPath, "vscode");
+const defaultExtensionsPath = path.join(libPath, "extensions");
 const pkgsPath = path.join(__dirname, "../packages");
-const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions");
 const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0";
+const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`;
 
 const buildServerBinary = register("build:server:binary", async (runner) => {
 	await ensureInstalled();
-	await copyForDefaultExtensions();
 	await Promise.all([
 		buildBootstrapFork(),
 		buildWeb(),
-		buildDefaultExtensions(),
 		buildServerBundle(),
 		buildAppBrowser(),
 	]);
@@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => {
 	await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]);
 });
 
-const extDirPath = path.join("lib", "vscode-default-extensions");
-const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => {
-	if (!fs.existsSync(defaultExtensionsPath)) {
-		await ensureClean();
-		await ensureInstalled();
-		await new Promise((resolve, reject): void => {
-			fse.remove(extDirPath, (err) => {
-				if (err) {
-					return reject(err);
-				}
-
-				resolve();
-			});
-		});
-		await new Promise((resolve, reject): void => {
-			fse.copy(vscodePath, extDirPath, (err) => {
-				if (err) {
-					return reject(err);
-				}
-
-				resolve();
-			});
-		});
-	}
-});
-
-const buildDefaultExtensions = register("build:default-extensions", async (runner) => {
-	if (!fs.existsSync(defaultExtensionsPath)) {
-		await copyForDefaultExtensions();
-		runner.cwd = extDirPath;
-		const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]);
-		if (resp.exitCode !== 0) {
-			throw new Error(`Failed to build default extensions: ${resp.stderr}`);
-		}
-	}
-});
-
 const ensureInstalled = register("vscode:install", async (runner) => {
-	await ensureCloned();
+	runner.cwd = libPath;
 
-	runner.cwd = vscodePath;
-	const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []);
-	if (install.exitCode !== 0) {
-		throw new Error(`Failed to install vscode dependencies: ${install.stderr}`);
-	}
-});
+	if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) {
+		const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version;
+		if (pkgVersion === vscodeVersion) {
+			runner.cwd = vscodePath;
 
-const ensureCloned = register("vscode:clone", async (runner) => {
-	if (fs.existsSync(vscodePath)) {
-		await ensureClean();
-	} else {
-		fse.mkdirpSync(libPath);
-		runner.cwd = libPath;
-		const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]);
-		if (clone.exitCode !== 0) {
-			throw new Error(`Failed to clone: ${clone.exitCode}`);
+			const reset = await runner.execute("git", ["reset", "--hard"]);
+			if (reset.exitCode !== 0) {
+				throw new Error(`Failed to clean git repository: ${reset.stderr}`);
+			}
+
+			return;
 		}
 	}
 
-	runner.cwd = vscodePath;
-	const checkout = await runner.execute("git", ["checkout", vscodeVersion]);
-	if (checkout.exitCode !== 0) {
-		throw new Error(`Failed to checkout: ${checkout.stderr}`);
-	}
-});
+	fse.removeSync(libPath);
+	fse.mkdirpSync(libPath);
 
-const ensureClean = register("vscode:clean", async (runner) => {
-	runner.cwd = vscodePath;
+	await new Promise<void>((resolve, reject): void => {
+		https.get(vsSourceUrl, (res) => {
+			if (res.statusCode !== 200) {
+				return reject(res.statusMessage);
+			}
 
-	const status = await runner.execute("git", ["status", "--porcelain"]);
-	if (status.stdout.trim() !== "") {
-		const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]);
-		if (clean.exitCode !== 0) {
-			throw new Error(`Failed to clean git repository: ${clean.stderr}`);
-		}
-		const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]);
-		if (removeUnstaged.exitCode !== 0) {
-			throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`);
-		}
-	}
-	const fetch = await runner.execute("git", ["fetch", "--prune"]);
-	if (fetch.exitCode !== 0) {
-		throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`);
-	}
+			res.pipe(tar.x({
+				C: libPath,
+			}).on("finish", () => {
+				resolve();
+			}).on("error", (err: Error) => {
+				reject(err);
+			}));
+		}).on("error", (err) => {
+			reject(err);
+		});
+	});
 });
 
 const ensurePatched = register("vscode:patch", async (runner) => {
 	if (!fs.existsSync(vscodePath)) {
 		throw new Error("vscode must be cloned to patch");
 	}
-	await ensureClean();
+	await ensureInstalled();
 
 	runner.cwd = vscodePath;
 	const patchPath = path.join(__dirname, "../scripts/vscode.patch");

package.json

@@ -15,7 +15,9 @@
   "devDependencies": {
     "@types/fs-extra": "^5.0.4",
     "@types/node": "^10.12.18",
+    "@types/tar": "^4.0.0",
     "@types/trash": "^4.3.1",
+    "cache-loader": "^2.0.1",
     "cross-env": "^5.2.0",
     "crypto-browserify": "^3.12.0",
     "css-loader": "^2.1.0",
@@ -34,6 +36,8 @@
     "sass-loader": "^7.1.0",
     "string-replace-loader": "^2.1.1",
     "style-loader": "^0.23.1",
+    "tar": "^4.4.8",
+    "terser-webpack-plugin": "^1.2.3",
     "ts-loader": "^5.3.3",
     "ts-node": "^7.0.1",
     "tsconfig-paths": "^3.8.0",

packages/protocol/src/browser/client.ts

@@ -271,6 +271,7 @@ export class Client {
 					workingDirectory: init.getWorkingDirectory(),
 					os: protoToOperatingSystem(init.getOperatingSystem()),
 					shell: init.getShell(),
+					extensionsDirectory: init.getExtensionsDirectory(),
 					builtInExtensionsDirectory: init.getBuiltinExtensionsDir(),
 				};
 				this.initDataEmitter.emit(this._initData);

packages/protocol/src/common/connection.ts

@@ -23,6 +23,7 @@ export interface InitData {
 	readonly homeDirectory: string;
 	readonly tmpDirectory: string;
 	readonly shell: string;
+	readonly extensionsDirectory: string;
 	readonly builtInExtensionsDirectory: string;
 }
 

packages/protocol/src/node/server.ts

@@ -14,6 +14,7 @@ export interface ServerOptions {
 	readonly dataDirectory: string;
 	readonly cacheDirectory: string;
 	readonly builtInExtensionsDirectory: string;
+	readonly extensionsDirectory: string;
 	readonly fork?: ForkProvider;
 }
 
@@ -93,6 +94,7 @@ export class Server {
 		initMsg.setDataDirectory(this.options.dataDirectory);
 		initMsg.setWorkingDirectory(this.options.workingDirectory);
 		initMsg.setBuiltinExtensionsDir(this.options.builtInExtensionsDirectory);
+		initMsg.setExtensionsDirectory(this.options.extensionsDirectory);
 		initMsg.setHomeDirectory(os.homedir());
 		initMsg.setTmpDirectory(os.tmpdir());
 		initMsg.setOperatingSystem(platformToProto(os.platform()));

packages/protocol/src/proto/client.proto

@@ -41,4 +41,5 @@ message WorkingInit {
 	OperatingSystem operating_system = 5;
 	string shell = 6;
 	string builtin_extensions_dir = 7;
+	string extensions_directory = 8;
 }

packages/protocol/src/proto/client_pb.d.ts

@@ -132,6 +132,9 @@ export class WorkingInit extends jspb.Message {
   getBuiltinExtensionsDir(): string;
   setBuiltinExtensionsDir(value: string): void;
 
+  getExtensionsDirectory(): string;
+  setExtensionsDirectory(value: string): void;
+
   serializeBinary(): Uint8Array;
   toObject(includeInstance?: boolean): WorkingInit.AsObject;
   static toObject(includeInstance: boolean, msg: WorkingInit): WorkingInit.AsObject;
@@ -151,6 +154,7 @@ export namespace WorkingInit {
     operatingSystem: WorkingInit.OperatingSystem,
     shell: string,
     builtinExtensionsDir: string,
+    extensionsDirectory: string,
   }
 
   export enum OperatingSystem {

packages/protocol/src/proto/client_pb.js

@@ -794,7 +794,8 @@ proto.WorkingInit.toObject = function(includeInstance, msg) {
     workingDirectory: jspb.Message.getFieldWithDefault(msg, 4, ""),
     operatingSystem: jspb.Message.getFieldWithDefault(msg, 5, 0),
     shell: jspb.Message.getFieldWithDefault(msg, 6, ""),
-    builtinExtensionsDir: jspb.Message.getFieldWithDefault(msg, 7, "")
+    builtinExtensionsDir: jspb.Message.getFieldWithDefault(msg, 7, ""),
+    extensionsDirectory: jspb.Message.getFieldWithDefault(msg, 8, "")
   };
 
   if (includeInstance) {
@@ -859,6 +860,10 @@ proto.WorkingInit.deserializeBinaryFromReader = function(msg, reader) {
       var value = /** @type {string} */ (reader.readString());
       msg.setBuiltinExtensionsDir(value);
       break;
+    case 8:
+      var value = /** @type {string} */ (reader.readString());
+      msg.setExtensionsDirectory(value);
+      break;
     default:
       reader.skipField();
       break;
@@ -937,6 +942,13 @@ proto.WorkingInit.serializeBinaryToWriter = function(message, writer) {
       f
     );
   }
+  f = message.getExtensionsDirectory();
+  if (f.length > 0) {
+    writer.writeString(
+      8,
+      f
+    );
+  }
 };
 
 
@@ -1054,4 +1066,19 @@ proto.WorkingInit.prototype.setBuiltinExtensionsDir = function(value) {
 };
 
 
+/**
+ * optional string extensions_directory = 8;
+ * @return {string}
+ */
+proto.WorkingInit.prototype.getExtensionsDirectory = function() {
+  return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 8, ""));
+};
+
+
+/** @param {string} value */
+proto.WorkingInit.prototype.setExtensionsDirectory = function(value) {
+  jspb.Message.setProto3StringField(this, 8, value);
+};
+
+
 goog.object.extend(exports, proto);

packages/server/package.json

@@ -9,7 +9,7 @@
 		"build:binary": "ts-node scripts/nbin.ts"
 	},
 	"dependencies": {
-		"@coder/nbin": "^1.0.4",
+		"@coder/nbin": "^1.0.5",
 		"commander": "^2.19.0",
 		"express": "^4.16.4",
 		"express-static-gzip": "^1.1.3",