chore: move Code to a submodule (#4990)

* Move Code to a submodule

Closes #4901.

* Base Code cache on hash and re-enable node_modules cache

The current setup appears to only rebuild VS Code if the dependencies
change but we need to rebuild it if anything changes.

I also re-enabled the commented out node_modules caches.  They look like
they should work to me with the submodule method.  I think the problem
occurred because Code itself was being installed in the yarn step.

Author: code-asher

.github/workflows/ci.yaml

@@ -29,6 +29,9 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -38,21 +41,17 @@ jobs:
       - name: Install helm
         uses: azure/setup-helm@v1.1
 
-      # NOTE@jsjoeio
-      # disabling this until we can audit the build process
-      # and the usefulness of this step
-      # See: https://github.com/coder/code-server/issues/4287
-      # - name: Fetch dependencies from cache
-      #   id: cache-yarn
-      #   uses: actions/cache@v2
-      #   with:
-      #     path: "**/node_modules"
-      #     key: yarn-build-${{ hashFiles('**/yarn.lock') }}
-      #     restore-keys: |
-      #       yarn-build-
+      - name: Fetch dependencies from cache
+        id: cache-yarn
+        uses: actions/cache@v2
+        with:
+          path: "**/node_modules"
+          key: yarn-build-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-build-
 
       - name: Install dependencies
-        # if: steps.cache-yarn.outputs.cache-hit != 'true'
+        if: steps.cache-yarn.outputs.cache-hit != 'true'
         run: yarn --frozen-lockfile
 
       - name: Run yarn fmt
@@ -71,6 +70,9 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -102,56 +104,49 @@ jobs:
     env:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
         with:
           node-version: "14"
 
-      # TODO@Teffen investigate why this omits code-oss-dev/node_modules
-      # - name: Fetch dependencies from cache
-      #   id: cache-yarn
-      #   uses: actions/cache@v2
-      #   with:
-      #     path: |
-      #       "**/node_modules"
-      #       "**/vendor/modules"
-      #       "**/vendor/modules/code-oss-dev/node_modules"
-      #     key: yarn-build-${{ hashFiles('**/yarn.lock') }}-${{ hashFiles('**/vendor/yarn.lock') }}
-      #     restore-keys: |
-      #       yarn-build-
+      - name: Fetch dependencies from cache
+        id: cache-yarn
+        uses: actions/cache@v2
+        with:
+          path: "**/node_modules"
+          key: yarn-build-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-build-
 
       - name: Install dependencies
-        # if: steps.cache-yarn.outputs.cache-hit != 'true'
+        if: steps.cache-yarn.outputs.cache-hit != 'true'
         run: yarn --frozen-lockfile
 
       - name: Build code-server
         run: yarn build
 
-      # Parse the hash of the latest commit inside vendor/modules/code-oss-dev
-      # use this to avoid rebuilding it if nothing changed
-      # How it works: the `git log` command fetches the hash of the last commit
-      # that changed a file inside `vendor/modules/code-oss-dev`. If a commit changes any file in there,
-      # the hash returned will change, and we rebuild vscode. If the hash did not change,
-      # (for example, a change to `src/` or `docs/`), we reuse the same build as last time.
-      # This saves a lot of time in CI, as compiling VSCode can take anywhere from 5-10 minutes.
-      - name: Get latest vendor/modules/code-oss-dev rev
+      # Get Code's git hash.  When this changes it means the content is
+      # different and we need to rebuild.  Use VSCODE_CACHE_VERSION to force a
+      # rebuild.
+      - name: Get latest lib/vscode rev
         id: vscode-rev
-        run: echo "::set-output name=rev::$(jq -r '.devDependencies["code-oss-dev"]' vendor/package.json | sed -r 's|.*#(.*)$|\1|')"
+        run: echo "::set-output name=rev::$(git rev-parse HEAD:./lib/vscode)"
 
-      - name: Attempt to fetch vscode build from cache
+      - name: Fetch Code build from cache
         id: cache-vscode-2
         uses: actions/cache@v2
         with:
           path: |
-            vendor/modules/code-oss-dev/.build
-            vendor/modules/code-oss-dev/package.json
-            vendor/modules/code-oss-dev/out-build
-            vendor/modules/code-oss-dev/out-vscode-reh-web
-            vendor/modules/code-oss-dev/out-vscode-reh-web-min
+            lib/vscode/.build
+            lib/vscode/out-build
+            lib/vscode/out-vscode-reh-web
+            lib/vscode/out-vscode-reh-web-min
           key: vscode-reh-build-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}
 
       - name: Build vscode
@@ -197,7 +192,10 @@ jobs:
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - uses: actions/download-artifact@v3
         id: download
@@ -226,7 +224,10 @@ jobs:
     container: "centos:7"
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -315,7 +316,10 @@ jobs:
       NODE_VERSION: v14.17.4
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -364,7 +368,10 @@ jobs:
     runs-on: macos-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -409,7 +416,11 @@ jobs:
       # since VS Code will load faster due to the bundling.
       CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64"
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -446,12 +457,6 @@ jobs:
           ./test/node_modules/.bin/playwright install-deps
           ./test/node_modules/.bin/playwright install
 
-      # TODO@jsjoeio - remove once we switch to submodules.
-      - name: Create package.json for testing
-        run: |
-          mkdir -p ./vendor/modules/code-oss-dev
-          echo '{ "version": "test" }' > ./vendor/modules/code-oss-dev/package.json
-
       - name: Run end-to-end tests
         run: yarn test:e2e
 
@@ -468,8 +473,11 @@ jobs:
   trivy-scan-repo:
     runs-on: ubuntu-20.04
     steps:
-      - name: Checkout code
+      - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
       - name: Run Trivy vulnerability scanner in repo mode
         uses: aquasecurity/trivy-action@296212627a1e693efa09c00adc3e03b2ba8edf18
         with:
@@ -480,6 +488,7 @@ jobs:
           template: "@/contrib/sarif.tpl"
           output: "trivy-repo-results.sarif"
           severity: "HIGH,CRITICAL"
+
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v1
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/vscode"]
+	path = lib/vscode
+	url = https://github.com/coder/vscode

.prettierrc.yaml

@@ -7,7 +7,7 @@ useTabs: false
 
 overrides:
   # Attempt to keep VScode's existing code style intact.
-  - files: "vendor/modules/code-oss-dev/**/*.ts"
+  - files: "lib/vscode/**/*.ts"
     options:
       # No limit defined upstream.
       printWidth: 10000

.tours/contributing.tour

@@ -143,7 +143,7 @@
       "description": "Static images and the manifest live here in `src/browser/media` (see the explorer)."
     },
     {
-      "directory": "vendor/modules/code-oss-dev",
+      "directory": "lib/vscode",
       "line": 1,
       "description": "code-server makes use of VS Code's frontend web/remote support. Most of the modifications implement the remote server since that portion of the code is closed source and not released with VS Code.\n\nWe also have a few bug fixes and have added some features (like client-side extensions). See [https://github.com/coder/code-server/blob/master/docs/CONTRIBUTING.md#modifications-to-vs-code](https://github.com/coder/code-server/blob/master/docs/CONTRIBUTING.md#modifications-to-vs-code) for a list.\n\nWe make an effort to keep the modifications as few as possible."
     }

ci/build/build-release.sh

@@ -15,8 +15,8 @@ main() {
 
   source ./ci/lib.sh
 
-  VSCODE_SRC_PATH="vendor/modules/code-oss-dev"
-  VSCODE_OUT_PATH="$RELEASE_PATH/vendor/modules/code-oss-dev"
+  VSCODE_SRC_PATH="lib/vscode"
+  VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode"
 
   mkdir -p "$RELEASE_PATH"
 
@@ -25,7 +25,7 @@ main() {
 
   rsync ./docs/README.md "$RELEASE_PATH"
   rsync LICENSE.txt "$RELEASE_PATH"
-  rsync ./vendor/modules/code-oss-dev/ThirdPartyNotices.txt "$RELEASE_PATH"
+  rsync ./lib/vscode/ThirdPartyNotices.txt "$RELEASE_PATH"
 }
 
 bundle_code_server() {

ci/build/build-standalone-release.sh

@@ -33,8 +33,7 @@ main() {
   # HACK: the version of Typescript vscode 1.57 uses in extensions/
   # leaves a few stray symlinks. Clean them up so nfpm does not fail.
   # Remove this line when its no longer needed.
-
-  rm -fr "$RELEASE_PATH/vendor/modules/code-oss-dev/extensions/node_modules/.bin"
+  rm -fr "$RELEASE_PATH/lib/vscode/extensions/node_modules/.bin"
 }
 
 main "$@"

ci/build/build-vscode.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Builds vscode into vendor/modules/code-oss-dev/out-vscode.
+# Builds vscode into lib/vscode/out-vscode.
 
 # MINIFY controls whether a minified version of vscode is built.
 MINIFY=${MINIFY-true}
 
 main() {
   cd "$(dirname "${0}")/../.."
 
-  cd vendor/modules/code-oss-dev
+  cd lib/vscode
 
   # Any platform works since we have our own packaging step (for now).
   yarn gulp "vscode-reh-web-linux-x64${MINIFY:+-min}"

ci/build/npm-postinstall.sh

@@ -90,8 +90,8 @@ symlink_asar() {
 }
 
 vscode_yarn() {
-  echo 'Installing vendor dependencies...'
-  cd vendor/modules/code-oss-dev
+  echo 'Installing Code dependencies...'
+  cd lib/vscode
   yarn --production --frozen-lockfile
 
   symlink_asar

ci/dev/fmt.sh

@@ -19,7 +19,7 @@ main() {
     "*.sh"
   )
   prettier --write --loglevel=warn $(
-    git ls-files "${prettierExts[@]}" | grep -v "lib/vscode" | grep -v "vendor/modules/code-oss-dev" | grep -v 'helm-chart'
+    git ls-files "${prettierExts[@]}" | grep -v "lib/vscode" | grep -v 'helm-chart'
   )
 
   doctoc --title '# FAQ' docs/FAQ.md > /dev/null

ci/dev/lint.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
 
-  eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
-  stylelint $(git ls-files "*.css" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
+  eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js" | grep -v "lib/vscode")
+  stylelint $(git ls-files "*.css" | grep -v "lib/vscode")
   tsc --noEmit --skipLibCheck
-  shellcheck -e SC2046,SC2164,SC2154,SC1091,SC1090,SC2002 $(git ls-files "*.sh" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
+  shellcheck -e SC2046,SC2164,SC2154,SC1091,SC1090,SC2002 $(git ls-files "*.sh" | grep -v "lib/vscode")
   if command -v helm && helm kubeval --help > /dev/null; then
     helm kubeval ci/helm-chart
   fi

ci/dev/postinstall.sh

@@ -1,50 +1,32 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-main() {
-  cd "$(dirname "$0")/../.."
-  source ./ci/lib.sh
-
-  pushd test
-  echo "Installing dependencies for $PWD"
-  yarn install
-  popd
-
+# Install dependencies in $1.
+install-deps() {
   local args=(install)
   if [[ ${CI-} ]]; then
     args+=(--frozen-lockfile)
   fi
-
-  pushd test
-  echo "Installing dependencies for $PWD"
-  yarn "${args[@]}"
-  popd
-
-  pushd test/e2e/extensions/test-extension
+  # If there is no package.json then yarn will look upward and end up installing
+  # from the root resulting in an infinite loop (this can happen if you have not
+  # checked out the submodule yet for example).
+  if [[ ! -f "$1/package.json" ]]; then
+    echo "$1/package.json is missing; did you run git submodule update --init?"
+    exit 1
+  fi
+  pushd "$1"
   echo "Installing dependencies for $PWD"
   yarn "${args[@]}"
   popd
+}
 
-  pushd vendor
-  echo "Installing dependencies for $PWD"
-
-  # We install in 'modules' instead of 'node_modules' because VS Code's
-  # extensions use a webpack config which cannot differentiate between its own
-  # node_modules and itself being in a directory with the same name.
-  args+=(--modules-folder modules)
-
-  # We ignore scripts because NPM/Yarn's default behavior is to assume that
-  # devDependencies are not needed, and that even git repo based packages are
-  # assumed to be compiled. Because the default behavior for VS Code's
-  # `postinstall` assumes we're also compiled, this needs to be ignored.
-  args+=(--ignore-scripts)
-
-  yarn "${args[@]}"
-
-  # Finally, run the vendor `postinstall`
-  yarn run postinstall
+main() {
+  cd "$(dirname "$0")/../.."
+  source ./ci/lib.sh
 
-  popd
+  install-deps test
+  install-deps test/e2e/extensions/test-extension
+  install-deps lib/vscode
 }
 
 main "$@"