fix(testing): reduce flakiness in extension e2e tests

test/e2e/extensions.test.ts

@@ -8,12 +8,14 @@ function runTestExtensionTests() {
   test("should have access to VSCODE_PROXY_URI", async ({ codeServerPage }) => {
     const address = await getMaybeProxiedCodeServer(codeServerPage)
 
+    await codeServerPage.reloadUntilTestExtensionIsLoaded()
     await codeServerPage.executeCommandViaMenus("code-server: Get proxy URI")
 
-    const text = await codeServerPage.page.locator(".notification-list-item-message").textContent()
+    await codeServerPage.page.waitForSelector("text=proxyUri", { timeout: 3000 })
+    const text = await codeServerPage.page.locator("text=proxyUri").first().textContent()
     // Remove end slash in address
     const normalizedAddress = address.replace(/\/+$/, "")
-    expect(text).toBe(`${normalizedAddress}/proxy/{{port}}`)
+    expect(text).toBe(`Info: proxyUri: ${normalizedAddress}/proxy/{{port}}`)
   })
 }
 

test/e2e/extensions/test-extension/extension.ts

@@ -1,10 +1,11 @@
 import * as vscode from "vscode"
 
 export function activate(context: vscode.ExtensionContext) {
+  vscode.window.showInformationMessage("test extension loaded")
   context.subscriptions.push(
     vscode.commands.registerCommand("codeServerTest.proxyUri", () => {
       if (process.env.VSCODE_PROXY_URI) {
-        vscode.window.showInformationMessage(process.env.VSCODE_PROXY_URI)
+        vscode.window.showInformationMessage(`proxyUri: ${process.env.VSCODE_PROXY_URI}`)
       } else {
         vscode.window.showErrorMessage("No proxy URI was set")
       }

test/e2e/extensions/test-extension/package.json

@@ -4,7 +4,7 @@
   "version": "0.0.1",
   "publisher": "coder",
   "activationEvents": [
-    "onCommand:codeServerTest.proxyUri"
+    "onStartupFinished"
   ],
   "engines": {
     "vscode": "^1.56.0"

test/e2e/models/CodeServer.ts

@@ -282,6 +282,32 @@ export class CodeServerPage {
     this.codeServer.logger.debug("Editor is ready!")
   }
 
+  /**
+   * Checks if the test extension has loaded
+   *
+   * Reload until both checks pass
+   */
+  async reloadUntilTestExtensionIsLoaded() {
+    this.codeServer.logger.debug("Waiting for test extenion to load...")
+
+    const extensionIsLoaded = await this.isTestExtensionLoaded()
+    let reloadCount = 0
+
+    while (!extensionIsLoaded) {
+      await this.page.waitForLoadState("load")
+      // Give it an extra second just in case it's feeling extra slow
+      await this.page.waitForTimeout(1000)
+      reloadCount += 1
+      if (await this.isTestExtensionLoaded()) {
+        this.codeServer.logger.debug(`test extension loaded after ${reloadCount} reloads`)
+        break
+      }
+      await this.reloadUntilEditorIsReady()
+    }
+
+    this.codeServer.logger.debug("Test extenion has loaded!")
+  }
+
   /**
    * Checks if the editor is visible
    */
@@ -296,6 +322,21 @@ export class CodeServerPage {
     return visible
   }
 
+  /**
+   * Checks if the test extension loaded
+   */
+  async isTestExtensionLoaded() {
+    const selector = "text=test extension loaded"
+    this.codeServer.logger.debug("Waiting for test extension to load...")
+
+    await this.page.waitForLoadState("load")
+    const loaded = await this.page.isVisible(selector)
+
+    this.codeServer.logger.debug(`Test extension has ${loaded ? "" : "not"} loaded`)
+
+    return loaded
+  }
+
   /**
    * Focuses the integrated terminal by navigating through the command palette.
    *