diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index d2b3a5302..4706b5c07 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -25,12 +25,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install qemu-system-arm qemu-efi-aarch64 -y
 
-    - name: Build
-      run: cargo xtask build --target aarch64
-
     - name: Run VM tests
       run: cargo xtask run --target aarch64 --headless --ci
-      timeout-minutes: 2
+      timeout-minutes: 4
 
   test_x86_64:
     name: Build and run tests on x86_64
@@ -44,12 +41,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install qemu-system-x86 ovmf swtpm -y
 
-    - name: Build (without unstable)
-      run: cargo xtask build --target x86_64
-
     - name: Run VM tests
       run: cargo xtask run --target x86_64 --headless --ci --tpm=v1
-      timeout-minutes: 2
+      timeout-minutes: 4
 
   test_ia32:
     name: Build and run tests on IA32
@@ -63,12 +57,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install qemu-system-x86 ovmf-ia32 swtpm -y
 
-    - name: Build
-      run: cargo xtask build --target ia32
-
     - name: Run VM tests
       run: cargo xtask run --target ia32 --headless --ci --tpm=v2
-      timeout-minutes: 2
+      timeout-minutes: 4
 
   test:
     name: Run tests and documentation tests
@@ -147,12 +138,9 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Build
-        run: cargo xtask build
-
       - name: Run VM tests
         run: cargo xtask run --target x86_64 --ci
-        timeout-minutes: 2
+        timeout-minutes: 4
 
   # Run the build with our current nightly MSRV (specified in
   # ./msrv_toolchain.toml). This serves to check that we don't