diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index 5d6fc7f5a..9653a18fd 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -156,16 +156,6 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
-        restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Push to container registry
@@ -181,18 +171,6 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: true
-    - name: Cache scons
-      id: scons-cache
-      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
-      env:
-        CACHE_SKIP_SAVE: true
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
     - name: Build Docker image
       run: eval "$BUILD"
     - name: pre-commit
@@ -247,13 +225,11 @@ jobs:
       # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
       uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
       env:
-        CACHE_SKIP_SAVE: true
+        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
       with:
         path: /tmp/scons_cache
-        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-        restore-keys: |
-          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
-          scons-
+        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
+        restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Run unit tests