ci: disable gha caching for docker builder image

Author: kateinoigakukun

.github/workflows/build.yml

@@ -66,8 +66,6 @@ jobs:
           push: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/tags/*' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=min
 
   build-rake-task-matrix:
     runs-on: ubuntu-24.04
@@ -120,8 +118,6 @@ jobs:
         with:
           context: builders/${{ matrix.entry.builder }}
           load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=min
       - name: Setup builder container
         run: |
           docker run -dit -v "$GITHUB_WORKSPACE:/home/me/build" -w /home/me/build \