Move benchmarks to separate repo, take N

.github/workflows/benchmark.yml

@@ -17,22 +17,26 @@ jobs:
   build:
     name: Benchmark Test on os ${{ matrix.os }} and llvm ${{ matrix.llvm }} mode ${{ matrix.build }} 
     runs-on: ${{ matrix.os }}
-
+    container:
+      image: ${{ (contains(matrix.os, 'linux') && 'ghcr.io/enzymead/reactant-docker-images@sha256:91e1edb7a7c869d5a70db06e417f22907be0e67ca86641d48adcea221fedc674' ) || '' }}
+
     strategy:
       fail-fast: false
       matrix:
-        llvm: ["16", "17", "18"]
+        llvm: ["15"]
         build: ["Release"] #, "Debug" "RelWithDebInfo"
-        os: [] # [openstack22]
+        os: ["linux-x86-n2-32"]
     timeout-minutes: 120 
     steps:
     - name: add llvm 
       run: |
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-          sudo apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
-          sudo apt-get install -y python3-pip autoconf cmake gcc g++ libtool gfortran libblas-dev llvm-${{ matrix.llvm }}-dev clang-${{ matrix.llvm }} libeigen3-dev libboost-dev libzstd-dev
-          sudo python3 -m pip install lit pathlib
-          sudo touch /usr/lib/llvm-${{ matrix.llvm }}/bin/yaml-bench
+          # curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+          apt-get update -y
+          apt-get install -y lsb-release software-properties-common
+          apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
+          apt-get  --allow-unauthenticated install -y python3-pip autoconf cmake gcc g++ libtool gfortran libblas-dev llvm-${{ matrix.llvm }}-dev clang-${{ matrix.llvm }} libeigen3-dev libboost-dev libzstd-dev
+          python3 -m pip install lit pathlib --break-system-packages
+          touch /usr/lib/llvm-${{ matrix.llvm }}/bin/yaml-bench
     - uses: actions/checkout@v4
     - name: mkdir
       run: rm -rf build && mkdir build
@@ -46,4 +50,5 @@ jobs:
       working-directory: build
       run: make bench-enzyme
     - name: results
-      run: cat enzyme/benchmarks/*/*/results.txt
+      working-directory: build
+      run: cat _deps/cpu_benchmarks-src/*/*/results.json

enzyme/CMakeLists.txt

@@ -296,8 +296,18 @@ endif()
 
 # The benchmarks data are not in git-exported source archives to minimize size.
 # Only add the benchmarks if the directory exists.
-if (ENZYME_ENABLE_PLUGINS AND (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/benchmarks"))
-    add_subdirectory(benchmarks)
+if (ENZYME_ENABLE_PLUGINS)
+	include (FetchContent)
+	FetchContent_Declare(cpu_benchmarks
+	    GIT_REPOSITORY https://github.com/EnzymeAD/enzyme-benchmarks
+	    GIT_TAG        "9fa1579296b28d9642f8b69a6fffb753211c3f88"
+	)
+
+	if(NOT cpu_benchmarks_POPULATED)
+	  FetchContent_Populate(cpu_benchmarks)
+
+	  add_subdirectory(${cpu_benchmarks_SOURCE_DIR} ${cpu_benchmarks_BINARY_DIR} EXCLUDE_FROM_ALL)
+	endif()
 endif()
 
 # Make relative paths absolute (needed later on)

enzyme/Enzyme/Utils.cpp

@@ -4040,6 +4040,7 @@ bool notCapturedBefore(llvm::Value *V, Instruction *inst,
     auto pair = todo.pop_back_val();
     if (seen.count(pair))
       continue;
+    seen.insert(pair);
     auto UI = std::get<0>(pair);
     auto level = std::get<1>(pair);
     auto prev = std::get<2>(pair);