chore(dev): adjust benchmark parameters (#229)

For increased accuracy/stability, the python binding is no longer run in
the benchmark CI.

It can still be run locally but would take an extra call
(`install-py-binding`) to build the binding before running benchmark.nu
(`main`).

Author: 2bndy5

.github/workflows/benchmark.yml

@@ -68,37 +68,37 @@ jobs:
           name: ${{ matrix.name }}
           path: ${{ env.BIN }}
 
-  build-py-binding:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          persist-credentials: false
-      - name: Set up Python
-        uses: actions/setup-python@v6
-        id: setup-python
-        with:
-          python-version: '3.x'
-      - name: Build wheels
-        uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 # v1.49.4
-        with:
-          target: x86_64
-          args: --release --out dist --find-interpreter
-          manylinux: auto
-          before-script-linux: |
-            # NOTE: rust-cross/manylinux docker images are CentOS based
-            yum update -y
-            yum install -y openssl openssl-devel
-      - name: Upload wheels
-        uses: actions/upload-artifact@v5
-        with:
-          name: wheel
-          path: dist/cpp_linter-*-cp3*-abi3-*.whl
-          if-no-files-found: error
+  # build-py-binding:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v6
+  #       with:
+  #         persist-credentials: false
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v6
+  #       id: setup-python
+  #       with:
+  #         python-version: '3.x'
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 # v1.49.4
+  #       with:
+  #         target: x86_64
+  #         args: --release --out dist --find-interpreter
+  #         manylinux: auto
+  #         before-script-linux: |
+  #           # NOTE: rust-cross/manylinux docker images are CentOS based
+  #           yum update -y
+  #           yum install -y openssl openssl-devel
+  #     - name: Upload wheels
+  #       uses: actions/upload-artifact@v5
+  #       with:
+  #         name: wheel
+  #         path: dist/cpp_linter-*-cp3*-abi3-*.whl
+  #         if-no-files-found: error
 
   benchmark:
     name: Measure Performance Difference
-    needs: [build-bin, build-py-binding]
+    needs: [build-bin] #, build-py-binding]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -132,11 +132,11 @@ jobs:
         working-directory: benchmark
         shell: nu {0}
         run: |-
-          let new_py = (
-            glob "../wheel/cpp_linter-*-cp3*-abi3-*.whl"
-            | first
-            | path expand
-          )
+          # let new_py = (
+          #   glob "../wheel/cpp_linter-*-cp3*-abi3-*.whl"
+          #   | first
+          #   | path expand
+          # )
           let prev_bin = "../previous/cpp-linter" | path expand
           let curr_bin = "../current/cpp-linter" | path expand
-          nu benchmark.nu --new-py $new_py --rust-bin $curr_bin --prev-rust-bin $prev_bin
+          nu benchmark.nu --rust-bin $curr_bin --prev-rust-bin $prev_bin

benchmark/benchmark.nu

@@ -122,8 +122,13 @@ export def run-hyperfine [
         --export-markdown ../benchmark.md
         --warmup 1
         --style color
-        --runs 3
+        --runs
     ]
+    if ($new_path | is-not-empty) {
+        $hyperfine_args = $hyperfine_args | append 3
+    } else {
+        $hyperfine_args = $hyperfine_args | append 4
+    }
     let common_args = "-l 0 -a 0 -i=|!src/libgit2 -p build -e c"
     if ($old_path | is-not-empty) {
         $hyperfine_args = $hyperfine_args | append [--command-name python-pure $"($old_path) ($common_args) -j 0"]
@@ -165,9 +170,9 @@ export def summarize [] {
 # 4. Clones the libgit2 repository if it does not exist.
 # 5. Runs benchmarks using hyperfine and saves the results to benchmark.json.
 export def main [
-    --new-py: string, # path to wheel of new version of cpp-linter (v2.x)
     --rust-bin: string, # path to rust binary of cpp-linter (v2.x)
     --prev-rust-bin: string, # path to previous commit's release build of rust binary (v2.x)
+    --new-py: string = "", # path to wheel of new version of cpp-linter (v2.x)
 ] {
     let is_on_win = sys host | get name | str starts-with "Windows"
     let rust_bin = if ($rust_bin | is-not-empty) {
@@ -182,7 +187,11 @@ export def main [
         run-cmd chmod +x $prev_rust_bin
     }
     let old_path = install-old
-    let new_path = install-py-binding $new_py
+    let new_path = if ($new_py | is-not-empty) {
+        install-py-binding $new_py
+    } else {
+        ""
+    }
     checkout-libgit2
     run-hyperfine $old_path $new_path $rust_bin $prev_rust_bin
     summarize