From 1ce3da701259e07b4550acb06960e51c60486df7 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Fri, 11 Oct 2024 17:06:37 +0200
Subject: [PATCH 01/13] Syntax benchmark: cleanup

---
 tests/syntax_benchmarks/Benchmark.ml | 48 ++++++++++++----------------
 1 file changed, 20 insertions(+), 28 deletions(-)

diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 4f8e2db38a..80810c5b94 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -182,14 +182,8 @@ end = struct
   type action = Parse | Print
   let string_of_action action =
     match action with
-    | Parse -> "parser"
-    | Print -> "printer"
-
-  (* TODO: we could at Reason here *)
-  type lang = Rescript
-  let string_of_lang lang =
-    match lang with
-    | Rescript -> "rescript"
+    | Parse -> "Parse"
+    | Print -> "Print"
 
   let parse_rescript src filename =
     let p = Parser.make src filename in
@@ -197,19 +191,20 @@ end = struct
     assert (p.diagnostics == []);
     structure
 
-  let benchmark filename lang action =
-    let src = IO.read_file filename in
-    let name =
-      filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action
-    in
+  let data_dir = "tests/syntax_benchmarks/data"
+
+  let benchmark filename action =
+    let path = Filename.concat data_dir filename in
+    let src = IO.read_file path in
+    let name = string_of_action action ^ " " ^ filename in
     let benchmark_fn =
-      match (lang, action) with
-      | Rescript, Parse ->
+      match action with
+      | Parse ->
         fun _ ->
-          let _ = Sys.opaque_identity (parse_rescript src filename) in
+          let _ = Sys.opaque_identity (parse_rescript src path) in
           ()
-      | Rescript, Print ->
-        let p = Parser.make src filename in
+      | Print ->
+        let p = Parser.make src path in
         let ast = ResParser.parse_implementation p in
         fun _ ->
           let _ =
@@ -226,16 +221,13 @@ end = struct
     Benchmark.report b
 
   let run () =
-    let data_dir = "tests/syntax_benchmarks/data" in
-    benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print;
-    benchmark
-      (Filename.concat data_dir "RedBlackTreeNoComments.res")
-      Rescript Print;
-    benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print;
-    benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print
+    benchmark "RedBlackTree.res" Parse;
+    benchmark "RedBlackTree.res" Print;
+    benchmark "RedBlackTreeNoComments.res" Print;
+    benchmark "Napkinscript.res" Parse;
+    benchmark "Napkinscript.res" Print;
+    benchmark "HeroGraphic.res" Parse;
+    benchmark "HeroGraphic.res" Print
 end
 
 let () = Benchmarks.run ()

From 60768c966f5b51078a9590102ed6f6775e7d6b66 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Fri, 11 Oct 2024 17:05:18 +0200
Subject: [PATCH 02/13] Output benchmark results as JSON

---
 .github/workflows/ci.yml             | 11 +++++-
 dune-project                         |  4 ++
 rescript.opam                        |  1 +
 tests/syntax_benchmarks/Benchmark.ml | 59 ++++++++++++++++------------
 tests/syntax_benchmarks/dune         |  2 +-
 5 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a19f176af3..2109dbc44e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -90,8 +90,9 @@ jobs:
             ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
             upload_binaries: true
             upload_libs: true
-            # Build the playground compiler on the fastest runner
+            # Build the playground compiler and run the benchmarks on the fastest runner
             build_playground: true
+            benchmarks: true
           - os: buildjet-2vcpu-ubuntu-2204-arm # ARM
             ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
             upload_binaries: true
@@ -320,6 +321,14 @@ jobs:
         if: runner.os != 'Windows'
         run: make -C tests/gentype_tests/typescript-react-example clean test
 
+      - name: Build playground compiler
+        if: matrix.benchmarks
+        run: |
+          opam exec -- node packages/playground-bundling/scripts/generate_cmijs.js
+          opam exec -- dune build --profile browser
+          cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js playground/compiler.js
+
+
       - name: Build playground compiler
         if: matrix.build_playground
         run: |
diff --git a/dune-project b/dune-project
index d0d3bdca87..2fa8675931 100644
--- a/dune-project
+++ b/dune-project
@@ -24,6 +24,10 @@
    (and
     :with-test
     (= 0.26.2)))
+  (yojson
+   (and
+    :with-test
+    (= 2.2.2)))
   (ocaml-lsp-server
    (and
     :with-dev-setup
diff --git a/rescript.opam b/rescript.opam
index ff2913835d..bc35c92939 100644
--- a/rescript.opam
+++ b/rescript.opam
@@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues"
 depends: [
   "ocaml" {>= "4.10"}
   "ocamlformat" {with-test & = "0.26.2"}
+  "yojson" {with-test & = "2.2.2"}
   "ocaml-lsp-server" {with-dev-setup & = "1.19.0"}
   "cppo" {= "1.6.9"}
   "js_of_ocaml" {= "5.8.1"}
diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 80810c5b94..075860cbad 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -79,7 +79,7 @@ module Benchmark : sig
 
   val make : name:string -> f:(t -> unit) -> unit -> t
   val launch : t -> unit
-  val report : t -> unit
+  val report : t -> Yojson.t list
 end = struct
   type t = {
     name: string;
@@ -98,23 +98,26 @@ end = struct
   }
 
   let report b =
-    print_endline (Format.sprintf "Benchmark: %s" b.name);
-    print_endline (Format.sprintf "Nbr of iterations: %d" b.n);
-    print_endline
-      (Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration));
-    print_endline
-      (Format.sprintf "Avg time/op: %fms"
-         (Time.print b.duration /. float_of_int b.n));
-    print_endline
-      (Format.sprintf "Allocs/op: %d"
-         (int_of_float (b.net_allocs /. float_of_int b.n)));
-    print_endline
-      (Format.sprintf "B/op: %d"
-         (int_of_float (b.net_bytes /. float_of_int b.n)));
-
-    (* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *)
-    print_newline ();
-    ()
+    [
+      `Assoc
+        [
+          ("name", `String (Format.sprintf "%s - avg. time" b.name));
+          ("unit", `String "ms");
+          ("value", `Float (Time.print b.duration /. float_of_int b.n));
+        ];
+      `Assoc
+        [
+          ("name", `String (Format.sprintf "%s - allocations" b.name));
+          ("unit", `String "");
+          ("value", `Int (int_of_float (b.net_allocs /. float_of_int b.n)));
+        ];
+      `Assoc
+        [
+          ("name", `String (Format.sprintf "%s - bytes allocated" b.name));
+          ("unit", `String "");
+          ("value", `Int (int_of_float (b.net_bytes /. float_of_int b.n)));
+        ];
+    ]
 
   let make ~name ~f () =
     {
@@ -221,13 +224,19 @@ end = struct
     Benchmark.report b
 
   let run () =
-    benchmark "RedBlackTree.res" Parse;
-    benchmark "RedBlackTree.res" Print;
-    benchmark "RedBlackTreeNoComments.res" Print;
-    benchmark "Napkinscript.res" Parse;
-    benchmark "Napkinscript.res" Print;
-    benchmark "HeroGraphic.res" Parse;
-    benchmark "HeroGraphic.res" Print
+    let results =
+      List.flatten
+        [
+          benchmark "RedBlackTree.res" Parse;
+          benchmark "RedBlackTree.res" Print;
+          benchmark "RedBlackTreeNoComments.res" Print;
+          benchmark "Napkinscript.res" Parse;
+          benchmark "Napkinscript.res" Print;
+          benchmark "HeroGraphic.res" Parse;
+          benchmark "HeroGraphic.res" Print;
+        ]
+    in
+    print_endline (Yojson.pretty_to_string (`List results))
 end
 
 let () = Benchmarks.run ()
diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune
index 0d234e3126..717ec85326 100644
--- a/tests/syntax_benchmarks/dune
+++ b/tests/syntax_benchmarks/dune
@@ -22,6 +22,6 @@
  (foreign_stubs
   (language c)
   (names time))
- (libraries syntax))
+ (libraries syntax yojson))
 
 (data_only_dirs data)

From 04cabc760ab6f8a8ac0c60d6966cc2177ba54a47 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 09:44:46 +0200
Subject: [PATCH 03/13] Output JSON lazily

---
 tests/syntax_benchmarks/Benchmark.ml | 34 ++++++++++++++++------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 075860cbad..6afcc83eb4 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -183,6 +183,7 @@ module Benchmarks : sig
   val run : unit -> unit
 end = struct
   type action = Parse | Print
+
   let string_of_action action =
     match action with
     | Parse -> "Parse"
@@ -196,7 +197,7 @@ end = struct
 
   let data_dir = "tests/syntax_benchmarks/data"
 
-  let benchmark filename action =
+  let benchmark (filename, action) =
     let path = Filename.concat data_dir filename in
     let src = IO.read_file path in
     let name = string_of_action action ^ " " ^ filename in
@@ -223,20 +224,25 @@ end = struct
     Benchmark.launch b;
     Benchmark.report b
 
+  let specs =
+    [
+      ("RedBlackTree.res", Parse);
+      ("RedBlackTree.res", Print);
+      ("RedBlackTreeNoComments.res", Print);
+      ("Napkinscript.res", Parse);
+      ("Napkinscript.res", Print);
+      ("HeroGraphic.res", Parse);
+      ("HeroGraphic.res", Print);
+    ]
+
   let run () =
-    let results =
-      List.flatten
-        [
-          benchmark "RedBlackTree.res" Parse;
-          benchmark "RedBlackTree.res" Print;
-          benchmark "RedBlackTreeNoComments.res" Print;
-          benchmark "Napkinscript.res" Parse;
-          benchmark "Napkinscript.res" Print;
-          benchmark "HeroGraphic.res" Parse;
-          benchmark "HeroGraphic.res" Print;
-        ]
-    in
-    print_endline (Yojson.pretty_to_string (`List results))
+    List.to_seq specs
+    |> Seq.flat_map (fun spec -> benchmark spec |> List.to_seq)
+    |> Seq.iteri (fun i json ->
+           print_endline (if i == 0 then "[" else ",");
+           print_string (Yojson.to_string json));
+    print_newline ();
+    print_endline "]"
 end
 
 let () = Benchmarks.run ()

From b6850bf0c24be6524b0f59e9879ceb26bb66be82 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 09:13:41 +0200
Subject: [PATCH 04/13] Increment opam cache key

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2109dbc44e..659ef79163 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -151,7 +151,7 @@ jobs:
       # matrix.ocaml_compiler may contain commas
       - name: Get OPAM cache key
         shell: bash
-        run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
+        run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
 
       - name: Restore OPAM environment
         id: cache-opam-env

From 5439c76643d35502ac6e5453ffaef8652a659123 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 10:22:35 +0200
Subject: [PATCH 05/13] Build benchmarks only for OCaml >= 4.14.0

---
 tests/syntax_benchmarks/dune | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune
index 717ec85326..d45805d291 100644
--- a/tests/syntax_benchmarks/dune
+++ b/tests/syntax_benchmarks/dune
@@ -9,6 +9,7 @@
  (enabled_if
   (and
    (<> %{profile} browser)
+   (>= %{ocaml_version} "4.14.0")
    (or
     (= %{system} macosx)
     ; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613)

From 8e0c826a8e601a0e908b0f73da60a4345d6fe323 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 09:10:39 +0200
Subject: [PATCH 06/13] Continuous benchmarking in CI

---
 .github/workflows/ci.yml | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 659ef79163..41ad7d1ef4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -321,13 +321,28 @@ jobs:
         if: runner.os != 'Windows'
         run: make -C tests/gentype_tests/typescript-react-example clean test
 
-      - name: Build playground compiler
+      - name: Run syntax benchmarks
         if: matrix.benchmarks
-        run: |
-          opam exec -- node packages/playground-bundling/scripts/generate_cmijs.js
-          opam exec -- dune build --profile browser
-          cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js playground/compiler.js
+        run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json
 
+      - name: Download previous benchmark data
+        if: matrix.benchmarks
+        uses: actions/cache@v4
+        with:
+          path: ./tests/benchmark-cache
+          key: syntax-benchmark-v1
+ 
+      - name: Store benchmark result
+        if: matrix.benchmarks
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: tests/benchmark-output.json
+          external-data-json-path: ./tests/benchmark-cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: "150%"
+          comment-always: true
+          comment-on-alert: true
 
       - name: Build playground compiler
         if: matrix.build_playground

From 13163ed998b9b07767be59aeaceaf411cf90b0c3 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 15:23:59 +0200
Subject: [PATCH 07/13] Permissions

---
 .github/workflows/ci.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 41ad7d1ef4..d8a104e675 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,6 +11,10 @@ on:
   pull_request:
     branches: [master, 11.0_release]
 
+permissions:
+  # allow posting comments to pull request
+  pull-requests: write
+  
 concurrency:
   group: ci-${{ github.ref }}-1
   # Cancel previous builds for pull requests only.

From beb38dc7f31f6d5de8b90096fdd337467b93f6f1 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sat, 12 Oct 2024 15:44:55 +0200
Subject: [PATCH 08/13] Beautify

---
 .github/workflows/ci.yml             | 3 ++-
 tests/syntax_benchmarks/Benchmark.ml | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d8a104e675..179455401a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -340,7 +340,8 @@ jobs:
         if: matrix.benchmarks
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          tool: "customSmallerIsBetter"
+          name: Syntax Benchmarks
+          tool: customSmallerIsBetter
           output-file-path: tests/benchmark-output.json
           external-data-json-path: ./tests/benchmark-cache/benchmark-data.json
           github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 6afcc83eb4..7d33194541 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -101,9 +101,10 @@ end = struct
     [
       `Assoc
         [
-          ("name", `String (Format.sprintf "%s - avg. time" b.name));
+          ( "name",
+            `String (Format.sprintf "%s - time (%d iterations)" b.name b.n) );
           ("unit", `String "ms");
-          ("value", `Float (Time.print b.duration /. float_of_int b.n));
+          ("value", `Float (Time.print b.duration));
         ];
       `Assoc
         [

From d12a9fa13566bb5a07fec0c92e2f4693d3c09cb2 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sun, 13 Oct 2024 09:39:57 +0200
Subject: [PATCH 09/13] Cleanup and fix benchmark

---
 tests/syntax_benchmarks/Benchmark.ml | 55 +++++++++-------------------
 1 file changed, 18 insertions(+), 37 deletions(-)

diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 7d33194541..8a0e448c72 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -78,45 +78,33 @@ module Benchmark : sig
   type t
 
   val make : name:string -> f:(t -> unit) -> unit -> t
-  val launch : t -> unit
+  val launch : t -> num_iterations:int -> unit
   val report : t -> Yojson.t list
 end = struct
   type t = {
     name: string;
     mutable start: Time.t;
-    mutable n: int; (* current iterations count *)
-    mutable duration: Time.t;
+    mutable n: int; (* current iteration count *)
+    mutable total_duration: Time.t;
     bench_func: t -> unit;
     mutable timer_on: bool;
-    (* mutable result: benchmarkResult; *)
-    (* The initial states *)
     mutable start_allocs: float;
-    mutable start_bytes: float;
-    (* The net total of this test after being run. *)
-    mutable net_allocs: float;
-    mutable net_bytes: float;
+    mutable total_allocs: float;
   }
 
   let report b =
     [
       `Assoc
         [
-          ( "name",
-            `String (Format.sprintf "%s - time (%d iterations)" b.name b.n) );
+          ("name", `String (Format.sprintf "%s - time/run" b.name));
           ("unit", `String "ms");
-          ("value", `Float (Time.print b.duration));
+          ("value", `Float (Time.print b.total_duration /. float_of_int b.n));
         ];
       `Assoc
         [
-          ("name", `String (Format.sprintf "%s - allocations" b.name));
-          ("unit", `String "");
-          ("value", `Int (int_of_float (b.net_allocs /. float_of_int b.n)));
-        ];
-      `Assoc
-        [
-          ("name", `String (Format.sprintf "%s - bytes allocated" b.name));
-          ("unit", `String "");
-          ("value", `Int (int_of_float (b.net_bytes /. float_of_int b.n)));
+          ("name", `String (Format.sprintf "%s - allocs/run" b.name));
+          ("unit", `String "words");
+          ("value", `Int (int_of_float (b.total_allocs /. float_of_int b.n)));
         ];
     ]
 
@@ -126,12 +114,10 @@ end = struct
       start = Time.zero;
       n = 0;
       bench_func = f;
-      duration = Time.zero;
+      total_duration = Time.zero;
       timer_on = false;
       start_allocs = 0.;
-      start_bytes = 0.;
-      net_allocs = 0.;
-      net_bytes = 0.;
+      total_allocs = 0.;
     }
 
   (* total amount of memory allocated by the program since it started in words *)
@@ -143,7 +129,6 @@ end = struct
     if not b.timer_on then (
       let allocated_words = mallocs () in
       b.start_allocs <- allocated_words;
-      b.start_bytes <- allocated_words *. 8.;
       b.start <- Time.now ();
       b.timer_on <- true)
 
@@ -151,19 +136,15 @@ end = struct
     if b.timer_on then (
       let allocated_words = mallocs () in
       let diff = Time.diff b.start (Time.now ()) in
-      b.duration <- Time.add b.duration diff;
-      b.net_allocs <- b.net_allocs +. (allocated_words -. b.start_allocs);
-      b.net_bytes <- b.net_bytes +. ((allocated_words *. 8.) -. b.start_bytes);
+      b.total_duration <- Time.add b.total_duration diff;
+      b.total_allocs <- b.total_allocs +. (allocated_words -. b.start_allocs);
       b.timer_on <- false)
 
   let reset_timer b =
     if b.timer_on then (
       let allocated_words = mallocs () in
       b.start_allocs <- allocated_words;
-      b.net_allocs <- allocated_words *. 8.;
-      b.start <- Time.now ());
-    b.net_allocs <- 0.;
-    b.net_bytes <- 0.
+      b.start <- Time.now ())
 
   let run_iteration b n =
     Gc.full_major ();
@@ -173,9 +154,8 @@ end = struct
     b.bench_func b;
     stop_timer b
 
-  let launch b =
-    (* 150 runs * all the benchmarks means around 1m of benchmark time *)
-    for n = 1 to 150 do
+  let launch b ~num_iterations =
+    for n = 1 to num_iterations do
       run_iteration b n
     done
 end
@@ -197,6 +177,7 @@ end = struct
     structure
 
   let data_dir = "tests/syntax_benchmarks/data"
+  let num_iterations = 150
 
   let benchmark (filename, action) =
     let path = Filename.concat data_dir filename in
@@ -222,7 +203,7 @@ end = struct
           ()
     in
     let b = Benchmark.make ~name ~f:benchmark_fn () in
-    Benchmark.launch b;
+    Benchmark.launch b ~num_iterations;
     Benchmark.report b
 
   let specs =

From 379b96adf34fa487c4e96d845d05d72aaf558d1a Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sun, 13 Oct 2024 10:04:49 +0200
Subject: [PATCH 10/13] More cleanup

---
 tests/syntax_benchmarks/Benchmark.ml | 68 ++++++++++++++--------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml
index 8a0e448c72..e947612513 100644
--- a/tests/syntax_benchmarks/Benchmark.ml
+++ b/tests/syntax_benchmarks/Benchmark.ml
@@ -75,42 +75,24 @@ end = struct
 end
 
 module Benchmark : sig
-  type t
+  type test_result = {ms_per_run: float; allocs_per_run: int}
 
-  val make : name:string -> f:(t -> unit) -> unit -> t
-  val launch : t -> num_iterations:int -> unit
-  val report : t -> Yojson.t list
+  val run : (unit -> unit) -> num_iterations:int -> test_result
 end = struct
   type t = {
-    name: string;
     mutable start: Time.t;
     mutable n: int; (* current iteration count *)
     mutable total_duration: Time.t;
-    bench_func: t -> unit;
+    bench_func: unit -> unit;
     mutable timer_on: bool;
     mutable start_allocs: float;
     mutable total_allocs: float;
   }
 
-  let report b =
-    [
-      `Assoc
-        [
-          ("name", `String (Format.sprintf "%s - time/run" b.name));
-          ("unit", `String "ms");
-          ("value", `Float (Time.print b.total_duration /. float_of_int b.n));
-        ];
-      `Assoc
-        [
-          ("name", `String (Format.sprintf "%s - allocs/run" b.name));
-          ("unit", `String "words");
-          ("value", `Int (int_of_float (b.total_allocs /. float_of_int b.n)));
-        ];
-    ]
+  type test_result = {ms_per_run: float; allocs_per_run: int}
 
-  let make ~name ~f () =
+  let make f =
     {
-      name;
       start = Time.zero;
       n = 0;
       bench_func = f;
@@ -151,13 +133,18 @@ end = struct
     b.n <- n;
     reset_timer b;
     start_timer b;
-    b.bench_func b;
+    b.bench_func ();
     stop_timer b
 
-  let launch b ~num_iterations =
+  let run f ~num_iterations =
+    let b = make f in
     for n = 1 to num_iterations do
       run_iteration b n
-    done
+    done;
+    {
+      ms_per_run = Time.print b.total_duration /. float_of_int b.n;
+      allocs_per_run = int_of_float (b.total_allocs /. float_of_int b.n);
+    }
 end
 
 module Benchmarks : sig
@@ -182,17 +169,16 @@ end = struct
   let benchmark (filename, action) =
     let path = Filename.concat data_dir filename in
     let src = IO.read_file path in
-    let name = string_of_action action ^ " " ^ filename in
     let benchmark_fn =
       match action with
       | Parse ->
-        fun _ ->
+        fun () ->
           let _ = Sys.opaque_identity (parse_rescript src path) in
           ()
       | Print ->
         let p = Parser.make src path in
         let ast = ResParser.parse_implementation p in
-        fun _ ->
+        fun () ->
           let _ =
             Sys.opaque_identity
               (let cmt_tbl = CommentTable.make () in
@@ -202,9 +188,7 @@ end = struct
           in
           ()
     in
-    let b = Benchmark.make ~name ~f:benchmark_fn () in
-    Benchmark.launch b ~num_iterations;
-    Benchmark.report b
+    Benchmark.run benchmark_fn ~num_iterations
 
   let specs =
     [
@@ -219,7 +203,25 @@ end = struct
 
   let run () =
     List.to_seq specs
-    |> Seq.flat_map (fun spec -> benchmark spec |> List.to_seq)
+    |> Seq.flat_map (fun spec ->
+           let filename, action = spec in
+           let test_name = string_of_action action ^ " " ^ filename in
+           let {Benchmark.ms_per_run; allocs_per_run} = benchmark spec in
+           [
+             `Assoc
+               [
+                 ("name", `String (Format.sprintf "%s - time/run" test_name));
+                 ("unit", `String "ms");
+                 ("value", `Float ms_per_run);
+               ];
+             `Assoc
+               [
+                 ("name", `String (Format.sprintf "%s - allocs/run" test_name));
+                 ("unit", `String "words");
+                 ("value", `Int allocs_per_run);
+               ];
+           ]
+           |> List.to_seq)
     |> Seq.iteri (fun i json ->
            print_endline (if i == 0 then "[" else ",");
            print_string (Yojson.to_string json));

From 7c62a9f6d2a34a0aa76400c752e2bbd7df9c7a41 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sun, 13 Oct 2024 10:14:10 +0200
Subject: [PATCH 11/13] Do not attempt to run action for PRs created from other
 repos

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 179455401a..94d374c334 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -337,7 +337,8 @@ jobs:
           key: syntax-benchmark-v1
  
       - name: Store benchmark result
-        if: matrix.benchmarks
+        # Do not run for PRs created from other repos as those won't be able to write to the pull request
+        if: ${{ matrix.benchmarks && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.repository.full_name) }}
         uses: benchmark-action/github-action-benchmark@v1
         with:
           name: Syntax Benchmarks

From 711d0332e2364f6bcbfc0eca6a231c3be30d9452 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sun, 13 Oct 2024 10:26:12 +0200
Subject: [PATCH 12/13] Fix caml_mach_absolute_time for Linux

---
 tests/syntax_benchmarks/time.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/syntax_benchmarks/time.c b/tests/syntax_benchmarks/time.c
index df29af7bf2..3b85f45214 100644
--- a/tests/syntax_benchmarks/time.c
+++ b/tests/syntax_benchmarks/time.c
@@ -37,7 +37,7 @@ CAMLprim value caml_mach_absolute_time(value unit) {
 #elif defined(__linux__)
   struct timespec now;
   clock_gettime(CLOCK_MONOTONIC, &now);
-  result = now.tv_sec * 1000 + now.tv_nsec / 1000000;
+  result = now.tv_sec * 1000000000 + now.tv_nsec;
 #endif
 
   return caml_copy_int64(result);

From 51206f2a16454bd24df511bc45ed8508b8fcb5f1 Mon Sep 17 00:00:00 2001
From: Christoph Knittel <ck@cca.io>
Date: Sun, 13 Oct 2024 10:39:28 +0200
Subject: [PATCH 13/13] Add job summary

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 94d374c334..b102a453c7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -349,6 +349,7 @@ jobs:
           alert-threshold: "150%"
           comment-always: true
           comment-on-alert: true
+          summary-always: true
 
       - name: Build playground compiler
         if: matrix.build_playground