diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a19f176af3..b102a453c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,10 @@ on: pull_request: branches: [master, 11.0_release] +permissions: + # allow posting comments to pull request + pull-requests: write + concurrency: group: ci-${{ github.ref }}-1 # Cancel previous builds for pull requests only. @@ -90,8 +94,9 @@ jobs: ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true upload_libs: true - # Build the playground compiler on the fastest runner + # Build the playground compiler and run the benchmarks on the fastest runner build_playground: true + benchmarks: true - os: buildjet-2vcpu-ubuntu-2204-arm # ARM ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static upload_binaries: true @@ -150,7 +155,7 @@ jobs: # matrix.ocaml_compiler may contain commas - name: Get OPAM cache key shell: bash - run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV + run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV - name: Restore OPAM environment id: cache-opam-env @@ -320,6 +325,32 @@ jobs: if: runner.os != 'Windows' run: make -C tests/gentype_tests/typescript-react-example clean test + - name: Run syntax benchmarks + if: matrix.benchmarks + run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json + + - name: Download previous benchmark data + if: matrix.benchmarks + uses: actions/cache@v4 + with: + path: ./tests/benchmark-cache + key: syntax-benchmark-v1 + + - name: Store benchmark result + # Do not run for PRs created from other repos as those won't be able to write to the pull request + if: ${{ matrix.benchmarks && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.repository.full_name) }} + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Syntax Benchmarks + tool: customSmallerIsBetter + output-file-path: tests/benchmark-output.json + external-data-json-path: ./tests/benchmark-cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: "150%" + comment-always: true + comment-on-alert: true + summary-always: true + - name: Build playground compiler if: matrix.build_playground run: | diff --git a/dune-project b/dune-project index d0d3bdca87..2fa8675931 100644 --- a/dune-project +++ b/dune-project @@ -24,6 +24,10 @@ (and :with-test (= 0.26.2))) + (yojson + (and + :with-test + (= 2.2.2))) (ocaml-lsp-server (and :with-dev-setup diff --git a/rescript.opam b/rescript.opam index ff2913835d..bc35c92939 100644 --- a/rescript.opam +++ b/rescript.opam @@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues" depends: [ "ocaml" {>= "4.10"} "ocamlformat" {with-test & = "0.26.2"} + "yojson" {with-test & = "2.2.2"} "ocaml-lsp-server" {with-dev-setup & = "1.19.0"} "cppo" {= "1.6.9"} "js_of_ocaml" {= "5.8.1"} diff --git a/tests/syntax_benchmarks/Benchmark.ml b/tests/syntax_benchmarks/Benchmark.ml index 4f8e2db38a..e947612513 100644 --- a/tests/syntax_benchmarks/Benchmark.ml +++ b/tests/syntax_benchmarks/Benchmark.ml @@ -75,59 +75,31 @@ end = struct end module Benchmark : sig - type t + type test_result = {ms_per_run: float; allocs_per_run: int} - val make : name:string -> f:(t -> unit) -> unit -> t - val launch : t -> unit - val report : t -> unit + val run : (unit -> unit) -> num_iterations:int -> test_result end = struct type t = { - name: string; mutable start: Time.t; - mutable n: int; (* current iterations count *) - mutable duration: Time.t; - bench_func: t -> unit; + mutable n: int; (* current iteration count *) + mutable total_duration: Time.t; + bench_func: unit -> unit; mutable timer_on: bool; - (* mutable result: benchmarkResult; *) - (* The initial states *) mutable start_allocs: float; - mutable start_bytes: float; - (* The net total of this test after being run. *) - mutable net_allocs: float; - mutable net_bytes: float; + mutable total_allocs: float; } - let report b = - print_endline (Format.sprintf "Benchmark: %s" b.name); - print_endline (Format.sprintf "Nbr of iterations: %d" b.n); - print_endline - (Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration)); - print_endline - (Format.sprintf "Avg time/op: %fms" - (Time.print b.duration /. float_of_int b.n)); - print_endline - (Format.sprintf "Allocs/op: %d" - (int_of_float (b.net_allocs /. float_of_int b.n))); - print_endline - (Format.sprintf "B/op: %d" - (int_of_float (b.net_bytes /. float_of_int b.n))); - - (* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *) - print_newline (); - () + type test_result = {ms_per_run: float; allocs_per_run: int} - let make ~name ~f () = + let make f = { - name; start = Time.zero; n = 0; bench_func = f; - duration = Time.zero; + total_duration = Time.zero; timer_on = false; start_allocs = 0.; - start_bytes = 0.; - net_allocs = 0.; - net_bytes = 0.; + total_allocs = 0.; } (* total amount of memory allocated by the program since it started in words *) @@ -139,7 +111,6 @@ end = struct if not b.timer_on then ( let allocated_words = mallocs () in b.start_allocs <- allocated_words; - b.start_bytes <- allocated_words *. 8.; b.start <- Time.now (); b.timer_on <- true) @@ -147,49 +118,44 @@ end = struct if b.timer_on then ( let allocated_words = mallocs () in let diff = Time.diff b.start (Time.now ()) in - b.duration <- Time.add b.duration diff; - b.net_allocs <- b.net_allocs +. (allocated_words -. b.start_allocs); - b.net_bytes <- b.net_bytes +. ((allocated_words *. 8.) -. b.start_bytes); + b.total_duration <- Time.add b.total_duration diff; + b.total_allocs <- b.total_allocs +. (allocated_words -. b.start_allocs); b.timer_on <- false) let reset_timer b = if b.timer_on then ( let allocated_words = mallocs () in b.start_allocs <- allocated_words; - b.net_allocs <- allocated_words *. 8.; - b.start <- Time.now ()); - b.net_allocs <- 0.; - b.net_bytes <- 0. + b.start <- Time.now ()) let run_iteration b n = Gc.full_major (); b.n <- n; reset_timer b; start_timer b; - b.bench_func b; + b.bench_func (); stop_timer b - let launch b = - (* 150 runs * all the benchmarks means around 1m of benchmark time *) - for n = 1 to 150 do + let run f ~num_iterations = + let b = make f in + for n = 1 to num_iterations do run_iteration b n - done + done; + { + ms_per_run = Time.print b.total_duration /. float_of_int b.n; + allocs_per_run = int_of_float (b.total_allocs /. float_of_int b.n); + } end module Benchmarks : sig val run : unit -> unit end = struct type action = Parse | Print + let string_of_action action = match action with - | Parse -> "parser" - | Print -> "printer" - - (* TODO: we could at Reason here *) - type lang = Rescript - let string_of_lang lang = - match lang with - | Rescript -> "rescript" + | Parse -> "Parse" + | Print -> "Print" let parse_rescript src filename = let p = Parser.make src filename in @@ -197,21 +163,22 @@ end = struct assert (p.diagnostics == []); structure - let benchmark filename lang action = - let src = IO.read_file filename in - let name = - filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action - in + let data_dir = "tests/syntax_benchmarks/data" + let num_iterations = 150 + + let benchmark (filename, action) = + let path = Filename.concat data_dir filename in + let src = IO.read_file path in let benchmark_fn = - match (lang, action) with - | Rescript, Parse -> - fun _ -> - let _ = Sys.opaque_identity (parse_rescript src filename) in + match action with + | Parse -> + fun () -> + let _ = Sys.opaque_identity (parse_rescript src path) in () - | Rescript, Print -> - let p = Parser.make src filename in + | Print -> + let p = Parser.make src path in let ast = ResParser.parse_implementation p in - fun _ -> + fun () -> let _ = Sys.opaque_identity (let cmt_tbl = CommentTable.make () in @@ -221,21 +188,45 @@ end = struct in () in - let b = Benchmark.make ~name ~f:benchmark_fn () in - Benchmark.launch b; - Benchmark.report b + Benchmark.run benchmark_fn ~num_iterations + + let specs = + [ + ("RedBlackTree.res", Parse); + ("RedBlackTree.res", Print); + ("RedBlackTreeNoComments.res", Print); + ("Napkinscript.res", Parse); + ("Napkinscript.res", Print); + ("HeroGraphic.res", Parse); + ("HeroGraphic.res", Print); + ] let run () = - let data_dir = "tests/syntax_benchmarks/data" in - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse; - benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print; - benchmark - (Filename.concat data_dir "RedBlackTreeNoComments.res") - Rescript Print; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse; - benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse; - benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print + List.to_seq specs + |> Seq.flat_map (fun spec -> + let filename, action = spec in + let test_name = string_of_action action ^ " " ^ filename in + let {Benchmark.ms_per_run; allocs_per_run} = benchmark spec in + [ + `Assoc + [ + ("name", `String (Format.sprintf "%s - time/run" test_name)); + ("unit", `String "ms"); + ("value", `Float ms_per_run); + ]; + `Assoc + [ + ("name", `String (Format.sprintf "%s - allocs/run" test_name)); + ("unit", `String "words"); + ("value", `Int allocs_per_run); + ]; + ] + |> List.to_seq) + |> Seq.iteri (fun i json -> + print_endline (if i == 0 then "[" else ","); + print_string (Yojson.to_string json)); + print_newline (); + print_endline "]" end let () = Benchmarks.run () diff --git a/tests/syntax_benchmarks/dune b/tests/syntax_benchmarks/dune index 0d234e3126..d45805d291 100644 --- a/tests/syntax_benchmarks/dune +++ b/tests/syntax_benchmarks/dune @@ -9,6 +9,7 @@ (enabled_if (and (<> %{profile} browser) + (>= %{ocaml_version} "4.14.0") (or (= %{system} macosx) ; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613) @@ -22,6 +23,6 @@ (foreign_stubs (language c) (names time)) - (libraries syntax)) + (libraries syntax yojson)) (data_only_dirs data) diff --git a/tests/syntax_benchmarks/time.c b/tests/syntax_benchmarks/time.c index df29af7bf2..3b85f45214 100644 --- a/tests/syntax_benchmarks/time.c +++ b/tests/syntax_benchmarks/time.c @@ -37,7 +37,7 @@ CAMLprim value caml_mach_absolute_time(value unit) { #elif defined(__linux__) struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); - result = now.tv_sec * 1000 + now.tv_nsec / 1000000; + result = now.tv_sec * 1000000000 + now.tv_nsec; #endif return caml_copy_int64(result);