From 78ef0f9878a5d284ef239c79dbc57113ce9cedfe Mon Sep 17 00:00:00 2001 From: Noah Betzen Date: Thu, 15 May 2025 09:30:42 -0700 Subject: [PATCH 1/3] Add mix test --dry-run flag --- lib/mix/lib/mix/tasks/test.ex | 157 ++++++++++++++++++--------- lib/mix/test/mix/tasks/test_test.exs | 34 ++++++ 2 files changed, 139 insertions(+), 52 deletions(-) diff --git a/lib/mix/lib/mix/tasks/test.ex b/lib/mix/lib/mix/tasks/test.ex index ad32028f915..d03490223c4 100644 --- a/lib/mix/lib/mix/tasks/test.ex +++ b/lib/mix/lib/mix/tasks/test.ex @@ -121,6 +121,10 @@ defmodule Mix.Tasks.Test do * `--cover` - runs coverage tool. See "Coverage" section below + * `--dry-run` - prints which tests would be run based on current options, + but does not actually run any tests. This combines with all other options + like `--stale`, `--only`, `--exclude`, etc. + * `--exclude` - excludes tests that match the filter. This option may be given several times to apply different filters, such as `--exclude ci --exclude slow` @@ -494,7 +498,8 @@ defmodule Mix.Tasks.Test do warnings_as_errors: :boolean, profile_require: :string, exit_status: :integer, - repeat_until_failure: :integer + repeat_until_failure: :integer, + dry_run: :boolean ] @cover [output: "cover", tool: Mix.Tasks.Test.Coverage] @@ -623,6 +628,7 @@ defmodule Mix.Tasks.Test do warnings_as_errors? = Keyword.get(opts, :warnings_as_errors, false) exit_status = Keyword.fetch!(ex_unit_opts, :exit_status) + dry_run? = Keyword.get(opts, :dry_run, false) # Prepare and extract all files to require and run test_paths = project[:test_paths] || default_test_paths() @@ -659,69 +665,115 @@ defmodule Mix.Tasks.Test do warn_files != [] && warn_misnamed_test_files(warn_files) - try do - Enum.each(test_paths, &require_test_helper(shell, &1)) - # test_opts always wins because those are given via args - ExUnit.configure(ex_unit_opts |> merge_helper_opts() |> Keyword.merge(test_opts)) - CT.require_and_run(matched_test_files, test_paths, test_elixirc_options, opts) - catch - kind, reason -> - # Also mark the whole suite as failed - file = Keyword.fetch!(opts, :failures_manifest_path) - ExUnit.Filters.fail_all!(file) - :erlang.raise(kind, reason, __STACKTRACE__) + if dry_run? do + do_dry_run(matched_test_files) + else + do_wet_run( + shell, + cover, + test_paths, + files, + matched_test_files, + test_opts, + test_elixirc_options, + ex_unit_opts, + warnings_as_errors?, + exit_status, + opts + ) + end + end + + defp do_dry_run(matched_test_files) do + if matched_test_files == [] do + Mix.shell().info(""" + -- DRY RUN -- + No tests would run + """) else - {:ok, %{excluded: excluded, failures: failures, warnings?: warnings?, total: total}} -> - Mix.shell(shell) - cover && cover.() + Mix.shell().info(""" + -- DRY RUN -- + The following test files would be run: + + #{Enum.join(matched_test_files, "\n")} + """) + end + end - cond do - warnings_as_errors? and warnings? and failures == 0 -> - message = - "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option" + defp do_wet_run( + shell, + cover, + test_paths, + files, + matched_test_files, + test_opts, + test_elixirc_options, + ex_unit_opts, + warnings_as_errors?, + exit_status, + opts + ) do + Enum.each(test_paths, &require_test_helper(shell, &1)) + # test_opts always wins because those are given via args + ExUnit.configure(ex_unit_opts |> merge_helper_opts() |> Keyword.merge(test_opts)) + CT.require_and_run(matched_test_files, test_paths, test_elixirc_options, opts) + catch + kind, reason -> + # Also mark the whole suite as failed + file = Keyword.fetch!(opts, :failures_manifest_path) + ExUnit.Filters.fail_all!(file) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:ok, %{excluded: excluded, failures: failures, warnings?: warnings?, total: total}} -> + Mix.shell(shell) + cover && cover.() - IO.puts(:stderr, IO.ANSI.format([:red, message])) + cond do + warnings_as_errors? and warnings? and failures == 0 -> + message = + "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option" + + IO.puts(:stderr, IO.ANSI.format([:red, message])) - System.at_exit(fn _ -> - exit({:shutdown, 1}) - end) + System.at_exit(fn _ -> + exit({:shutdown, 1}) + end) - failures > 0 and opts[:raise] -> - raise_with_shell(shell, "\"mix test\" failed") + failures > 0 and opts[:raise] -> + raise_with_shell(shell, "\"mix test\" failed") - warnings_as_errors? and warnings? and failures > 0 -> - System.at_exit(fn _ -> - exit({:shutdown, exit_status + 1}) - end) + warnings_as_errors? and warnings? and failures > 0 -> + System.at_exit(fn _ -> + exit({:shutdown, exit_status + 1}) + end) - failures > 0 -> - System.at_exit(fn _ -> - exit({:shutdown, exit_status}) - end) + failures > 0 -> + System.at_exit(fn _ -> + exit({:shutdown, exit_status}) + end) - excluded == total and Keyword.has_key?(opts, :only) -> - message = "The --only option was given to \"mix test\" but no test was executed" - raise_or_error_at_exit(shell, message, opts) + excluded == total and Keyword.has_key?(opts, :only) -> + message = "The --only option was given to \"mix test\" but no test was executed" + raise_or_error_at_exit(shell, message, opts) - true -> - :ok - end + true -> + :ok + end - :noop -> - cond do - opts[:stale] -> - Mix.shell().info("No stale tests") + :noop -> + cond do + opts[:stale] -> + Mix.shell().info("No stale tests") - opts[:failed] || files == [] -> - Mix.shell().info("There are no tests to run") + opts[:failed] || files == [] -> + Mix.shell().info("There are no tests to run") - true -> - message = "Paths given to \"mix test\" did not match any directory/file: " - raise_or_error_at_exit(shell, message <> Enum.join(files, ", "), opts) - end + true -> + message = "Paths given to \"mix test\" did not match any directory/file: " + raise_or_error_at_exit(shell, message <> Enum.join(files, ", "), opts) + end - :ok - end + :ok end # similar to Mix.Utils.extract_files/2, but returns a list of directly included test files, @@ -847,7 +899,8 @@ defmodule Mix.Tasks.Test do :only_test_ids, :test_location_relative_path, :exit_status, - :repeat_until_failure + :repeat_until_failure, + :dry_run ] @doc false diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs index f5d792ce473..20a50f78933 100644 --- a/lib/mix/test/mix/tasks/test_test.exs +++ b/lib/mix/test/mix/tasks/test_test.exs @@ -716,6 +716,40 @@ defmodule Mix.Tasks.TestTest do end end + describe "--dry-run" do + test "prints which tests would run without executing them" do + in_fixture("test_stale", fn -> + File.write!("test/dry_run_test.exs", """ + defmodule DryRunTest do + use ExUnit.Case + + test "passing test" do + assert true + end + + test "failing test" do + assert false + end + end + """) + + assert {output, 0} = mix_code(["test", "--dry-run", "--stale"]) + assert output =~ "DRY RUN" + assert output =~ "test/dry_run_test.exs" + refute output =~ "Finished in" + end) + end + + test "prints message when no tests would run" do + in_fixture("test_stale", fn -> + assert {output, 0} = mix_code(["test", "--dry-run", "non_existent_test.exs"]) + assert output =~ "DRY RUN" + assert output =~ "No tests would run" + refute output =~ "Finished in" + end) + end + end + defp receive_until_match(port, expected, acc) do receive do {^port, {:data, output}} -> From a2d19b21c5d5526ab65d7b9567a99295ff0890cb Mon Sep 17 00:00:00 2001 From: Noah Betzen Date: Fri, 16 May 2025 09:59:16 -0700 Subject: [PATCH 2/3] Move dry_run check into ExUnit.Runner --- lib/ex_unit/lib/ex_unit/runner.ex | 30 ++++-- lib/mix/lib/mix/tasks/test.ex | 151 +++++++++------------------ lib/mix/test/mix/tasks/test_test.exs | 2 + 3 files changed, 75 insertions(+), 108 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/runner.ex b/lib/ex_unit/lib/ex_unit/runner.ex index ff4c3b218fc..ff0cdc42c7b 100644 --- a/lib/ex_unit/lib/ex_unit/runner.ex +++ b/lib/ex_unit/lib/ex_unit/runner.ex @@ -44,12 +44,27 @@ defmodule ExUnit.Runner do config = configure(opts, manager, self(), stats_pid) :erlang.system_flag(:backtrace_depth, Keyword.fetch!(opts, :stacktrace_depth)) - start_time = System.monotonic_time() - EM.suite_started(config.manager, opts) - modules_to_restore = if Keyword.fetch!(opts, :repeat_until_failure) > 0, do: {[], []}, else: nil + modules_to_restore = + if not opts[:dry_run] do + do_run(config, modules_to_restore, opts, load_us) + else + nil + end + + stats = ExUnit.RunnerStats.stats(stats_pid) + EM.stop(config.manager) + after_suite_callbacks = Application.fetch_env!(:ex_unit, :after_suite) + Enum.each(after_suite_callbacks, fn callback -> callback.(stats) end) + {stats, modules_to_restore} + end + + defp do_run(config, modules_to_restore, opts, load_us) do + start_time = System.monotonic_time() + EM.suite_started(config.manager, opts) + {async_stop_time, modules_to_restore} = async_loop(config, %{}, false, modules_to_restore) stop_time = System.monotonic_time() @@ -65,11 +80,7 @@ defmodule ExUnit.Runner do times_us = %{async: async_us, load: load_us, run: run_us} EM.suite_finished(config.manager, times_us) - stats = ExUnit.RunnerStats.stats(stats_pid) - EM.stop(config.manager) - after_suite_callbacks = Application.fetch_env!(:ex_unit, :after_suite) - Enum.each(after_suite_callbacks, fn callback -> callback.(stats) end) - {stats, modules_to_restore} + modules_to_restore end defp configure(opts, manager, runner_pid, stats_pid) do @@ -88,7 +99,8 @@ defmodule ExUnit.Runner do seed: opts[:seed], stats_pid: stats_pid, timeout: opts[:timeout], - trace: opts[:trace] + trace: opts[:trace], + dry_run: opts[:dry_run] } end diff --git a/lib/mix/lib/mix/tasks/test.ex b/lib/mix/lib/mix/tasks/test.ex index d03490223c4..f6eb1a44d0e 100644 --- a/lib/mix/lib/mix/tasks/test.ex +++ b/lib/mix/lib/mix/tasks/test.ex @@ -121,9 +121,9 @@ defmodule Mix.Tasks.Test do * `--cover` - runs coverage tool. See "Coverage" section below - * `--dry-run` - prints which tests would be run based on current options, + * `--dry-run` *(since v1.19.0)* - prints which tests would be run based on current options, but does not actually run any tests. This combines with all other options - like `--stale`, `--only`, `--exclude`, etc. + like `--stale`, `--only`, `--exclude`, and so on. * `--exclude` - excludes tests that match the filter. This option may be given several times to apply different filters, such as `--exclude ci --exclude slow` @@ -628,7 +628,6 @@ defmodule Mix.Tasks.Test do warnings_as_errors? = Keyword.get(opts, :warnings_as_errors, false) exit_status = Keyword.fetch!(ex_unit_opts, :exit_status) - dry_run? = Keyword.get(opts, :dry_run, false) # Prepare and extract all files to require and run test_paths = project[:test_paths] || default_test_paths() @@ -665,115 +664,69 @@ defmodule Mix.Tasks.Test do warn_files != [] && warn_misnamed_test_files(warn_files) - if dry_run? do - do_dry_run(matched_test_files) + try do + Enum.each(test_paths, &require_test_helper(shell, &1)) + # test_opts always wins because those are given via args + ExUnit.configure(ex_unit_opts |> merge_helper_opts() |> Keyword.merge(test_opts)) + CT.require_and_run(matched_test_files, test_paths, test_elixirc_options, opts) + catch + kind, reason -> + # Also mark the whole suite as failed + file = Keyword.fetch!(opts, :failures_manifest_path) + ExUnit.Filters.fail_all!(file) + :erlang.raise(kind, reason, __STACKTRACE__) else - do_wet_run( - shell, - cover, - test_paths, - files, - matched_test_files, - test_opts, - test_elixirc_options, - ex_unit_opts, - warnings_as_errors?, - exit_status, - opts - ) - end - end + {:ok, %{excluded: excluded, failures: failures, warnings?: warnings?, total: total}} -> + Mix.shell(shell) + cover && cover.() - defp do_dry_run(matched_test_files) do - if matched_test_files == [] do - Mix.shell().info(""" - -- DRY RUN -- - No tests would run - """) - else - Mix.shell().info(""" - -- DRY RUN -- - The following test files would be run: + cond do + warnings_as_errors? and warnings? and failures == 0 -> + message = + "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option" - #{Enum.join(matched_test_files, "\n")} - """) - end - end + IO.puts(:stderr, IO.ANSI.format([:red, message])) - defp do_wet_run( - shell, - cover, - test_paths, - files, - matched_test_files, - test_opts, - test_elixirc_options, - ex_unit_opts, - warnings_as_errors?, - exit_status, - opts - ) do - Enum.each(test_paths, &require_test_helper(shell, &1)) - # test_opts always wins because those are given via args - ExUnit.configure(ex_unit_opts |> merge_helper_opts() |> Keyword.merge(test_opts)) - CT.require_and_run(matched_test_files, test_paths, test_elixirc_options, opts) - catch - kind, reason -> - # Also mark the whole suite as failed - file = Keyword.fetch!(opts, :failures_manifest_path) - ExUnit.Filters.fail_all!(file) - :erlang.raise(kind, reason, __STACKTRACE__) - else - {:ok, %{excluded: excluded, failures: failures, warnings?: warnings?, total: total}} -> - Mix.shell(shell) - cover && cover.() + System.at_exit(fn _ -> + exit({:shutdown, 1}) + end) - cond do - warnings_as_errors? and warnings? and failures == 0 -> - message = - "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option" - - IO.puts(:stderr, IO.ANSI.format([:red, message])) - - System.at_exit(fn _ -> - exit({:shutdown, 1}) - end) + failures > 0 and opts[:raise] -> + raise_with_shell(shell, "\"mix test\" failed") - failures > 0 and opts[:raise] -> - raise_with_shell(shell, "\"mix test\" failed") + warnings_as_errors? and warnings? and failures > 0 -> + System.at_exit(fn _ -> + exit({:shutdown, exit_status + 1}) + end) - warnings_as_errors? and warnings? and failures > 0 -> - System.at_exit(fn _ -> - exit({:shutdown, exit_status + 1}) - end) + failures > 0 -> + System.at_exit(fn _ -> + exit({:shutdown, exit_status}) + end) - failures > 0 -> - System.at_exit(fn _ -> - exit({:shutdown, exit_status}) - end) + excluded == total and Keyword.has_key?(opts, :only) -> + message = "The --only option was given to \"mix test\" but no test was executed" + raise_or_error_at_exit(shell, message, opts) - excluded == total and Keyword.has_key?(opts, :only) -> - message = "The --only option was given to \"mix test\" but no test was executed" - raise_or_error_at_exit(shell, message, opts) + true -> + :ok + end - true -> - :ok - end + :noop -> + cond do + opts[:stale] -> + Mix.shell().info("No stale tests") - :noop -> - cond do - opts[:stale] -> - Mix.shell().info("No stale tests") + opts[:failed] || files == [] -> + Mix.shell().info("There are no tests to run") - opts[:failed] || files == [] -> - Mix.shell().info("There are no tests to run") + true -> + message = "Paths given to \"mix test\" did not match any directory/file: " + raise_or_error_at_exit(shell, message <> Enum.join(files, ", "), opts) + end - true -> - message = "Paths given to \"mix test\" did not match any directory/file: " - raise_or_error_at_exit(shell, message <> Enum.join(files, ", "), opts) - end - - :ok + :ok + end end # similar to Mix.Utils.extract_files/2, but returns a list of directly included test files, diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs index 20a50f78933..f610cc83179 100644 --- a/lib/mix/test/mix/tasks/test_test.exs +++ b/lib/mix/test/mix/tasks/test_test.exs @@ -716,6 +716,8 @@ defmodule Mix.Tasks.TestTest do end end + # TODO: Get to pass after deciding on implementation + @tag :skip describe "--dry-run" do test "prints which tests would run without executing them" do in_fixture("test_stale", fn -> From 56e03ed230b0bfb4b93aaa80bdf9848775bf4b0a Mon Sep 17 00:00:00 2001 From: Noah Betzen Date: Mon, 2 Jun 2025 16:34:01 -0700 Subject: [PATCH 3/3] Finish dry run implementation --- lib/ex_unit/lib/ex_unit/cli_formatter.ex | 18 ++++++++++-- lib/ex_unit/lib/ex_unit/runner.ex | 31 ++++++++------------- lib/mix/test/mix/tasks/test_test.exs | 35 +++++++++++------------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/lib/ex_unit/lib/ex_unit/cli_formatter.ex b/lib/ex_unit/lib/ex_unit/cli_formatter.ex index f4d2982c3d5..06d63c06699 100644 --- a/lib/ex_unit/lib/ex_unit/cli_formatter.ex +++ b/lib/ex_unit/lib/ex_unit/cli_formatter.ex @@ -18,6 +18,7 @@ defmodule ExUnit.CLIFormatter do IO.puts("") config = %{ + dry_run: opts[:dry_run], trace: opts[:trace], colors: colors(opts), width: get_terminal_width(), @@ -154,7 +155,16 @@ defmodule ExUnit.CLIFormatter do {:noreply, config} end - def handle_cast({:module_finished, %ExUnit.TestModule{state: nil}}, config) do + def handle_cast({:module_finished, %ExUnit.TestModule{state: nil} = module}, config) do + if config.dry_run do + IO.puts("Test dry run:") + file_path = Path.relative_to_cwd(module.file) + + Enum.each(module.tests, fn test -> + IO.puts("#{file_path}:#{test.tags.line}") + end) + end + {:noreply, config} end @@ -356,7 +366,11 @@ defmodule ExUnit.CLIFormatter do ) |> if_true( config.excluded_counter > 0, - &(&1 <> " (#{config.excluded_counter} excluded)") + &(&1 <> ", (#{config.excluded_counter} excluded)") + ) + |> if_true( + config.dry_run == true, + &(&1 <> " (dry run)") ) cond do diff --git a/lib/ex_unit/lib/ex_unit/runner.ex b/lib/ex_unit/lib/ex_unit/runner.ex index ff0cdc42c7b..67fc19431db 100644 --- a/lib/ex_unit/lib/ex_unit/runner.ex +++ b/lib/ex_unit/lib/ex_unit/runner.ex @@ -44,27 +44,12 @@ defmodule ExUnit.Runner do config = configure(opts, manager, self(), stats_pid) :erlang.system_flag(:backtrace_depth, Keyword.fetch!(opts, :stacktrace_depth)) - modules_to_restore = - if Keyword.fetch!(opts, :repeat_until_failure) > 0, do: {[], []}, else: nil - - modules_to_restore = - if not opts[:dry_run] do - do_run(config, modules_to_restore, opts, load_us) - else - nil - end - - stats = ExUnit.RunnerStats.stats(stats_pid) - EM.stop(config.manager) - after_suite_callbacks = Application.fetch_env!(:ex_unit, :after_suite) - Enum.each(after_suite_callbacks, fn callback -> callback.(stats) end) - {stats, modules_to_restore} - end - - defp do_run(config, modules_to_restore, opts, load_us) do start_time = System.monotonic_time() EM.suite_started(config.manager, opts) + modules_to_restore = + if Keyword.fetch!(opts, :repeat_until_failure) > 0, do: {[], []}, else: nil + {async_stop_time, modules_to_restore} = async_loop(config, %{}, false, modules_to_restore) stop_time = System.monotonic_time() @@ -80,7 +65,11 @@ defmodule ExUnit.Runner do times_us = %{async: async_us, load: load_us, run: run_us} EM.suite_finished(config.manager, times_us) - modules_to_restore + stats = ExUnit.RunnerStats.stats(stats_pid) + EM.stop(config.manager) + after_suite_callbacks = Application.fetch_env!(:ex_unit, :after_suite) + Enum.each(after_suite_callbacks, fn callback -> callback.(stats) end) + {stats, modules_to_restore} end defp configure(opts, manager, runner_pid, stats_pid) do @@ -318,6 +307,10 @@ defmodule ExUnit.Runner do {test_module, [], []} end + defp run_module_tests(%{dry_run: true}, test_module, _async?, tests) do + {test_module, [], tests} + end + defp run_module_tests(config, test_module, async?, tests) do Process.put(@current_key, test_module) %ExUnit.TestModule{name: module, tags: tags, parameters: params} = test_module diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs index f610cc83179..24037a63d51 100644 --- a/lib/mix/test/mix/tasks/test_test.exs +++ b/lib/mix/test/mix/tasks/test_test.exs @@ -716,38 +716,35 @@ defmodule Mix.Tasks.TestTest do end end - # TODO: Get to pass after deciding on implementation - @tag :skip describe "--dry-run" do - test "prints which tests would run without executing them" do + test "works with --stale" do in_fixture("test_stale", fn -> - File.write!("test/dry_run_test.exs", """ + File.write!("test/dry_run_test_stale.exs", """ defmodule DryRunTest do use ExUnit.Case - test "passing test" do + test "new test" do assert true end - - test "failing test" do - assert false - end end """) - assert {output, 0} = mix_code(["test", "--dry-run", "--stale"]) - assert output =~ "DRY RUN" - assert output =~ "test/dry_run_test.exs" - refute output =~ "Finished in" + output = mix(["test", "--dry-run", "--stale"]) + + assert output =~ "Test dry run:" + assert output =~ "test/dry_run_test_stale.exs:4" + assert output =~ "0 tests, 0 failures (dry run)" end) end - test "prints message when no tests would run" do - in_fixture("test_stale", fn -> - assert {output, 0} = mix_code(["test", "--dry-run", "non_existent_test.exs"]) - assert output =~ "DRY RUN" - assert output =~ "No tests would run" - refute output =~ "Finished in" + test "works with --failed" do + in_fixture("test_failed", fn -> + _initial_run = mix(["test"]) + output = mix(["test", "--dry-run", "--failed"]) + + assert output =~ "Test dry run:" + assert output =~ "test/passing_and_failing_test_failed.exs:5" + assert output =~ "0 tests, 0 failures (dry run)" end) end end