From 87a15f88a1d61bb389f07bbfd1463bac6d3dfca7 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Sat, 23 Mar 2024 13:13:57 +0800 Subject: [PATCH 1/4] generate checksum file for each target --- lib/elixir_make/artefact.ex | 24 ++++++- lib/mix/tasks/elixir_make.checksum.ex | 91 ++++++++++++++++++------- lib/mix/tasks/elixir_make.precompile.ex | 2 + 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/lib/elixir_make/artefact.ex b/lib/elixir_make/artefact.ex index 9e910c6..787a82a 100644 --- a/lib/elixir_make/artefact.ex +++ b/lib/elixir_make/artefact.ex @@ -23,13 +23,33 @@ defmodule ElixirMake.Artefact do cache_dir end + @doc """ + Returns the checksum algorithm + """ + def checksum_algo do + @checksum_algo + end + @doc """ Computes the checksum and artefact for the given contents. """ def checksum(basename, contents) do - hash = :crypto.hash(@checksum_algo, contents) + hash = :crypto.hash(checksum_algo(), contents) checksum = Base.encode16(hash, case: :lower) - %Artefact{basename: basename, checksum: checksum, checksum_algo: @checksum_algo} + %Artefact{basename: basename, checksum: checksum, checksum_algo: checksum_algo()} + end + + @doc """ + Writes checksum for the target to disk. + """ + def write_checksum_for_target!(%Artefact{ + basename: basename, + checksum: checksum, + checksum_algo: checksum_algo + }) do + cache_dir = Artefact.cache_dir() + file = Path.join(cache_dir, "#{basename}.#{Atom.to_string(checksum_algo)}") + File.write!(file, [checksum, " ", basename, "\n"]) end @doc """ diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex index 58fd32b..d9aeef7 100644 --- a/lib/mix/tasks/elixir_make.checksum.ex +++ b/lib/mix/tasks/elixir_make.checksum.ex @@ -103,38 +103,79 @@ defmodule Mix.Tasks.ElixirMake.Checksum do tasks = Task.async_stream( urls, - fn {{_target, _nif_version}, url} -> {url, Artefact.download(url)} end, + fn {{_target, _nif_version}, url} -> + checksum_algo = Artefact.checksum_algo() + checksum_file_url = "#{url}.#{Atom.to_string(checksum_algo)}" + artifact_checksum = Artefact.download(checksum_file_url) + + from_checksum_file = + case artifact_checksum do + {:ok, body} -> + case String.split(body, " ", trim: true) do + [checksum, basename] -> + {:ok, + {:checksum, url, + %Artefact{ + basename: basename, + checksum: checksum, + checksum_algo: checksum_algo + }}} + + _ -> + :download_and_compute + end + + _ -> + :download_and_compute + end + + case from_checksum_file do + {:ok, packed} -> + packed + + _ -> + {:download, url, Artefact.download(url)} + end + end, timeout: :infinity, ordered: false ) cache_dir = Artefact.cache_dir() - Enum.flat_map(tasks, fn {:ok, {url, download}} -> - case download do - {:ok, body} -> - basename = basename_from_url(url) - path = Path.join(cache_dir, basename) - File.write!(path, body) - artefact = Artefact.checksum(basename, body) - - Mix.shell().info( - "NIF cached at #{path} with checksum #{artefact.checksum} (#{artefact.checksum_algo})" - ) - - [artefact] - - result -> - if ignore_unavailable? do - msg = "Skipped unavailable NIF artifact. Reason: #{inspect(result)}" - Mix.shell().info(msg) - else - msg = "Could not finish the download of NIF artifacts. Reason: #{inspect(result)}" - Mix.shell().error(msg) - end + Enum.flat_map(tasks, fn + {:ok, :checksum, url, artefact} -> + Mix.shell().info( + "NIF checksum file with checksum #{artefact.checksum} (#{artefact.checksum_algo})" + ) - [] - end + [artefact] + + {:ok, {:download, url, download}} -> + case download do + {:ok, body} -> + basename = basename_from_url(url) + path = Path.join(cache_dir, basename) + File.write!(path, body) + artefact = Artefact.checksum(basename, body) + + Mix.shell().info( + "NIF cached at #{path} with checksum #{artefact.checksum} (#{artefact.checksum_algo})" + ) + + [artefact] + + result -> + if ignore_unavailable? do + msg = "Skipped unavailable NIF artifact. Reason: #{inspect(result)}" + Mix.shell().info(msg) + else + msg = "Could not finish the download of NIF artifacts. Reason: #{inspect(result)}" + Mix.shell().error(msg) + end + + [] + end end) end diff --git a/lib/mix/tasks/elixir_make.precompile.ex b/lib/mix/tasks/elixir_make.precompile.ex index 67d751d..79bb2f3 100644 --- a/lib/mix/tasks/elixir_make.precompile.ex +++ b/lib/mix/tasks/elixir_make.precompile.ex @@ -48,6 +48,8 @@ defmodule Mix.Tasks.ElixirMake.Precompile do precompiler.post_precompile_target(target) end + Artefact.write_checksum_for_target!(precompiled_artefacts) + precompiled_artefacts {:error, msg} -> From 6af9e013f4cc0f70e3f310a28a79289df1897407 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Sat, 23 Mar 2024 15:44:03 +0800 Subject: [PATCH 2/4] underscore unused variable --- lib/mix/tasks/elixir_make.checksum.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex index d9aeef7..af518e7 100644 --- a/lib/mix/tasks/elixir_make.checksum.ex +++ b/lib/mix/tasks/elixir_make.checksum.ex @@ -144,7 +144,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do cache_dir = Artefact.cache_dir() Enum.flat_map(tasks, fn - {:ok, :checksum, url, artefact} -> + {:ok, :checksum, _url, artefact} -> Mix.shell().info( "NIF checksum file with checksum #{artefact.checksum} (#{artefact.checksum_algo})" ) From ac70d83aa49e22744bdb132bc960a6d63c953577 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Sun, 24 Mar 2024 00:04:51 +0800 Subject: [PATCH 3/4] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- lib/mix/tasks/elixir_make.checksum.ex | 39 ++++++++------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex index af518e7..7590f7f 100644 --- a/lib/mix/tasks/elixir_make.checksum.ex +++ b/lib/mix/tasks/elixir_make.checksum.ex @@ -108,33 +108,16 @@ defmodule Mix.Tasks.ElixirMake.Checksum do checksum_file_url = "#{url}.#{Atom.to_string(checksum_algo)}" artifact_checksum = Artefact.download(checksum_file_url) - from_checksum_file = - case artifact_checksum do - {:ok, body} -> - case String.split(body, " ", trim: true) do - [checksum, basename] -> - {:ok, - {:checksum, url, - %Artefact{ - basename: basename, - checksum: checksum, - checksum_algo: checksum_algo - }}} - - _ -> - :download_and_compute - end - - _ -> - :download_and_compute - end - - case from_checksum_file do - {:ok, packed} -> - packed - - _ -> - {:download, url, Artefact.download(url)} + with {:ok, body} <- artifact_checksum, + [checksum, basename] <- String.split(body, " ", trim: true) do + {:checksum, url, + %Artefact{ + basename: basename, + checksum: checksum, + checksum_algo: checksum_algo + }} + else + _ -> {:download, url, Artefact.download(url)} end end, timeout: :infinity, @@ -144,7 +127,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do cache_dir = Artefact.cache_dir() Enum.flat_map(tasks, fn - {:ok, :checksum, _url, artefact} -> + {:ok, {:checksum, _url, artefact}} -> Mix.shell().info( "NIF checksum file with checksum #{artefact.checksum} (#{artefact.checksum_algo})" ) From ef538dbd48542d9d3827e80c92c34d49741c4a39 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Sun, 24 Mar 2024 19:51:53 +0800 Subject: [PATCH 4/4] trim basename --- lib/mix/tasks/elixir_make.checksum.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex index 7590f7f..31a0648 100644 --- a/lib/mix/tasks/elixir_make.checksum.ex +++ b/lib/mix/tasks/elixir_make.checksum.ex @@ -112,7 +112,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do [checksum, basename] <- String.split(body, " ", trim: true) do {:checksum, url, %Artefact{ - basename: basename, + basename: String.trim(basename), checksum: checksum, checksum_algo: checksum_algo }}