From 1e4ca4440405c2599fa00090cd4b335dc2f82eb9 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 1 Jan 2025 19:10:47 -0500 Subject: [PATCH 1/3] By default, propagate `Base.active_project()`, `Base.LOAD_PATH`, and `Base.DEPOT_PATH` to the workers (but allow the user to override that behavior by specifying `JULIA_{PROJECT,LOAD_PATH,DEPOT_PATH}` in `params[:env]`) --- src/slurmmanager.jl | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/slurmmanager.jl b/src/slurmmanager.jl index 91e43ae..2bbc339 100644 --- a/src/slurmmanager.jl +++ b/src/slurmmanager.jl @@ -74,11 +74,67 @@ end function _new_environment_additions(params_env::Dict{String, String}) env2 = Dict{String, String}() + user_did_specify_JULIA_PROJECT = false + user_did_specify_JULIA_LOAD_PATH = false + user_did_specify_JULIA_DEPOT_PATH = false + for (name, value) in pairs(params_env) # For each key-value mapping in `params[:env]`, we respect that mapping and we pass it # to the workers. env2[name] = value + + # If the user did specify `JULIA_{PROJECT,LOAD_PATH,DEPOT_PATH}` in `params[:env]`, then + # we respect that value, and we pass it to the workers. + if name == "JULIA_PROJECT" + user_did_specify_JULIA_PROJECT = true + @debug "The user did specify a value for JULIA_PROJECT in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_PROJECT] + end + if name == "JULIA_LOAD_PATH" + user_did_specify_JULIA_LOAD_PATH = true + @debug "The user did specify a value for JULIA_LOAD_PATH in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_LOAD_PATH] + end + if name == "JULIA_DEPOT_PATH" + user_did_specify_JULIA_DEPOT_PATH = true + @debug "The user did specify a value for JULIA_DEPOT_PATH in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_DEPOT_PATH] + end end + + directory_separator = Sys.iswindows ? ';' : ':' + + # If the user did not specify `JULIA_PROJECT` in `params[:env]`, then we pass + # JULIA_PROJECT=Base.active_project() to the workers. + # + # This use case is commonly hit when the user does NOT set the `JULIA_PROJECT` environment + # variable but DOES start Julia with either `julia --project` or `julia --project=something`. + # + # https://github.com/kleinhenz/SlurmClusterManager.jl/issues/16 + if !user_did_specify_JULIA_PROJECT + # Important note: We use Base.active_project() here. + # We do NOT use Base.ACTIVE_PROJECT[], because it is not part of Julia's public API. + env2["JULIA_PROJECT"] = Base.active_project() + @debug "Passing JULIA_PROJECT=Base.active_project() to the workers" env2["JULIA_PROJECT"] + end + + # If the user did not specify `JULIA_LOAD_PATH` in `params[:env]`, then we pass + # JULIA_LOAD_PATH=Base.LOAD_PATH to the workers. + # + # This is a bit of an edge case, and I doubt that most users will need it. + # But upstream Distributed.jl does it, so we might as well do it too. + if !user_did_specify_JULIA_LOAD_PATH + env2["JULIA_LOAD_PATH"] = join(Base.LOAD_PATH, directory_separator) + @debug "Passing JULIA_LOAD_PATH=Base.LOAD_PATH to the workers" env2["JULIA_LOAD_PATH"] + end + + # If the user did not specify `JULIA_DEPOT_PATH` in `params[:env]`, then we pass + # JULIA_DEPOT_PATH=Base.DEPOT_PATH to the workers. + # + # This is a bit of an edge case, and I doubt that most users will need it. + # But upstream Distributed.jl does it, so we might as well do it too. + if !user_did_specify_JULIA_DEPOT_PATH + env2["JULIA_DEPOT_PATH"] = join(Base.DEPOT_PATH, directory_separator) + @debug "Passing JULIA_DEPOT_PATH=Base.DEPOT_PATH to the workers" env2["JULIA_DEPOT_PATH"] + end + return env2 end From 890918d38c509911ec728b21834a5d35a0db5dca Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 16 Jan 2025 17:36:20 -0500 Subject: [PATCH 2/3] Refactoring --- src/slurmmanager.jl | 50 ++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/src/slurmmanager.jl b/src/slurmmanager.jl index 2bbc339..74dc26d 100644 --- a/src/slurmmanager.jl +++ b/src/slurmmanager.jl @@ -73,29 +73,13 @@ elseif Base.VERSION < v"1.6.0" end function _new_environment_additions(params_env::Dict{String, String}) - env2 = Dict{String, String}() - user_did_specify_JULIA_PROJECT = false - user_did_specify_JULIA_LOAD_PATH = false - user_did_specify_JULIA_DEPOT_PATH = false - - for (name, value) in pairs(params_env) - # For each key-value mapping in `params[:env]`, we respect that mapping and we pass it - # to the workers. - env2[name] = value - - # If the user did specify `JULIA_{PROJECT,LOAD_PATH,DEPOT_PATH}` in `params[:env]`, then - # we respect that value, and we pass it to the workers. - if name == "JULIA_PROJECT" - user_did_specify_JULIA_PROJECT = true - @debug "The user did specify a value for JULIA_PROJECT in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_PROJECT] - end - if name == "JULIA_LOAD_PATH" - user_did_specify_JULIA_LOAD_PATH = true - @debug "The user did specify a value for JULIA_LOAD_PATH in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_LOAD_PATH] - end - if name == "JULIA_DEPOT_PATH" - user_did_specify_JULIA_DEPOT_PATH = true - @debug "The user did specify a value for JULIA_DEPOT_PATH in the `env` kwarg to `addprocs()`; that value will be passed to the workers" env2[JULIA_DEPOT_PATH] + # For each key-value mapping in `params[:env]`, we respect that mapping and we pass it + # to the workers. + env2 = copy(params_env) + + for name in ["JULIA_PROJECT", "JULIA_LOAD_PATH", "JULIA_DEPOT_PATH"] + if haskey(env2, name) + @debug "The user did specify $(name)=$(env2[name]) in the `env` kwarg to `addprocs()`. That value will be passed to the workers." end end @@ -108,11 +92,11 @@ function _new_environment_additions(params_env::Dict{String, String}) # variable but DOES start Julia with either `julia --project` or `julia --project=something`. # # https://github.com/kleinhenz/SlurmClusterManager.jl/issues/16 - if !user_did_specify_JULIA_PROJECT - # Important note: We use Base.active_project() here. + if !haskey(env2, "JULIA_PROJECT") + # Important note: We use Base.active_project() here. # We do NOT use Base.ACTIVE_PROJECT[], because it is not part of Julia's public API. env2["JULIA_PROJECT"] = Base.active_project() - @debug "Passing JULIA_PROJECT=Base.active_project() to the workers" env2["JULIA_PROJECT"] + @debug "Passing JULIA_PROJECT=Base.active_project()=$(env2["JULIA_PROJECT"]) to the workers" end # If the user did not specify `JULIA_LOAD_PATH` in `params[:env]`, then we pass @@ -120,9 +104,9 @@ function _new_environment_additions(params_env::Dict{String, String}) # # This is a bit of an edge case, and I doubt that most users will need it. # But upstream Distributed.jl does it, so we might as well do it too. - if !user_did_specify_JULIA_LOAD_PATH + if !haskey(env2, "JULIA_LOAD_PATH") env2["JULIA_LOAD_PATH"] = join(Base.LOAD_PATH, directory_separator) - @debug "Passing JULIA_LOAD_PATH=Base.LOAD_PATH to the workers" env2["JULIA_LOAD_PATH"] + @debug "Passing JULIA_LOAD_PATH=Base.LOAD_PATH=$(env2["JULIA_LOAD_PATH"]) to the workers" end # If the user did not specify `JULIA_DEPOT_PATH` in `params[:env]`, then we pass @@ -130,9 +114,9 @@ function _new_environment_additions(params_env::Dict{String, String}) # # This is a bit of an edge case, and I doubt that most users will need it. # But upstream Distributed.jl does it, so we might as well do it too. - if !user_did_specify_JULIA_DEPOT_PATH + if !haskey(env2, "JULIA_DEPOT_PATH") env2["JULIA_DEPOT_PATH"] = join(Base.DEPOT_PATH, directory_separator) - @debug "Passing JULIA_DEPOT_PATH=Base.DEPOT_PATH to the workers" env2["JULIA_DEPOT_PATH"] + @debug "Passing JULIA_DEPOT_PATH=Base.DEPOT_PATH=$(env2["JULIA_DEPOT_PATH"]) to the workers" end return env2 @@ -147,12 +131,8 @@ function launch(manager::SlurmManager, params::Dict, instances_arr::Array, c::Co _srun_cmd_without_env = `srun -D $exehome $exename $exeflags --worker` @static if Base.VERSION >= v"1.6.0" - env_arr = params[:env] # Pass the key-value pairs from `params[:env]` to the `srun` command: - env2 = Dict{String,String}() - for (name, value) in pairs(Dict{String,String}(env_arr)) - env2[name] = value - end + env2 = _new_environment_additions(Dict{String,String}(params[:env])) srun_cmd_with_env = addenv(_srun_cmd_without_env, env2) else # See discussion above for why we don't support this functionality on Julia 1.5 and earlier. From 0c3c2036d73ae25013033e50e717e365952c07db Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 16 Jan 2025 17:42:04 -0500 Subject: [PATCH 3/3] Fix a syntax mistake --- src/slurmmanager.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/slurmmanager.jl b/src/slurmmanager.jl index 74dc26d..1e3cd7d 100644 --- a/src/slurmmanager.jl +++ b/src/slurmmanager.jl @@ -83,7 +83,7 @@ function _new_environment_additions(params_env::Dict{String, String}) end end - directory_separator = Sys.iswindows ? ';' : ':' + directory_separator = Sys.iswindows() ? ';' : ':' # If the user did not specify `JULIA_PROJECT` in `params[:env]`, then we pass # JULIA_PROJECT=Base.active_project() to the workers.