diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index 7f2fee48fda77..192db87043177 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -19,11 +19,12 @@ set(CROSS_PROJECT_TEST_DEPS FileCheck check-gdb-llvm-support count - llvm-dwarfdump + llvm-ar llvm-config + llvm-dwarfdump llvm-objdump - split-file not + split-file ) if ("clang" IN_LIST LLVM_ENABLE_PROJECTS) @@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests" DEPENDS clang ) +# DTLTO tests. +add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests" + ${CMAKE_CURRENT_BINARY_DIR}/dtlto + EXCLUDE_FROM_CHECK_ALL + DEPENDS ${CROSS_PROJECT_TEST_DEPS} + ) + # Add check-cross-project-* targets. add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${CROSS_PROJECT_TEST_DEPS} diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md new file mode 100644 index 0000000000000..12f9aa19b0d9b --- /dev/null +++ b/cross-project-tests/dtlto/README.md @@ -0,0 +1,3 @@ +Tests for DTLTO (Integrated Distributed ThinLTO) functionality. + +These are integration tests as DTLTO invokes `clang` for code-generation. \ No newline at end of file diff --git a/cross-project-tests/dtlto/archive-thin.test b/cross-project-tests/dtlto/archive-thin.test new file mode 100644 index 0000000000000..2d6b7f1cb7e37 --- /dev/null +++ b/cross-project-tests/dtlto/archive-thin.test @@ -0,0 +1,65 @@ +# REQUIRES: x86-registered-target,ld.lld,llvm-ar + +# Test that a DTLTO link succeeds and outputs the expected set of files +# correctly when thin archives are present. + +RUN: rm -rf %t && split-file %s %t && cd %t +RUN: %clang --target=x86_64-linux-gnu -flto=thin -c \ +RUN: foo.c bar.c dog.c cat.c _start.c + +RUN: llvm-ar rcs foo.a foo.o --thin +# Create this bitcode thin archive in a subdirectory to test the expansion of +# the path to a bitcode file that is referenced using "..", e.g., in this case +# "../bar.o". +RUN: mkdir lib +RUN: llvm-ar rcs lib/bar.a bar.o --thin +# Create this bitcode thin archive with an absolute path entry containing "..". +RUN: llvm-ar rcs dog.a %t/lib/../dog.o --thin +RUN: llvm-ar rcs cat.a cat.o --thin +RUN: llvm-ar rcs _start.a _start.o --thin + +RUN: mkdir %t/out && cd %t/out + +RUN: ld.lld %t/foo.a %t/lib/bar.a ../_start.a %t/cat.a \ +RUN: --whole-archive ../dog.a \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +RUN: --thinlto-remote-compiler=%clang \ +RUN: --save-temps + +# Check that the required output files have been created. +RUN: ls | FileCheck %s --check-prefix=OUTPUTS \ +RUN: --implicit-check-not=cat --implicit-check-not=foo + +# JSON jobs description. +OUTPUTS-DAG: a.[[PID:[a-zA-Z0-9_]+]].dist-file.json + +# Individual summary index files. +OUTPUTS-DAG: dog.1.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: _start.2.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: bar.3.[[PID]].native.o.thinlto.bc{{$}} + +# Native output object files. +OUTPUTS-DAG: dog.1.[[PID]].native.o{{$}} +OUTPUTS-DAG: _start.2.[[PID]].native.o{{$}} +OUTPUTS-DAG: bar.3.[[PID]].native.o{{$}} + +#--- foo.c +__attribute__((retain)) void foo() {} + +#--- bar.c +extern void foo(); +__attribute__((retain)) void bar() { foo(); } + +#--- dog.c +__attribute__((retain)) void dog() {} + +#--- cat.c +__attribute__((retain)) void cat() {} + +#--- _start.c +extern void bar(); +__attribute__((retain)) void _start() { + bar(); +} + diff --git a/cross-project-tests/dtlto/dtlto.c b/cross-project-tests/dtlto/dtlto.c new file mode 100644 index 0000000000000..e967ede35b09b --- /dev/null +++ b/cross-project-tests/dtlto/dtlto.c @@ -0,0 +1,35 @@ +// REQUIRES: x86-registered-target,ld.lld + +/// Simple test that DTLTO works with a single input bitcode file and that +/// --save-temps can be applied to the remote compilation. +// RUN: rm -rf %t && mkdir %t && cd %t + +// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s + +// RUN: ld.lld dtlto.o \ +// RUN: --thinlto-distributor=%python \ +// RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +// RUN: --thinlto-remote-compiler=%clang \ +// RUN: --thinlto-remote-compiler-arg=--save-temps + +/// Check that the required output files have been created. +// RUN: ls | count 10 +// RUN: ls | FileCheck %s + +/// Produced by the bitcode compilation. +// CHECK-DAG: {{^}}dtlto.o{{$}} + +/// Linked ELF. +// CHECK-DAG: {{^}}a.out{{$}} + +/// --save-temps output for the backend compilation. +// CHECK-DAG: {{^}}dtlto.s{{$}} +// CHECK-DAG: {{^}}dtlto.s.0.preopt.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.1.promote.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.2.internalize.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.3.import.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.4.opt.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.5.precodegen.bc{{$}} +// CHECK-DAG: {{^}}dtlto.s.resolution.txt{{$}} + +int _start() { return 0; } diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg new file mode 100644 index 0000000000000..530f4c01646ff --- /dev/null +++ b/cross-project-tests/dtlto/lit.local.cfg @@ -0,0 +1,2 @@ +if "clang" not in config.available_features: + config.unsupported = True diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py index b35c643ac898c..ac27753472646 100644 --- a/cross-project-tests/lit.cfg.py +++ b/cross-project-tests/lit.cfg.py @@ -19,7 +19,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = [".c", ".cl", ".cpp", ".m"] +config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -107,6 +107,8 @@ def get_required_attr(config, attr_name): if lldb_path is not None: config.available_features.add("lldb") +if llvm_config.use_llvm_tool("llvm-ar"): + config.available_features.add("llvm-ar") def configure_dexter_substitutions(): """Configure substitutions for host platform and return list of dependencies""" diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index f0e9592d85dd6..5e0fb9c9b00ab 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -249,6 +249,10 @@ struct Config { llvm::SmallVector searchPaths; llvm::SmallVector symbolOrderingFile; llvm::SmallVector thinLTOModulesToCompile; + llvm::StringRef dtltoDistributor; + llvm::SmallVector dtltoDistributorArgs; + llvm::StringRef dtltoCompiler; + llvm::SmallVector dtltoCompilerArgs; llvm::SmallVector undefined; llvm::SmallVector dynamicList; llvm::SmallVector buildIdVector; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 6150fe072156f..e89eaef27d6bd 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1341,6 +1341,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) { args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true); ctx.arg.disableVerify = args.hasArg(OPT_disable_verify); ctx.arg.discard = getDiscard(args); + ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq); + ctx.arg.dtltoDistributorArgs = + args::getStrings(args, OPT_thinlto_distributor_arg); + ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq); + ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg); ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq); ctx.arg.dynamicLinker = getDynamicLinker(ctx, args); ctx.arg.ehFrameHdr = diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 12a77736aba7f..5523d929f6bfd 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/STLExtras.h" #include "llvm/LTO/LTO.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Support/ARMAttributeParser.h" #include "llvm/Support/ARMBuildAttributes.h" @@ -1739,6 +1740,33 @@ static uint8_t getOsAbi(const Triple &t) { } } +// For DTLTO, bitcode member names must be valid paths to files on disk. +// For thin archives, resolve `memberPath` relative to the archive's location. +// Returns true if adjusted; false otherwise. Non-thin archives are unsupported. +static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath, + std::string &memberPath) { + assert(!archivePath.empty() && !ctx.arg.dtltoDistributor.empty()); + + // Check if the archive file is a thin archive by reading its header. + auto bufferOrErr = + MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0); + if (std::error_code ec = bufferOrErr.getError()) { + ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message(); + return false; + } + + if (!bufferOrErr->get()->getBuffer().starts_with(ThinArchiveMagic)) + return false; + + SmallString<64> resolvedPath; + if (path::is_relative(memberPath)) { + resolvedPath = path::parent_path(archivePath); + path::append(resolvedPath, memberPath); + memberPath = resolvedPath.str(); + } + return true; +} + BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, bool lazy) : InputFile(ctx, BitcodeKind, mb) { @@ -1756,10 +1784,13 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, // symbols later in the link stage). So we append file offset to make // filename unique. StringSaver &ss = ctx.saver; - StringRef name = archiveName.empty() - ? ss.save(path) - : ss.save(archiveName + "(" + path::filename(path) + - " at " + utostr(offsetInArchive) + ")"); + StringRef name = + (archiveName.empty() || + (!ctx.arg.dtltoDistributor.empty() && + dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path))) + ? ss.save(path) + : ss.save(archiveName + "(" + path::filename(path) + " at " + + utostr(offsetInArchive) + ")"); MemoryBufferRef mbref(mb.getBuffer(), name); obj = CHECK2(lto::InputFile::create(mbref), this); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 82a7463446a94..8d4a6c9e3a81e 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { std::string(ctx.arg.thinLTOPrefixReplaceNew), std::string(ctx.arg.thinLTOPrefixReplaceNativeObject), ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite); + } else if (!ctx.arg.dtltoDistributor.empty()) { + backend = lto::createOutOfProcessThinBackend( + llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite, + ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles, + ctx.arg.outputFile, ctx.arg.dtltoDistributor, + ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler, + ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty()); } else { backend = lto::createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs), diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index c795147eb9662..6599691244fe8 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -710,7 +710,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">; def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">; def thinlto_single_module_eq: JJ<"thinlto-single-module=">, HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">; - +def thinlto_distributor_eq: JJ<"thinlto-distributor=">, + HelpText<"Distributor to use for ThinLTO backend compilations. If specified, " + "ThinLTO backend compilations will be distributed.">; +defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to " + "pass to the ThinLTO distributor">; +def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">, + HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend " + "compilations">; +defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler " + "arguments for the ThinLTO distributor to pass for ThinLTO backend " + "compilations">; defm fat_lto_objects: BB<"fat-lto-objects", "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.", "Ignore the .llvm.lto section in relocatable object files (default).">; diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst new file mode 100644 index 0000000000000..985decf6c7db8 --- /dev/null +++ b/lld/docs/DTLTO.rst @@ -0,0 +1,42 @@ +Integrated Distributed ThinLTO (DTLTO) +====================================== + +Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend +ThinLTO compilations via external distribution systems, such as Incredibuild, +during the traditional link step. + +The implementation is documented here: https://llvm.org/docs/DTLTO.html. + +Currently, DTLTO is only supported in ELF LLD. Support will be added to other +LLD flavours in the future. + +ELF LLD +------- + +The command-line interface is as follows: + +- ``--thinlto-distributor=`` + Specifies the file to execute as the distributor process. If specified, + ThinLTO backend compilations will be distributed. + +- ``--thinlto-remote-compiler=`` + Specifies the path to the compiler that the distributor process will use for + backend compilations. The compiler invoked must match the version of LLD. + +- ``--thinlto-distributor-arg=`` + Specifies ```` on the command line when invoking the distributor. + Can be specified multiple times. + +- ``--thinlto-remote-compiler-arg=`` + Appends ```` to the remote compiler's command line. + Can be specified multiple times. + + Options that introduce extra input/output files may cause miscompilation if + the distribution system does not automatically handle pushing/fetching them to + remote nodes. In such cases, configure the distributor - possibly using + ``--thinlto-distributor-arg=`` - to manage these dependencies. See the + distributor documentation for details. + +Some LLD LTO options (e.g., ``--lto-sample-profile=``) are supported. +Currently, other options are silently accepted but do not have the intended +effect. Support for such options will be expanded in the future. diff --git a/lld/docs/index.rst b/lld/docs/index.rst index 8260461c36905..69792e3b575be 100644 --- a/lld/docs/index.rst +++ b/lld/docs/index.rst @@ -147,3 +147,4 @@ document soon. ELF/start-stop-gc ELF/warn_backrefs MachO/index + DTLTO diff --git a/lld/test/ELF/dtlto/archive-thin.text b/lld/test/ELF/dtlto/archive-thin.text new file mode 100644 index 0000000000000..c9f2c40cd934c --- /dev/null +++ b/lld/test/ELF/dtlto/archive-thin.text @@ -0,0 +1,84 @@ +# REQUIRES: x86 + +# Test that a DTLTO link succeeds and outputs the expected set of files +# correctly when thin archives are present. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc -O2 +RUN: opt -thinlto-bc t2.ll -o t2.bc -O2 +RUN: opt -thinlto-bc t3.ll -o t3.bc -O2 + +# Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic +RUN: llc t2.ll --filetype=obj -o t2.o --relocation-model=pic +RUN: llc t3.ll --filetype=obj -o t3.o --relocation-model=pic + +RUN: llvm-ar rcs t1.a t1.bc --thin +# Create this bitcode thin archive in a subdirectory to test the expansion of +# the path to a bitcode file which is referenced using "..", e.g., in this case +# "../t2.bc". The ".." should be collapsed in any expansion to avoid +# referencing an unknown directory on the remote side. +RUN: mkdir lib +RUN: llvm-ar rcs lib/t2.a t2.bc --thin +# Create this bitcode thin archive with an absolute path entry containing "..". +RUN: llvm-ar rcs t3.a %t/lib/../t3.bc --thin +RUN: llvm-ar rcs t4.a t1.bc --thin + +RUN: mkdir %t/out && cd %t/out + +# Note that mock.py does not do any compilation, instead it simply writes the +# contents of the object files supplied on the command line into the output +# object files in job order. +RUN: ld.lld --whole-archive %t/t1.a %t/lib/t2.a ../t3.a \ +RUN: --no-whole-archive %t/t4.a \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=../t1.o \ +RUN: --thinlto-distributor-arg=../t2.o \ +RUN: --thinlto-distributor-arg=../t3.o \ +RUN: --save-temps + +RUN: ls | FileCheck %s --implicit-check-not=4 + +# JSON jobs description and summary shards. +CHECK-DAG: a.{{[0-9]+}}.dist-file.json +CHECK-DAG: t1.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} +CHECK-DAG: t2.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} +CHECK-DAG: t3.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} + +# Native output object files. +CHECK-DAG: t1.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} +CHECK-DAG: t2.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} +CHECK-DAG: t3.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} + +# Check that t2.o and t3.o are not referenced using "..". +# RUN: not grep '\.\.\(/\|\\\\\)\(t2\|t3\)\.bc' a.*.dist-file.json + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} + +#--- t2.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t2() { +entry: + ret void +} + +#--- t3.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t3() { +entry: + ret void +} diff --git a/lld/test/ELF/dtlto/empty.test b/lld/test/ELF/dtlto/empty.test new file mode 100644 index 0000000000000..5f69d90051220 --- /dev/null +++ b/lld/test/ELF/dtlto/empty.test @@ -0,0 +1,44 @@ +# REQUIRES: x86 + +# Test that DTLTO options are a no-op if no thinLTO is performed. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: opt t1.ll -o t1.o -O2 +RUN: opt t2.ll -o t2.o -O2 + +# Common command-line arguments. Note that mock.py does not do any compilation; +# instead, it simply writes the contents of the object files supplied on the +# command line into the output object files in job order. +RUN: echo "t1.o t2.o \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=no-exist1.o \ +RUN: --thinlto-distributor-arg=no-exist2.o" > l.rsp + +# Check linking succeeds when all input files are Full LTO. +RUN: ld.lld @l.rsp + +RUN: llc -filetype=obj t1.ll -o t1.o +RUN: llc -filetype=obj t2.ll -o t2.o + +# Check linking succeeds when all input files are ET_RELs. +RUN: ld.lld @l.rsp + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} + +#--- t2.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t2() { +entry: + ret void +} diff --git a/lld/test/ELF/dtlto/imports.test b/lld/test/ELF/dtlto/imports.test new file mode 100644 index 0000000000000..74d5223f04e24 --- /dev/null +++ b/lld/test/ELF/dtlto/imports.test @@ -0,0 +1,53 @@ +# REQUIRES: x86 + +# Check that DTLTO creates imports lists if requested. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: opt -thinlto-bc t1.ll -o t1.bc -O2 +RUN: opt -thinlto-bc t2.ll -o t2.bc -O2 + +# Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic +RUN: llc t2.ll --filetype=obj -o t2.o --relocation-model=pic + +# Common command-line arguments. Note that mock.py does not do any compilation; +# instead, it simply writes the contents of the object files supplied on the +# command line into the output object files in job order. +RUN: echo "t1.bc t2.bc \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=t1.o \ +RUN: --thinlto-distributor-arg=t2.o" > l.rsp + +# Check that imports files are not created normally. +RUN: ld.lld @l.rsp +RUN: ls | FileCheck %s --check-prefix=NOIMPORTSFILES +NOIMPORTSFILES-NOT: .imports + +# Check that imports files are created with --thinlto-emit-imports-files. +RUN: ld.lld @l.rsp --thinlto-emit-imports-files +RUN: ls | FileCheck %s --check-prefix=IMPORTSFILES +IMPORTSFILES: t1.bc.imports +IMPORTSFILES: t2.bc.imports + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} + +#--- t2.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @t1(...) + +define void @t2() { +entry: + call void (...) @t1() + ret void +} diff --git a/lld/test/ELF/dtlto/index.test b/lld/test/ELF/dtlto/index.test new file mode 100644 index 0000000000000..4a869c65468fe --- /dev/null +++ b/lld/test/ELF/dtlto/index.test @@ -0,0 +1,44 @@ +# REQUIRES: x86 + +# Check that DTLTO creates individual summary index files if requested. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc -O2 +RUN: opt -thinlto-bc t1.ll -o t2.bc -O2 + +# Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic + +# Use a response file for the common command-line arguments. +# Note that mock.py does not perform any compilation; instead, it copies the +# contents of the specified object files into the output object files, in the +# order the jobs are received. +# The "--start-lib/--end-lib" options are used to exercise the special case +# where unused lazy object inputs result in empty index files. +RUN: echo "t1.bc --start-lib t2.bc --end-lib \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=t1.o \ +RUN: --thinlto-distributor-arg=t2.o" > l.rsp + +# Check that index files are not created normally. +RUN: ld.lld @l.rsp +RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES +NOINDEXFILES-NOT: .thinlto.bc + +# Check that index files are created with --thinlto-emit-index-files. +RUN: ld.lld @l.rsp --thinlto-emit-index-files +RUN: ls | FileCheck %s --check-prefix=INDEXFILES +INDEXFILES: t1.bc.thinlto.bc +INDEXFILES: t2.bc.thinlto.bc + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} diff --git a/lld/test/ELF/dtlto/options.test b/lld/test/ELF/dtlto/options.test new file mode 100644 index 0000000000000..53f0eb41952d0 --- /dev/null +++ b/lld/test/ELF/dtlto/options.test @@ -0,0 +1,41 @@ +# REQUIRES: x86 + +# Test that DTLTO options are passed correctly to the distributor and +# remote compiler. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: opt -thinlto-bc foo.ll -o foo.o + +# Note: validate.py does not perform any compilation. Instead, it validates the +# received JSON, pretty-prints the JSON and the supplied arguments, and then +# exits with an error. This allows FileCheck directives to verify the +# distributor inputs. +RUN: not ld.lld foo.o \ +RUN: -o my.elf \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ +RUN: --thinlto-distributor-arg=darg1=10 \ +RUN: --thinlto-distributor-arg=darg2=20 \ +RUN: --thinlto-remote-compiler=my_clang.exe \ +RUN: --thinlto-remote-compiler-arg=carg1=20 \ +RUN: --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s + +CHECK: distributor_args=['darg1=10', 'darg2=20'] + +CHECK: "linker_output": "my.elf" + +CHECK: "my_clang.exe" +CHECK: "carg1=20" +CHECK: "carg2=30" + +CHECK: ld.lld: error: DTLTO backend compilation: cannot open native object file: + +#--- foo.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo() { +entry: + ret void +} diff --git a/lld/test/ELF/dtlto/partitions.test b/lld/test/ELF/dtlto/partitions.test new file mode 100644 index 0000000000000..306dbedcbb19c --- /dev/null +++ b/lld/test/ELF/dtlto/partitions.test @@ -0,0 +1,61 @@ +# REQUIRES: x86 + +# Test that DTLTO works with more than one LTO partition. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Compile bitcode. +RUN: llvm-as -o full.bc full.ll +RUN: opt -thinlto-bc thin1.ll -o thin1.bc +RUN: opt -thinlto-bc thin2.ll -o thin2.bc + +# Generate object files for mock.py to return. +RUN: llc thin1.ll --filetype=obj -o thin1.o --relocation-model=pic +RUN: llc thin2.ll --filetype=obj -o thin2.o --relocation-model=pic + +# Link with 3 LTO partitions. +RUN: ld.lld full.bc thin1.bc thin2.bc \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=thin1.o \ +RUN: --thinlto-distributor-arg=thin2.o \ +RUN: --save-temps \ +RUN: --lto-partitions=3 + +# DTLTO temporary object files include the task number and a PID component. The +# task number should incorporate the LTO partition number. +RUN: ls | FileCheck %s +CHECK: thin1.3.[[#]].native.o +CHECK: thin2.4.[[#]].native.o + +#--- full.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo() { + call void @bar() + ret void +} + +define void @bar() { + call void @foo() + ret void +} + +#--- thin1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @baz() { +entry: + ret void +} + +#--- thin2.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @_start() { +entry: + ret void +} diff --git a/lld/test/ELF/dtlto/save-temps.test b/lld/test/ELF/dtlto/save-temps.test new file mode 100644 index 0000000000000..07e5b0f4d8e7d --- /dev/null +++ b/lld/test/ELF/dtlto/save-temps.test @@ -0,0 +1,39 @@ +# REQUIRES: x86 + +# Check that DTLTO responds to --save-temps. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc -O2 + +# Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic + +# Use a response file for common command-line arguments. +# Note that mock.py does not perform any compilation; instead, it simply writes +# the contents of the object files supplied on the command line into the output +# object files in job order. +RUN: echo "t1.bc -o my.elf \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=t1.o" > l.rsp + +# Check that DTLTO temporary files are removed by default. +RUN: ld.lld @l.rsp +RUN: ls | FileCheck %s --check-prefix=REMOVED +REMOVED-NOT: .json + +# Check that DTLTO temporary files are retained with --save-temps. +RUN: ld.lld @l.rsp --save-temps +RUN: ls | FileCheck %s --check-prefix=RETAINED +RETAINED: my.[[#]].dist-file.json + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py index 9e6b0e839d9a8..10f556567cdc8 100644 --- a/lld/test/lit.cfg.py +++ b/lld/test/lit.cfg.py @@ -36,6 +36,7 @@ llvm_config.use_default_substitutions() llvm_config.use_lld() +config.substitutions.append(("%llvm_src_root", config.llvm_src_root)) tool_patterns = [ "llc",