cold start performance

[tomerd]

this issue is to track finding on cold start performance

[tomerd]

current benchmark based on ./scripts/performance_test.sh

-----------------------------
results
-----------------------------
string, cold: 32658752 (ns)
string, warm: 489904 (ns)
json, cold: 34155567 (ns)
json, warm: 626493 (ns)
-----------------------------

[tomerd]

cold start strace

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 26.13    0.002934          30        97        75 openat
 23.06    0.002590          28        92           futex
 10.18    0.001143          21        54           mmap
  6.30    0.000708          31        23        19 stat
  6.08    0.000683          18        39           mprotect
  5.36    0.000602          55        11           write
  4.12    0.000463          19        24           fstat
  3.46    0.000388          19        20           read
  3.24    0.000364          17        22           close
  2.63    0.000295          27        11        11 access
  2.27    0.000255          36         7           clock_gettime
  1.87    0.000210          30         7           brk
  0.80    0.000090          45         2           readlink
  0.77    0.000087          44         2           munmap
  0.75    0.000084          21         4           rt_sigaction
  0.69    0.000078          20         4           getdents
  0.62    0.000070          35         2           clone
  0.46    0.000052          17         3           getrandom
  0.31    0.000035          35         1           execve
  0.28    0.000032          32         1           lseek
  0.16    0.000018          18         1           sched_getaffinity
  0.15    0.000017          17         1           set_tid_address
  0.15    0.000017          17         1           clock_getres
  0.04    0.000004           4         1           rt_sigprocmask
  0.04    0.000004           4         1           set_robust_list
  0.04    0.000004           4         1           prlimit64
  0.03    0.000003           3         1           gettid
  0.00    0.000000           0         1           arch_prctl
  0.00    0.000000           0         1           epoll_ctl
  0.00    0.000000           0         1           eventfd2
  0.00    0.000000           0         1           epoll_create1
------ ----------- ----------- --------- --------- ----------------
100.00    0.011230                   437       105 total

so ~30% of sys calls time goes into loading dylibs

[tomerd]

cold start flame graph (see ./scripts/linux_performance_setup.sh to generate)

also showing lots of time spent in ld

[tomerd]

compiling with -static-stdlib

root@cd3b6a8d640d:/code# swift build -c release -Xswiftc -g
[14/14] Linking SwiftAwsLambdaCodableSample
root@cd3b6a8d640d:/code# objdump -p .build/release/SwiftAwsLambdaStringSample | egrep "NEEDED"
  NEEDED               libswiftGlibc.so
  NEEDED               libswiftCore.so
  NEEDED               libpthread.so.0
  NEEDED               libutil.so.1
  NEEDED               libdl.so.2
  NEEDED               libm.so.6
  NEEDED               libswiftDispatch.so
  NEEDED               libdispatch.so
  NEEDED               libBlocksRuntime.so
  NEEDED               libFoundation.so
  NEEDED               libstdc++.so.6
  NEEDED               libgcc_s.so.1
  NEEDED               libc.so.6
root@cd3b6a8d640d:/code# swift build -c release -Xswiftc -g -Xswiftc -static-stdlib -Xswiftc -L/usr/lib/swift/linux
[14/14] Linking SwiftAwsLambdaCodableSample
root@cd3b6a8d640d:/code# objdump -p .build/release/SwiftAwsLambdaStringSample | egrep "NEEDED"
  NEEDED               libpthread.so.0
  NEEDED               libutil.so.1
  NEEDED               libdl.so.2
  NEEDED               libm.so.6
  NEEDED               libswiftDispatch.so
  NEEDED               libdispatch.so
  NEEDED               libBlocksRuntime.so
  NEEDED               libFoundation.so
  NEEDED               libatomic.so.1
  NEEDED               libstdc++.so.6
  NEEDED               libgcc_s.so.1
  NEEDED               libc.so.6
  NEEDED               ld-linux-x86-64.so.2

strace

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 36.26    0.004858          46       106         1 futex
 20.86    0.002795          32        87        65 openat
  9.85    0.001319          24        54           mmap
  5.38    0.000721          66        11           write
  5.37    0.000720          18        39           mprotect
  4.00    0.000536          22        24           fstat
  3.17    0.000424          61         7           clock_gettime
  2.46    0.000330          17        20           read
  2.16    0.000290          13        22           close
  1.87    0.000251          16        16        12 stat
  1.80    0.000241          24        10           brk
  1.66    0.000222          20        11        11 access
  1.30    0.000174          87         2           clone
  0.72    0.000096          24         4           rt_sigaction
  0.64    0.000086          43         2           readlink
  0.52    0.000070          35         2           munmap
  0.33    0.000044          44         1           epoll_ctl
  0.31    0.000042          42         1           lseek
  0.29    0.000039           7         6           getrandom
  0.26    0.000035           9         4           getdents
  0.21    0.000028          28         1           execve
  0.18    0.000024          24         1           eventfd2
  0.08    0.000011          11         1           arch_prctl
  0.04    0.000006           6         1           sched_getaffinity
  0.04    0.000006           6         1           set_tid_address
  0.04    0.000006           6         1           clock_getres
  0.04    0.000006           6         1           set_robust_list
  0.04    0.000006           6         1           prlimit64
  0.04    0.000005           5         1           rt_sigprocmask
  0.04    0.000005           5         1           gettid
  0.00    0.000000           0         1           epoll_create1
------ ----------- ----------- --------- --------- ----------------
100.00    0.013396                   440        89 total

perf test results:

-----------------------------
results
-----------------------------
string, cold: 50266686 (ns)
string, warm: 443636 (ns)
-----------------------------

so less openat but perf not much different

[fabianfett]

Am I reading the flame graph correctly?

I see
5.15% in main
17.53% in NIO-ELT
4.12% in libFoundation
2.06% in __CFInitialize

The rest looks like bootup overhead to me which would mean around 70%. According to the flamegraph removing Foundation could mean a speed of of about 6% which is slightly less then what I've seen in playing with it in #5.

[tomerd]

latest test (with the refactored code from #13)
main: 12.5%
NIO-ETL: 11.25%

so boot up overhead is significant. goal is to find way to reduce that

[compnerd]

libatomic should no longer be linked to - that was a bug that @drexin recently fixed. Im now left wondering how much we really need for the C++ runtime. If we statically link the C++ runtime into swiftCore, I wonder if we can get a decent chunk of savings out of it - for relatively low cost - the linker should be able to dead strip most of the C++ runtime.

Another thing to consider: -Xlinker -z,lazy - that way you can avoid iterating the PLT and lazily fault things in. Which linker is in use here? If its not lld (I dont remember gold's default), -Xlinker -z,combreloc might be interesting on libswiftCore as well.

[tomerd]

one other interesting observation from running strace is that the load search path is not optimized (on linux):

02:02:30.513226 execve(".build/release/SwiftAwsLambdaStringSample", [".build/release/SwiftAwsLambdaStr"...], 0x7ffe39c49218 /* 16 vars */) = 0
02:02:30.518883 brk(NULL)               = 0x56554c94e000
02:02:30.521503 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
02:02:30.522703 readlink("/proc/self/exe", "/code/.build/x86_64-unknown-linu"..., 4096) = 68
02:02:30.524075 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
02:02:30.525424 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.527021 stat("/usr/lib/swift/linux/tls/x86_64/x86_64", 0x7ffc94d88110) = -1 ENOENT (No such file or directory)
02:02:30.529195 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.530950 stat("/usr/lib/swift/linux/tls/x86_64", 0x7ffc94d88110) = -1 ENOENT (No such file or directory)
02:02:30.532334 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.534237 stat("/usr/lib/swift/linux/tls/x86_64", 0x7ffc94d88110) = -1 ENOENT (No such file or directory)
02:02:30.535714 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.537202 stat("/usr/lib/swift/linux/tls", 0x7ffc94d88110) = -1 ENOENT (No such file or directory)
02:02:30.538981 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.540656 stat("/usr/lib/swift/linux/x86_64/x86_64", 0x7ffc94d88110) = -1 ENOENT (No such file or directory)
02:02:30.541855 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.543024 stat("/usr/lib/swift/linux/x86_64", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
02:02:30.544132 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.545212 stat("/usr/lib/swift/linux/x86_64", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
02:02:30.546496 openat(AT_FDCWD, "/usr/lib/swift/linux/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = 3
02:02:30.547796 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
02:02:30.549036 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f91d1d79000
02:02:30.550117 fstat(3, {st_mode=S_IFREG|0755, st_size=93600, ...}) = 0
02:02:30.551138 mmap(NULL, 64824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f91d1d69000
02:02:30.552282 mprotect(0x7f91d1d76000, 4096, PROT_NONE) = 0
02:02:30.553367 mmap(0x7f91d1d77000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7f91d1d77000
02:02:30.554503 close(3)                = 0
02:02:30.555611 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.556590 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
02:02:30.557836 openat(AT_FDCWD, "/usr/lib/swift/linux/libswiftCore.so", O_RDONLY|O_CLOEXEC) = 3
02:02:30.559027 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
02:02:30.560122 fstat(3, {st_mode=S_IFREG|0755, st_size=7674920, ...}) = 0

you can see the details on how to reproduce this on ./scripts/linux_performance_setup.sh, use MAX_REQUESTS=1

few observations:

1. not sure where the search path is coming from, ie. why does it search in /usr/lib/swift/linux/tls/x86_64/x86_64 for example. its not in the RPATH
2. it visits the same location more than once eg /usr/lib/swift/linux/x86_64/libswiftCore.so

bash-4.2# objdump -p .build/release/SwiftAwsLambdaStringSample | egrep "RPATH|RUNPATH"
  RUNPATH              /usr/lib/swift/linux:/usr/lib/swift:$ORIGIN

[tomerd]

cc @jckarter @gottesmm

[compnerd]

@tomerd -Xlinker --enable-new-dtags -Xlinker -rpath -Xlinker ... should help reduce the search space.

Failure to pass in --enable-new-dtags will create a DT_RPATH entry rather than a DT_RUNPATH entry which not only deprecated, but makes debugging somethings much harder (DT_RUNPATH has a lower precedence than LD_LIBRARY_PATH and DT_RPATH, but DT_RPATH has a higher precedence than LD_LIBRARY_PATH).

[tomerd]

@compnerd not sure if I am doing it wrong but:

root@43072d5b1bd7:/code# swift build -c release -Xswiftc -g  -Xlinker --enable-new-dtags
[4/4] Linking SwiftAwsLambdaStringSample
root@43072d5b1bd7:/code# objdump -p .build/release/SwiftAwsLambdaStringSample | egrep "PATH"
  RUNPATH              /usr/lib/swift/linux:/usr/lib/swift:$ORIGIN
root@43072d5b1bd7:/code# swift build -c release -Xswiftc -g
[4/4] Linking SwiftAwsLambdaStringSample
root@43072d5b1bd7:/code# objdump -p .build/release/SwiftAwsLambdaStringSample | egrep "PATH"
  RUNPATH              /usr/lib/swift/linux:/usr/lib/swift:$ORIGIN

and still

04:11:44.129245 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.129881 stat("/usr/lib/swift/linux/tls/x86_64/x86_64", 0x7fff2a425720) = -1 ENOENT (No such file or directory)
04:11:44.130309 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.130901 stat("/usr/lib/swift/linux/tls/x86_64", 0x7fff2a425720) = -1 ENOENT (No such file or directory)
04:11:44.131284 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.131871 stat("/usr/lib/swift/linux/tls/x86_64", 0x7fff2a425720) = -1 ENOENT (No such file or directory)
04:11:44.132329 openat(AT_FDCWD, "/usr/lib/swift/linux/tls/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.132888 stat("/usr/lib/swift/linux/tls", 0x7fff2a425720) = -1 ENOENT (No such file or directory)
04:11:44.133422 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.133819 stat("/usr/lib/swift/linux/x86_64/x86_64", 0x7fff2a425720) = -1 ENOENT (No such file or directory)
04:11:44.134211 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.134825 stat("/usr/lib/swift/linux/x86_64", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
04:11:44.135295 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.135926 stat("/usr/lib/swift/linux/x86_64", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
04:11:44.136391 openat(AT_FDCWD, "/usr/lib/swift/linux/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = 3
04:11:44.136929 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
04:11:44.137445 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fec79039000
04:11:44.137788 fstat(3, {st_mode=S_IFREG|0755, st_size=93600, ...}) = 0
04:11:44.138284 mmap(NULL, 64824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fec79029000
04:11:44.138765 mprotect(0x7fec79036000, 4096, PROT_NONE) = 0
04:11:44.139095 mmap(0x7fec79037000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7fec79037000
04:11:44.139623 close(3)                = 0
04:11:44.139925 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.140274 openat(AT_FDCWD, "/usr/lib/swift/linux/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
04:11:44.140835 openat(AT_FDCWD, "/usr/lib/swift/linux/libswiftCore.so", O_RDONLY|O_CLOEXEC) = 3
04:11:44.141342 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
0

[compnerd]

@tomerd okay, digging a bit more into this, its not pretty - theres only so much we can do here, but there is one option that will work assuming you are okay with a distro specific approach (with knowledge that it comes at a price):

strace -e openat .build/x86_64-unknown-linux-gnu/release/SwiftAwsLambdaCodableSample
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libswiftDispatch.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libswiftSwiftOnoneSupport.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libdispatch.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libBlocksRuntime.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libFoundation.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/tls/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/tls/x86_64/libicui18nswift.so.65", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/tls/x86_64/libicuucswift.so.65", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/tls/x86_64/tls/x86_64/libicudataswift.so.65", O_RDONLY|O_CLOEXEC) = 3

[compnerd]

The problem comes down to libc trying to be clever and dynamically detecting things - but we can forcefully prevent this: LD_HWCAP_MASK=0 as an environment variable will prevent the loader from checking hardware capabilities.

The problem is ... tls is special. It is hardcoded right into the loader (cause apparently compatibility ... ugh). So, we just need to add a small hack to the toolchain - mkdir <snapshot>/usr/lib/swift/linux/tls; ln -s .. <snapshot>/usr/lib/swift/linux/tls/$(arch) should get most of the libraries.

For the extra cherry on top - distro specific behavior: link in paths to libc, libc, libdl, libutil, libstdc++.

That basically ensures that the paths are respected and we can easily distribute something in general.

The environment variable can be controlled by use of /etc/environment for docker and such, and can be a recommendation for others.

Note that this disables the ability to replace libraries with architecture optimizations. That is, this is tantamount to disabling the Haswell slice on MachO (x86_64h stripped system dylibs!)

[compnerd]

After a bit of discussion between @gottesmm @tomerd and I, we determined that setting the environment variable is a bit of a non-starter. Talking through this with Michael, I came to the realization that what he was proposing with hacking out the paths could actually work in a nicer way. The key thing is the order of the DT_RUNPATH. By pushing $ORIGIN to the top, we gain control over the paths and can control the packaging. This would need adjustments for the AWS Linux 2 layout, but, this tree works:

SwiftAwsLambdaCodableSample
tls
└── x86_64
    └── x86_64
        ├── libBlocksRuntime.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libBlocksRuntime.so
        ├── libc.so.6 -> /lib/x86_64-linux-gnu/libc.so.6
        ├── libdispatch.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libdispatch.so
        ├── libdl.so.2 -> /lib/x86_64-linux-gnu/libdl.so.2
        ├── libFoundation.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libFoundation.so
        ├── libgcc_s.so.1 -> /lib/x86_64-linux-gnu/libgcc_s.so.1
        ├── libm.so.6 -> /lib/x86_64-linux-gnu/libm.so.6
        ├── libpthread.so.0 -> /lib/x86_64-linux-gnu/libpthread.so.0
        ├── libswiftCore.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libswiftCore.so
        ├── libswiftDispatch.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libswiftDispatch.so
        ├── libswiftGlibc.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libswiftGlibc.so
        ├── libswiftSwiftOnoneSupport.so -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libswiftSwiftOnoneSupport.so
        ├── libutil.so.1 -> /lib/x86_64-linux-gnu/libutil.so.1
        └── tls
            └── x86_64
                └── x86_64
                    ├── libicudataswift.so.65 -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libicudataswift.so.65
                    ├── libicui18nswift.so.65 -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libicui18nswift.so.65
                    ├── libicuucswift.so.65 -> /home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux/libicuucswift.so.65
                    └── libstdc++.so.6 -> /usr/lib/x86_64-linux-gnu/libstdc++.so.6

$ strace -e openat ./SwiftAwsLambdaCodableSample
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libswiftGlibc.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libswiftCore.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libswiftSwiftOnoneSupport.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libswiftDispatch.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libdispatch.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libBlocksRuntime.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libFoundation.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/tls/x86_64/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/tls/x86_64/x86_64/libicui18nswift.so.65", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/tls/x86_64/x86_64/libicuucswift.so.65", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/compnerd/SourceCache/swift-aws-lambda/.build/x86_64-unknown-linux-gnu/debug/tls/x86_64/x86_64/tls/x86_64/x86_64/libicudataswift.so.65", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
openat(AT_FDCWD, "/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
openat(AT_FDCWD, "/etc/localtime", O_RDONLY|O_CLOEXEC) = 9

$ readelf -Wd SwiftAwsLambdaCodableSample | grep RUNPATH
 0x000000000000001d (RUNPATH)            Library runpath: [$ORIGIN:/home/compnerd/Downloads/swift-DEVELOPMENT-SNAPSHOT-2020-02-13-a-ubuntu18.04/usr/lib/swift/linux]