Skip to content

Commit 02ad34e

Browse files
bwbarrettbwbarrett
committed
jenkins: make autotools script more general
Refactor the script used by the Open MPI dist tarball builder to build the Autotools necessary for that tarball to be more general. The goal is to have a script usable not just for the dist job, but also for nightly tarballs and CI tests. The nightly tarballs requirement complicates the autotools script considerably, as the HWLOC nightly tarballs require an autotools install, but does not update the required Autotools versions in the dist script, so we allow setting component versions explicitly on the command line. Signed-off-by: Brian Barrett <bbarrett@amazon.com>
1 parent 6607dfc commit 02ad34e

File tree

2 files changed

+244
-62
lines changed

2 files changed

+244
-62
lines changed

‎jenkins/open-mpi-autotools-build.sh

Lines changed: 233 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,171 @@
11
#!/bin/bash
22
#
33
# Install (building if necessary) the autotools packages necessary for
4-
# building a particular version of Open MPI.
4+
# building a particular package.
55
#
66
# We build and save artifacts outside of the Jenkins workspace, which
77
# is a little odd, but allows us to persist builds across jobs (and
8-
# avoid any job-specific naming problems in paths). We will write the
9-
# autotools installations into $HOME/autotools-setup/ unless otherwise
10-
# instructed. If JENKINS_AGENT_HOME is set, we will use that instead
11-
# of $HOME.
8+
# avoid any job-specific naming problems in paths).
129
#
1310
# In addition to building and saving artifacts (including to S3 for
1411
# EC2 instances), this script can be called for every job to create a
15-
# simlink from the built autotools into $WORKSPACE/autotools-install.
16-
#
17-
# usage: ./open-mpi-autotools-build.sh <ompi tree>
12+
# simlink from the built autotools into <target>
1813
#
14+
# usage: ./open-mpi-autotools-build.sh
15+
# [-z <dist_script_for_versions]
16+
# [-r <autotools_root>
17+
# [-c <autoconf_version] [-m <automake version]
18+
# [-l <libtool_version] [-n <m4 version>]
19+
# [-f <flex_version>]
20+
21+
dist_script=
22+
debug=0
23+
autotools_root=
24+
target_link=
25+
dist_script_path="/dev/null"
26+
patch_file_directory=
27+
s3_build_path="s3://ompi-jenkins-config/autotools-builds"
28+
autotools_scratch_dir=
1929

20-
# be lazy...
21-
set -e
30+
usage() {
31+
echo "Usage: $0 -r <DIR> -t <LINK> [OPTION]..."
32+
echo "Build autotools necessary for OMPI-related project builds"
33+
echo ""
34+
echo "Mandatory Arguments:"
35+
echo " -r DIR Leave build artifacts in DIR and use DIR for building"
36+
echo " temporary artifacts. On non-ephemeral instances, DIR"
37+
echo " should be outside of a Jenkins workspace to avoid"
38+
echo " unnecessary rebuilding of autotools."
39+
echo " -t LINK Create symlnk LINK to the autotools build requested."
40+
echo " Unlike -r, this option frequently is in a Jenkins"
41+
echo " workspace, as it is ephemeral to a build."
42+
echo ""
43+
echo "Optional Arguments"
44+
echo " -d If specified, enable debug output."
45+
echo " -p DIR Directory to search for patch files. If a patch file"
46+
echo " named <package_name>-<version>.patch is found in DIR"
47+
echo " it will be applied to <package_name> before building."
48+
echo " -z FILE Pull required Autotools versions from an Open MPI like"
49+
echo " dist script. This option assumes certain variables are"
50+
echo " set in FILE. This option is exclusive with -c, -m, -l,"
51+
echo " -n, and -f options."
52+
echo " -{c,m,l,n,f} VERSION Use VERSION as the required version string for"
53+
echo " Autoconf, Automake, Libtool, M4, and Flex (respectively)."
54+
echo " If one of these arguments is set, all must be set. Either"
55+
echo " these arguments or -z must be specified."
56+
echo " -h Print this usage message and exit."
57+
}
2258

23-
ompi_tree=$1
24-
dist_script=${ompi_tree}/contrib/dist/make_dist_tarball
25-
s3_path="s3://ompi-jenkins-config/autotools-builds"
59+
debug_print() {
60+
if [[ $debug -ne 0 ]] ; then
61+
echo $1
62+
fi
63+
}
64+
65+
clean_scratch_dir() {
66+
if [[ -n "$autotools_scratch_dir" ]] ; then
67+
echo "cleaning $autotools_scratch_dir"
68+
rm -rf "$autotools_scratch_dir"
69+
fi
70+
}
2671

27-
if test ! -n "${JENKINS_AGENT_HOME}" ; then
28-
JENKINS_AGENT_HOME=${HOME}
72+
if [[ $# -eq 0 ]]; then
73+
usage
74+
exit 1
2975
fi
30-
autotools_root=${JENKINS_AGENT_HOME}/autotools-builds
31-
mkdir -p "${autotools_root}"
76+
while getopts ":dz:r:t:c:m:l:n:f:p:h" arg; do
77+
case $arg in
78+
d)
79+
debug=1
80+
;;
81+
p)
82+
patch_file_directory=${OPTARG}
83+
patch_file_directory=`realpath ${patch_file_directory}`
84+
;;
85+
r)
86+
autotools_root=${OPTARG}
87+
;;
88+
t)
89+
target_link=${OPTARG}
90+
;;
91+
z)
92+
dist_script=${OPTARG}
93+
;;
94+
c)
95+
AC_VERSION=${OPTARG}
96+
;;
97+
m)
98+
AM_VERSION=${OPTARG}
99+
;;
100+
l)
101+
LT_VERSION=${OPTARG}
102+
;;
103+
n)
104+
M4_VERSION=${OPTARG}
105+
;;
106+
f)
107+
FLEX_VERSION=${OPTARG}
108+
;;
109+
h)
110+
usage
111+
exit 0
112+
;;
113+
*)
114+
usage
115+
exit 1
116+
;;
117+
esac
118+
done
32119

33-
if test ! -r ${dist_script} ; then
34-
echo "Can not read ${dist_script}. Aborting."
120+
# command line checking
121+
if [[ -z "${target_link}" ]] ; then
122+
echo "-t <target_link> is a required option, but is not set."
123+
exit 1
124+
fi
125+
if [[ -z "${autotools_root}" ]] ; then
126+
echo "-r <build root> is a required option, but is not set."
35127
exit 1
36128
fi
37129

130+
trap clean_scratch_dir EXIT
131+
132+
mkdir -p "${autotools_root}"
133+
if [[ $? -ne 0 ]] ; then
134+
echo "Could not create directory ${autotools_root}. Cannot continue."
135+
exit 2
136+
fi
137+
138+
# If a dist script was specified, grab the version files from there.
139+
# Otherwise, expect all the versions to be explicitly specified.
140+
if [[ -n "${dist_script}" ]] ; then
141+
debug_print "Finding versions from dist script ${dist_script}"
142+
if [[ ! -r ${dist_script} ]] ; then
143+
echo "Cannot read ${dist_script}. Aborting."
144+
exit 1
145+
fi
146+
147+
for pkg in AC AM LT M4 FLEX ; do
148+
eval "${pkg}_VERSION=`sed -ne \"s/^${pkg}_TARGET_VERSION=\(.*\)/\1/p\" ${dist_script}`"
149+
eval "var=${pkg}_VERSION"
150+
if test -z "${var}" ; then
151+
echo "${pkg_VERSION} not set in ${dist_script}"
152+
exit 2
153+
fi
154+
done
155+
else
156+
for pkg in AC AM LT M4 FLEX ; do
157+
eval "var=\"\$${pkg}_VERSION\""
158+
if test -z "${var}" ; then
159+
echo "${pkg}_VERSION not set on command line"
160+
exit 1
161+
fi
162+
done
163+
fi
164+
for pkg in AC AM LT M4 FLEX ; do
165+
eval "var=\"\$${pkg}_VERSION\""
166+
debug_print "${pkg}_VERSION: $var"
167+
done
168+
38169
os=`uname -s`
39170
if test "${os}" = "Linux"; then
40171
eval "PLATFORM_ID=`sed -n 's/^ID=//p' /etc/os-release`"
@@ -45,87 +176,132 @@ else
45176
fi
46177

47178
if `echo ${NODE_NAME} | grep -q '^EC2'` ; then
48-
IS_EC2="yes"
179+
IS_EC2_JENKINS="yes"
49180
else
50-
IS_EC2="no"
181+
IS_EC2_JENKINS="no"
51182
fi
52183

53-
echo "==> Platform: $PLATFORM_ID"
54-
echo "==> Version: $VERSION_ID"
55-
echo "==> EC2: $IS_EC2"
56-
57-
for pkg in AC AM LT M4 FLEX; do
58-
eval "${pkg}_VERSION=`sed -ne \"s/^${pkg}_TARGET_VERSION=\(.*\)/\1/p\" ${dist_script}`"
59-
done
184+
debug_print "Platform: $PLATFORM_ID"
185+
debug_print "Version: $VERSION_ID"
186+
debug_print "EC2 Jenkins Worker: $IS_EC2_JENKINS"
60187

61188
version_string="${AC_VERSION}-${AM_VERSION}-${LT_VERSION}-${M4_VERSION}-${FLEX_VERSION}"
62189
tarball_name="autotools-${PLATFORM_ID}_${VERSION_ID}-${version_string}.tar.gz"
63190
autotools_install_short="autotools-install-${version_string}"
64191
autotools_install="${autotools_root}/${autotools_install_short}"
65-
echo "==> Version string: ${version_string}"
192+
debug_print "Version string: ${version_string}"
66193

67194
cd ${autotools_root}
68195

69-
if test ${IS_EC2} = "yes" && test ! -d ${autotools_install} ; then
70-
if aws s3 cp ${s3_path}/${tarball_name} . >& /dev/null ; then
71-
echo "==> Downloaded build from S3"
196+
if [[ ${IS_EC2_JENKINS} = "yes" && ! -d ${autotools_install} ]] ; then
197+
debug_print "Attempting to download cached build ${s3_build_path}/${tarball_name}"
198+
if aws s3 cp ${s3_build_path}/${tarball_name} . >& /dev/null ; then
199+
debug_print "Downloaded build from S3"
72200
tar xf ${tarball_name}
73201
rm ${tarball_name}
74202
fi
75203
fi
76204

77-
if test ! -d ${autotools_install} ; then
78-
echo "==> No build found ; building from scratch"
79-
80-
autotools_srcdir="${autotools_root}/autotools-src.$$"
205+
if [[ ! -d ${autotools_install} ]] ; then
206+
debug_print "==> No build found ; building from scratch"
81207

82-
mkdir -p ${autotools_srcdir}
83-
cd ${autotools_srcdir}
208+
autotools_scratch_dir=$(mktemp -d ${autotools_root}/autotools-src.XXXXXXXX)
209+
cd ${autotools_scratch_dir}
210+
debug_print "build dir: $autotools_scratch_dir"
84211

85212
export PATH=${autotools_install/bin}:${PATH}
86213
export LD_LIBRARY_PATH=${autotools_install}/lib:${LD_LIBRARY_PATH}
87214

88-
curl -fO http://ftp.gnu.org/gnu/autoconf/autoconf-${AC_VERSION}.tar.gz
215+
build_cleanup() {
216+
echo "Building autotools failed. Cleaning ${autotools_install}."
217+
if [[ -d "${autotools_install}" ]] ; then
218+
rm -rf "${autotools_install}"
219+
fi
220+
exit 5
221+
}
222+
trap build_cleanup ERR
223+
224+
# TODO: Error checking!
225+
curl -fLO http://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz
226+
tar xf m4-${M4_VERSION}.tar.gz
227+
patch_file="${patch_file_directory}/m4-${M4_VERSION}.patch"
228+
if [[ -r ${patch_file} ]] ; then
229+
debug_print "Appying patch m4-${M4_VERSION}.patch"
230+
(cd m4-${M4_VERSION} ; patch -p 1 < ${patch_file})
231+
else
232+
debug_print "patch m4-${M4_VERSION}.patch not found."
233+
fi
234+
(cd m4-${M4_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
235+
236+
curl -fLO http://ftp.gnu.org/gnu/autoconf/autoconf-${AC_VERSION}.tar.gz
89237
tar xf autoconf-${AC_VERSION}.tar.gz
238+
patch_file="${patch_file_directory}/autoconf-${AC_VERSION}.patch"
239+
if [[ -r ${patch_file} ]] ; then
240+
debug_print "Appying patch autoconf-${AC_VERSION}.patch"
241+
(cd autoconf-${AC_VERSION} ; patch -p 1 < ${patch_file})
242+
else
243+
debug_print "patch autoconf-${AC_VERSION}.patch not found."
244+
fi
90245
(cd autoconf-${AC_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
91246

92-
curl -fO http://ftp.gnu.org/gnu/automake/automake-${AM_VERSION}.tar.gz
247+
curl -fLO http://ftp.gnu.org/gnu/automake/automake-${AM_VERSION}.tar.gz
93248
tar xf automake-${AM_VERSION}.tar.gz
249+
patch_file="${patch_file_directory}/automake-${AM_VERSION}.patch"
250+
if [[ -r ${patch_file} ]] ; then
251+
debug_print "Appying patch automake-${AM_VERSION}.patch"
252+
(cd automake-${AM_VERSION} ; patch -p 1 < ${patch_file})
253+
else
254+
debug_print "patch automake-${AM_VERSION}.patch not found."
255+
fi
94256
(cd automake-${AM_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
95257

96-
curl -fO http://ftp.gnu.org/gnu/libtool/libtool-${LT_VERSION}.tar.gz
258+
curl -fLO http://ftp.gnu.org/gnu/libtool/libtool-${LT_VERSION}.tar.gz
97259
tar xf libtool-${LT_VERSION}.tar.gz
260+
patch_file="${patch_file_directory}/libtool-${LT_VERSION}.patch"
261+
if [[ -r ${patch_file} ]] ; then
262+
debug_print "Appying patch libtool-${LT_VERSION}.patch"
263+
(cd libtool-${LT_VERSION} ; patch -p 1 < ${patch_file})
264+
else
265+
debug_print "patch libtool-${LT_VERSION}.patch not found."
266+
fi
98267
(cd libtool-${LT_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
99268

100-
curl -fO http://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz
101-
tar xf m4-${M4_VERSION}.tar.gz
102-
(cd m4-${M4_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
103-
104269
# When flex moved from ftp.gnu.org to sourceforge to GitHub for
105270
# downloads, they dropped all the archive versions. Including the
106271
# one we say we require (sigh). So we archive that tarball
107272
# (stolen from a distro archive repository) for use. Hopefully,
108273
# one day, we will be able to update :).
109274
flex_tarball="flex-${FLEX_VERSION}.tar.gz"
110-
if ! curl -fO https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/${flex_tarball} ; then
111-
curl -fO https://download.open-mpi.org/archive/flex/${flex_tarball}
275+
if ! curl -fLO https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/${flex_tarball} ; then
276+
curl -fLO https://download.open-mpi.org/archive/flex/${flex_tarball}
112277
fi
113278
tar xf ${flex_tarball}
279+
patch_file="${patch_file_directory}/flex-${FLEX_VERSION}.patch"
280+
if [[ -r ${patch_file} ]] ; then
281+
debug_print "Appying patch flex-${FLEX_VERSION}.patch"
282+
(cd flex-${FLEX_VERSION} ; patch -p 1 < ${patch_file})
283+
else
284+
debug_print "patch flex-${FLEX_VERSION}.patch not found."
285+
fi
114286
(cd flex-${FLEX_VERSION} ; ./configure --prefix=${autotools_install} ; make install)
115287

116-
cd ${autotools_root}
288+
trap - ERR
117289

118-
# autotools_srcdir was unique to this process, so this is safe
119-
# even in a concurrent Jenkins jobs situation.
120-
rm -rf ${autotools_srcdir}
290+
cd ${autotools_root}
121291

122-
if test "$IS_EC2" = "yes" ; then
292+
if test "$IS_EC2_JENKINS" = "yes" ; then
123293
echo "==> Archiving build to S3"
124-
tar czf ${tarball_name} ${autotools_install_short}
125-
aws s3 cp ${tarball_name} ${s3_path}/${tarball_name}
126-
rm ${tarball_name}
294+
tar czf "${tarball_name}" "${autotools_install_short}"
295+
aws s3 cp "${tarball_name}" "${s3_build_path}/${tarball_name}"
296+
rm "${tarball_name}"
127297
fi
128298
fi
129299

130-
echo "==> Symlinking ${autotools_install} to ${WORKSPACE}/autotools-install"
131-
ln -s ${autotools_install} ${WORKSPACE}/autotools-install
300+
if [[ -e "${target_link}" && ! -L "${target_link}" ]] ; then
301+
echo "${target_link} exists but is not a symlink. Cowardly not creating link."
302+
exit 99
303+
else
304+
echo "==> Symlinking ${autotools_install} to ${target_link}"
305+
rm -f "${target_link}"
306+
ln -s "${autotools_install}" "${target_link}"
307+
fi

‎jenkins/open-mpi.dist.create-tarball.groovy

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,17 @@ node(rpm_builder) {
3737
}
3838

3939
stage('Installing Dependencies') {
40-
// because we build tags or hashes, not just branch heads,
41-
// there's not a great way to do the branch -> Autotools
42-
// version matching. So instead grab them from the
43-
// make_dist_tarball script
44-
sh '/bin/bash ompi-scripts/jenkins/open-mpi-autotools-build.sh ompi'
40+
// Build Autotools based on the dist script found in the active build
41+
// branch.
42+
43+
// The tarball builder jobs only ever run on EC2 instances, so it
44+
// should always be safe to use $HOME as the location for autotools
45+
// build artifacts, as the EC2 instances always have a dedicated
46+
// Jenkins user. If you are copying this code for another job (like
47+
// a CI test), please be careful about writing into $HOME. The Cray
48+
// builders in particular are using shared accounts.
49+
50+
sh "/bin/bash ompi-scripts/jenkins/open-mpi-autotools-build.sh -d -p ompi-scripts/jenkins/autotools-patches -t ${WORKSPACE}/autotools-install -r ${env.HOME}/autotools-builds -z ${WORKSPACE}/ompi/contrib/dist/make_dist_tarball"
4551
}
4652

4753
// Build the initial tarball, verify that the resulting tarball

0 commit comments

Comments
 (0)