diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc new file mode 100644 index 0000000..c114db0 --- /dev/null +++ b/.github/docker/Dockerfile.glibc @@ -0,0 +1,11 @@ +ARG NODE_BUILD_IMAGE=node:16.20.1-bullseye +FROM $NODE_BUILD_IMAGE AS build + +WORKDIR /mongodb-client-encryption +COPY . . + +RUN node /mongodb-client-encryption/.github/scripts/libmongocrypt.mjs + +FROM scratch + +COPY --from=build /mongodb-client-encryption/prebuilds/ / diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 5846bd2..3566aa8 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -4,24 +4,42 @@ import fs from 'node:fs/promises'; import child_process from 'node:child_process'; import events from 'node:events'; import path from 'node:path'; +import https from 'node:https'; +import stream from 'node:stream/promises'; +import url from 'node:url'; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +/** Resolves to the root of this repository */ +function resolveRoot(...paths) { + return path.resolve(__dirname, '..', '..', ...paths); +} + +async function exists(fsPath) { + try { + await fs.access(fsPath); + return true; + } catch { + return false; + } +} async function parseArguments() { - const jsonImport = { [process.version.split('.').at(0) === 'v16' ? 'assert' : 'with']: { type: 'json' } }; - const pkg = (await import('../../package.json', jsonImport)).default; - const libmongocryptVersion = pkg['mongodb:libmongocrypt']; + const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8')); const options = { - url: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' }, - libversion: { short: 'l', type: 'string', default: libmongocryptVersion }, - clean: { short: 'c', type: 'boolean' }, - help: { short: 'h', type: 'boolean' } + gitURL: { short: 'u', type: 'string', default: 'https://github.com/mongodb/libmongocrypt.git' }, + libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] }, + clean: { short: 'c', type: 'boolean', default: false }, + build: { short: 'b', type: 'boolean', default: false }, + help: { short: 'h', type: 'boolean', default: false } }; const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false }); if (args.values.help) { console.log( - `${process.argv[1]} ${[...Object.keys(options)] + `${path.basename(process.argv[1])} ${[...Object.keys(options)] .filter(k => k !== 'help') .map(k => `[--${k}=${options[k].type}]`) .join(' ')}` @@ -30,15 +48,25 @@ async function parseArguments() { } return { - libmongocrypt: { url: args.values.url, ref: args.values.libversion }, - clean: args.values.clean + libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion }, + clean: args.values.clean, + build: args.values.build }; } /** `xtrace` style command runner, uses spawn so that stdio is inherited */ async function run(command, args = [], options = {}) { - console.error(`+ ${command} ${args.join(' ')}`, options.cwd ? `(in: ${options.cwd})` : ''); - await events.once(child_process.spawn(command, args, { stdio: 'inherit', ...options }), 'exit'); + const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`; + console.error(commandDetails); + const proc = child_process.spawn(command, args, { + shell: process.platform === 'win32', + stdio: 'inherit', + cwd: resolveRoot('.'), + ...options + }); + await events.once(proc, 'exit'); + + if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); } /** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */ @@ -46,30 +74,21 @@ function toFlags(object) { return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`); } -const args = await parseArguments(); -const libmongocryptRoot = path.resolve('_libmongocrypt'); - -const currentLibMongoCryptBranch = await fs.readFile(path.join(libmongocryptRoot, '.git', 'HEAD'), 'utf8').catch(() => '') -const libmongocryptAlreadyClonedAndCheckedOut = currentLibMongoCryptBranch.trim().endsWith(`r-${args.libmongocrypt.ref}`); - -if (args.clean || !libmongocryptAlreadyClonedAndCheckedOut) { - console.error('fetching libmongocrypt...', args.libmongocrypt); +export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) { + console.error('fetching libmongocrypt...', { url, ref }); await fs.rm(libmongocryptRoot, { recursive: true, force: true }); - await run('git', ['clone', args.libmongocrypt.url, libmongocryptRoot]); - await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot }); - await run('git', ['checkout', args.libmongocrypt.ref, '-b', `r-${args.libmongocrypt.ref}`], { cwd: libmongocryptRoot }); -} else { - console.error('libmongocrypt already up to date...', args.libmongocrypt); + await run('git', ['clone', url, libmongocryptRoot]); + if (ref !== 'latest') { + // Support "latest" as leaving the clone as-is so whatever the default branch name is works + await run('git', ['fetch', '--tags'], { cwd: libmongocryptRoot }); + await run('git', ['checkout', ref, '-b', `r-${ref}`], { cwd: libmongocryptRoot }); + } } -const libmongocryptBuiltVersion = await fs.readFile(path.join(libmongocryptRoot, 'VERSION_CURRENT'), 'utf8').catch(() => ''); -const libmongocryptAlreadyBuilt = libmongocryptBuiltVersion.trim() === args.libmongocrypt.ref; - -if (args.clean || !libmongocryptAlreadyBuilt) { - console.error('building libmongocrypt...\n', args); +export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { + console.error('building libmongocrypt...'); - const nodeDepsRoot = path.resolve('deps'); - const nodeBuildRoot = path.resolve(nodeDepsRoot, 'tmp', 'libmongocrypt-build'); + const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build'); await fs.rm(nodeBuildRoot, { recursive: true, force: true }); await fs.mkdir(nodeBuildRoot, { recursive: true }); @@ -115,11 +134,109 @@ if (args.clean || !libmongocryptAlreadyBuilt) { ? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' }) : []; - await run('cmake', [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], { cwd: nodeBuildRoot }); - await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { cwd: nodeBuildRoot }); -} else { - console.error('libmongocrypt already built...'); + await run( + 'cmake', + [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], + { cwd: nodeBuildRoot } + ); + await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { + cwd: nodeBuildRoot + }); +} + +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { + const downloadURL = + ref === 'latest' + ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' + : `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`; + + console.error('downloading libmongocrypt...', downloadURL); + const destination = resolveRoot(`_libmongocrypt-${ref}`); + + await fs.rm(destination, { recursive: true, force: true }); + await fs.mkdir(destination); + + const platformMatrix = { + ['darwin-arm64']: 'macos', + ['darwin-x64']: 'macos', + ['linux-ppc64']: 'rhel-71-ppc64el', + ['linux-s390x']: 'rhel72-zseries-test', + ['linux-arm64']: 'ubuntu1804-arm64', + ['linux-x64']: 'rhel-70-64-bit', + ['win32-x64']: 'windows-test' + }; + + const detectedPlatform = `${process.platform}-${process.arch}`; + const prebuild = platformMatrix[detectedPlatform]; + if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`); + + console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`); + + const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`]; + console.error(`+ tar ${unzipArgs.join(' ')}`); + const unzip = child_process.spawn('tar', unzipArgs, { + stdio: ['pipe', 'inherit'], + cwd: resolveRoot('.') + }); + + const [response] = await events.once(https.get(downloadURL), 'response'); + + const start = performance.now(); + await stream.pipeline(response, unzip.stdin); + const end = performance.now(); + + console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`); + + await fs.rm(nodeDepsRoot, { recursive: true, force: true }); + await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true }); + const currentPath = path.join(nodeDepsRoot, 'lib64'); + try { + await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib')); + } catch (error) { + console.error(`error renaming ${currentPath}: ${error.message}`); + } +} + +async function main() { + const { libmongocrypt, build, clean } = await parseArguments(); + + const nodeDepsDir = resolveRoot('deps'); + + if (build) { + const libmongocryptCloneDir = resolveRoot('_libmongocrypt'); + + const currentLibMongoCryptBranch = await fs + .readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8') + .catch(() => ''); + const isClonedAndCheckedOut = currentLibMongoCryptBranch + .trim() + .endsWith(`r-${libmongocrypt.ref}`); + + if (clean || !isClonedAndCheckedOut) { + await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt); + } + + const libmongocryptBuiltVersion = await fs + .readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8') + .catch(() => ''); + const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; + + if (clean || !isBuilt) { + await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir); + } + } else { + // Download + await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt); + } + + await fs.rm(resolveRoot('build'), { force: true, recursive: true }); + await fs.rm(resolveRoot('prebuilds'), { force: true, recursive: true }); + + // install with "ignore-scripts" so that we don't attempt to download a prebuild + await run('npm', ['install', '--ignore-scripts']); + // The prebuild command will make both a .node file in `./build` (local and CI testing will run on current code) + // it will also produce `./prebuilds/mongodb-client-encryption-vVERSION-napi-vNAPI_VERSION-OS-ARCH.tar.gz`. + await run('npm', ['run', 'prebuild']); } -await run('npm', ['install', '--ignore-scripts']); -await run('npm', ['run', 'rebuild'], { env: { ...process.env, BUILD_TYPE: 'static' } }); +await main(); diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86d30cc..1270be7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,24 +1,81 @@ on: push: branches: [main] + pull_request: + branches: [main] workflow_dispatch: {} name: build jobs: - build: + host_builds: + strategy: + matrix: + os: [macos-11, macos-latest, windows-2019] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Build ${{ matrix.os }} Prebuild + run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }} + shell: bash + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.os }} + path: prebuilds/ + if-no-files-found: 'error' + retention-days: 1 + compression-level: 0 + + container_builds: + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} runs-on: ubuntu-latest strategy: - matrix: - node: ['20.x'] # '16.x', '18.x', - name: Node.js ${{ matrix.node }} build + matrix: + linux_arch: [s390x, arm64, amd64] steps: - - uses: actions/setup-node@v4 + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker buildx build --platform linux/${{ matrix.linux_arch }} --output type=local,dest=./prebuilds,platform-split=false -f ./.github/docker/Dockerfile.glibc . + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 with: - node-version: ${{ matrix.node }} - cache: 'npm' - registry-url: 'https://registry.npmjs.org' - - run: npm install -g npm@latest - shell: bash - - run: node .github/scripts/libmongocrypt.mjs - shell: bash + name: build-linux-${{ matrix.linux_arch }} + path: prebuilds/ + if-no-files-found: 'error' + retention-days: 1 + compression-level: 0 + + collect: + needs: [host_builds, container_builds] + runs-on: ubunutu-latest + steps: + - uses: actions/download-artifact@v4 + + - name: Display structure of downloaded files + run: ls -R + + - id: upload + name: Upload all prebuilds + uses: actions/upload-artifact@v4 + with: + name: all-build + path: '*.tar.gz' + if-no-files-found: 'error' + retention-days: 1 + compression-level: 0 diff --git a/.gitignore b/.gitignore index 677c37f..6e615cf 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ xunit.xml lib prebuilds -_libmongocrypt/ +_libmongocrypt* diff --git a/package.json b/package.json index 76a75cd..090045a 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "check:clang-format": "clang-format --style=file:.clang-format --dry-run --Werror addon/*", "test": "mocha test", "prepare": "tsc", - "rebuild": "prebuild --compile", "prebuild": "prebuild --runtime napi --strip --verbose --all" }, "author": {