Add release CI pipeline

.buildkite/pipeline.yml

@@ -1,5 +1,5 @@
 steps:
-  - label: ":java: :elasticsearch: Elasticsearch Java API client - {{matrix.workflow}}"
+  - label: ":java: :elasticsearch: Elasticsearch Java API client DRA - {{matrix.workflow}}"
     agents:
       provider: "gcp"
     branches: [ "main", "7.17", "8.17", "8.18", "8.x", "9.0" ]
@@ -8,4 +8,4 @@ steps:
         workflow:
           - "snapshot"
           - "staging"
-    command: ".ci/release.sh {{matrix.workflow}}"
+    command: ".ci/release_dra.sh {{matrix.workflow}}"

.buildkite/release_central.yaml

@@ -0,0 +1,14 @@
+agents:
+  provider: "gcp"
+
+steps:
+  - label: ":java: :elasticsearch: Elasticsearch Java API client - Release"
+    key: "release"
+    command: ".ci/release_central.sh"
+    artifact_paths:
+      - ".ci/output/repository/**/*"
+
+#notify:
+#  - slack: "#devtools-notify"
+#    # skip slack messages if no failures and dry-run mode
+#    if: 'build.state != "passed" && build.env("dry_run") == "false"'

.ci/Dockerfile

@@ -50,6 +50,9 @@ COPY --chown=$BUILDER_UID:$BUILDER_GID config ./config/
 COPY --chown=$BUILDER_UID:$BUILDER_GID java-client/build.gradle.kts ./java-client/
 RUN ./gradlew resolveDependencies
 
+# Note: trailing '*' avoids failing if the file doesn't exist
+COPY --chown=$BUILDER_UID:$BUILDER_GID gradle.properties* ./
+
 # Build artifacts. Expects these mounted directories:
 # /elasticsearch-java/.git - git index (read-only)
 # /elasticsearch-java/java-client/src - source files (read-only)

.ci/configure_signing.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#see https://central.sonatype.org/publish/publish-gradle/#distributing-your-public-key
+
+set -euo pipefail
+
+if grep -sq "signing.keyId" gradle.properties; then
+  # Keys already present
+  exit 0
+fi
+
+mkdir -p /tmp
+keyring_file="/tmp/keyring.gpg"
+
+vault_path="kv/ci-shared/release-eng/team-release-secrets/elasticsearch-java"
+
+vault kv get --field="keyring" $vault_path/gpg | base64 -d > $keyring_file
+## NOTE: passphase is the name of the field.
+signing_password=$(vault kv get --field="passphase" $vault_path/gpg)
+signing_key=$(vault kv get --field="key_id" $vault_path/gpg)
+
+maven_username=$(vault kv get --field="username" $vault_path/maven_central)
+maven_password=$(vault kv get --field="password" $vault_path/maven_central)
+
+cat >> gradle.properties <<EOF
+signing.keyId=${signing_key: -8}
+signing.password=${signing_password}
+signing.secretKeyRingFile=${keyring_file}
+
+ossrhUsername=${maven_username}
+ossrhPassword=${maven_password}
+EOF
+

.ci/make.sh

@@ -42,7 +42,9 @@
 # Bootstrap
 # ------------------------------------------------------- #
 
-script_path=$(dirname "$(realpath -s "$0")")
+set -euo pipefail
+
+script_path=$(dirname "$(realpath "$0")")
 repo=$(realpath "$script_path/../")
 
 # shellcheck disable=SC1090
@@ -51,7 +53,6 @@ TASK=$1
 TASK_ARGS=()
 VERSION=$2
 STACK_VERSION=$VERSION
-set -euo pipefail
 
 product="elastic/elasticsearch-java"
 output_folder=".ci/output"
@@ -81,6 +82,15 @@ case $CMD in
             exit 1
         fi
         echo -e "\033[36;1mTARGET: assemble artefact $VERSION\033[0m"
+        TASK=assemble
+        TASK_ARGS=("$VERSION" "$output_folder")
+        ;;
+    release)
+        if [ -z "$VERSION" ]; then
+            echo -e "\033[31;1mTARGET: release -> missing version parameter\033[0m"
+            exit 1
+        fi
+        echo -e "\033[36;1mTARGET: release artefact $VERSION\033[0m"
         TASK=release
         TASK_ARGS=("$VERSION" "$output_folder")
         ;;
@@ -159,7 +169,7 @@ if [[ "$CMD" == "assemble" ]]; then
   fi
 
   build_image
-    echo -e "\033[34;1mINFO:\033[0m Building version ${assemble_version}\033[0m"
+  echo -e "\033[34;1mINFO:\033[0m Building version ${assemble_version}\033[0m"
   docker run --rm --env VERSION=$assemble_version -u "$(id -u)" \
     $git_mount $src_mount $output_mount \
     $docker_image \
@@ -176,6 +186,25 @@ if [[ "$CMD" == "assemble" ]]; then
 	fi
 fi
 
+if [[ "$CMD" == "release" ]]; then
+  rm -rf .ci/output/repository
+  build_image
+  echo -e "\033[34;1mINFO:\033[0m Building version ${VERSION}\033[0m"
+
+  if [[ "$DRY_RUN" = "false" ]]; then
+    echo "Dry run: building and publishing to the local repository"
+    gradle_task="java-client:publishAllPublicationsToBuildRepository"
+  else
+    echo "Releasing to Maven (pretending for now)"
+    gradle_task="java-client:publishAllPublicationsToBuildRepository"
+    #gradle_task="java-client:publishAllPublicationsToMavenCentralSnapshotRepository"
+  fi
+  docker run --rm --env VERSION=$VERSION -u "$(id -u)" \
+    $git_mount $src_mount $output_mount \
+    $docker_image \
+    $gradle_task
+fi
+
 if [[ "$CMD" == "bump" ]]; then
   echo $VERSION > config/version.txt
 fi

.ci/release_central.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#see https://central.sonatype.org/publish/publish-gradle/#distributing-your-public-key
+
+set -euo pipefail
+
+.ci/configure_signing.sh
+
+.ci/make.sh release $VERSION

.github/workflows/release_central.yml

@@ -0,0 +1,56 @@
+name: "Release to Maven Central"
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Branch or tag ref to run the workflow on"
+        required: true
+      version:
+        description: "The version to release. Must start with the one in config/version.txt"
+        required: true
+      dry_run:
+        description: Used to test other workflow steps, does not publish to Maven Central.
+        type: boolean
+        required: true
+        default: false
+
+env:
+  BRANCH: ${{ inputs.branch }}
+  VERSION: ${{ inputs.version }}
+  DRY_RUN: ${{ inputs.dry_run }}
+
+jobs:
+  validate-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: '1'
+      - name: Validate version
+        shell: bash
+        run: |
+          repo_version="$(cat config/version.txt)"
+          if [[ ! "$VERSION" = $repo_version* ]]; then
+            echo "Workflow version ($VERSION) and config/version.txt ($repo_version) do not match."
+            exit 1
+          fi
+
+  maven-central-deploy:
+    name: "Deploy to Maven Central (Buildkite)"
+    runs-on: ubuntu-latest
+    needs:
+      - validate-version
+    steps:
+      - id: buildkite-run
+        uses: elastic/oblt-actions/buildkite/run@v1
+        with:
+          pipeline: "elasticsearch-java-release"
+          wait-for: true
+          token: ${{ secrets.BUILDKITE_TOKEN }}
+          branch: ${{ inputs.branch }}
+          env-vars: |
+            DRY_RUN=${{ inputs.dry_run }}
+            VERSION=${{ inputs.version }}

build.gradle.kts

@@ -25,7 +25,8 @@ allprojects {
     if (System.getenv("VERSION")==null) {
         version = (File(project.rootDir, "config/version.txt").readText().trim() + "-SNAPSHOT")
     }
-    else if (System.getenv("VERSION").endsWith("-SNAPSHOT")) {
+    else if (System.getenv("VERSION").contains("-")) {
+        // Either SNAPSHOT or a version qualifier included in $VERSION for ad-hoc releases
         version = System.getenv("VERSION")
     }
     else {

catalog-info.yaml

@@ -26,3 +26,39 @@ spec:
         devtools-team: {}
         everyone:
           access_level: READ_ONLY
+
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-java-release
+  description: Buildkite Release pipeline for elasticsearch-java
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-java-release
+  tags:
+    - buildkite
+    - gpg-sign
+    - maven-central
+spec:
+  type: buildkite-pipeline
+  owner: group:devtools-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: Elasticsearch Java Client Release
+      name: elasticsearch-java-release
+    spec:
+      repository: elastic/elasticsearch-java
+      pipeline_file: ".buildkite/release_central.yml"
+      provider_settings:
+        trigger_mode: none
+      teams:
+        devtools-team:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

java-client/build.gradle.kts

@@ -27,6 +27,7 @@ plugins {
     `java-library`
     checkstyle
     `maven-publish`
+    signing
     id("com.github.jk1.dependency-license-report") version "2.2"
     id("de.thetaphi.forbiddenapis") version "3.4"
 }
@@ -115,6 +116,12 @@ tasks.withType<Javadoc> {
     }
 }
 
+signing {
+    // Only sign if a key has been configured in gradle.properties
+    isRequired = providers.gradleProperty("signing.keyId").isPresent
+    sign(publishing.publications)
+}
+
 publishing {
     repositories {
         maven {
@@ -128,6 +135,15 @@ publishing {
             name = "Build"
             url = uri("${rootProject.buildDir}/repository")
         }
+
+        maven {
+            name = "MavenCentralSnapshot"
+            url = uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")
+            credentials {
+                username = providers.gradleProperty("ossrhUsername").getOrNull()
+                password = providers.gradleProperty("ossrhPassword").getOrNull()
+            }
+        }
     }
 
     publications {