[Build] Support publishing maven artifacts directly to maven central (#2397)

* Adjusted nmcp compatibility with scala variants
* Fixed dist maven publishing
* Fix pom generation to ignore embedded dependencies
* Minor cleanup
* Build maven aggregation zip as part of dra build

Author: breskeby

.buildkite/dra.sh

@@ -45,7 +45,7 @@ mkdir localRepo
 wget --quiet "https://artifacts-$DRA_WORKFLOW.elastic.co/elasticsearch/${ES_BUILD_ID}/maven/org/elasticsearch/gradle/build-tools/${HADOOP_VERSION}${VERSION_SUFFIX}/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar" \
   -O "localRepo/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar"
 
-./gradlew -S -PlocalRepo=true "${BUILD_ARGS[@]}" -Dorg.gradle.warning.mode=summary -Dcsv="$WORKSPACE/build/distributions/dependencies-${HADOOP_VERSION}${VERSION_SUFFIX}.csv" :dist:generateDependenciesReport distribution
+./gradlew -S -PlocalRepo=true "${BUILD_ARGS[@]}" -Dorg.gradle.warning.mode=summary -Dcsv="$WORKSPACE/build/distributions/dependencies-${HADOOP_VERSION}${VERSION_SUFFIX}.csv" :dist:generateDependenciesReport distribution zipAggregation
 
 # Allow other users access to read the artifacts so they are readable in the container
 find "$WORKSPACE" -type f -path "*/build/distributions/*" -exec chmod a+r {} \;

build.gradle

@@ -1,6 +1,7 @@
 import org.elasticsearch.hadoop.gradle.buildtools.ConcatFilesTask
 import java.lang.management.ManagementFactory;
 import java.time.LocalDateTime;
+import org.elasticsearch.gradle.VersionProperties
 
 import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
@@ -11,9 +12,23 @@ import java.time.LocalDateTime
 description = 'Elasticsearch for Apache Hadoop'
 
 apply plugin: 'es.hadoop.build.root'
+apply plugin: 'com.gradleup.nmcp.aggregation'
 
 defaultTasks 'build'
 
+dependencies {
+    nmcpAggregation(project(":dist"))
+    nmcpAggregation(project(":elasticsearch-hadoop-mr"))
+    nmcpAggregation(project(":elasticsearch-hadoop-hive"))
+    nmcpAggregation(project(":elasticsearch-spark-30"))
+}
+
+tasks.named('zipAggregation').configure {
+    archiveFileName.unset();
+    archiveBaseName.set("elasticsearch-maven-aggregration")
+    archiveVersion.set(VersionProperties.elasticsearch)
+}
+
 allprojects {
     group = "org.elasticsearch"
     tasks.withType(AbstractCopyTask) {

buildSrc/build.gradle

@@ -72,6 +72,7 @@ dependencies {
     // Required for dependency licenses task
     implementation 'org.apache.rat:apache-rat:0.11'
     implementation 'commons-codec:commons-codec:1.12'
+    implementation 'com.gradleup.nmcp:nmcp:0.1.4'
 
     if (localRepo) {
         implementation name: "build-tools-${buildToolsVersion}"

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy

@@ -567,6 +567,7 @@ class BuildPlugin implements Plugin<Project>  {
 
     private void configureMaven(Project project) {
         project.getPluginManager().apply("maven-publish")
+        project.getPluginManager().apply("com.gradleup.nmcp")
 
         // Configure Maven publication
         project.publishing {
@@ -587,13 +588,6 @@ class BuildPlugin implements Plugin<Project>  {
         // Configure Maven Pom
         configurePom(project, project.publishing.publications.main)
 
-        // Disable the publishing tasks since we only need the pom generation tasks.
-        // If we are working with a project that has a scala variant (see below), we need to modify the pom's
-        // artifact id which the publish task does not like (it fails validation when run).
-        project.getTasks().withType(PublishToMavenRepository) { PublishToMavenRepository m ->
-            m.enabled = false
-        }
-
         // Configure Scala Variants if present
         project.getPlugins().withType(SparkVariantPlugin).whenPluginAdded {
             // Publishing gets weird when you introduce variants into the project.
@@ -608,7 +602,8 @@ class BuildPlugin implements Plugin<Project>  {
 
             // Main variant needs the least configuration on its own, since it is the default publication created above.
             sparkVariants.defaultVariant { SparkVariant variant ->
-                updateVariantPomLocationAndArtifactId(project, project.publishing.publications.main, variant)
+                project.publishing.publications.main.setAlias(true)
+                updateVariantArtifactId(project, project.publishing.publications.main, variant)
             }
 
             // For each spark variant added, we need to do a few things:
@@ -659,7 +654,7 @@ class BuildPlugin implements Plugin<Project>  {
                             suppressAllPomMetadataWarnings() // We get it. Gradle metadata is better than Maven Poms
                         }
                         configurePom(project, variantPublication)
-                        updateVariantPomLocationAndArtifactId(project, variantPublication, variant)
+                        updateVariantArtifactId(project, variantPublication, variant)
                     }
                 }
             }
@@ -672,14 +667,6 @@ class BuildPlugin implements Plugin<Project>  {
     }
 
     private static void configurePom(Project project, MavenPublication publication) {
-        // Set the pom's destination to the distribution directory
-        project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom ->
-            if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") {
-                BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class);
-                pom.destination = project.provider({"${project.buildDir}/distributions/${baseExtension.archivesName.get()}-${project.getVersion()}.pom"})
-            }
-        }
-
         // add all items necessary for publication
         Provider<String> descriptionProvider = project.provider({ project.getDescription() })
         MavenPom pom = publication.getPom()
@@ -722,7 +709,8 @@ class BuildPlugin implements Plugin<Project>  {
                     while (dependenciesIterator.hasNext()) {
                         Node dependencyNode = dependenciesIterator.next()
                         String artifact = dependencyNode.get("artifactId").text()
-                        if (artifact == dependency.getName()) {
+                        // handle scala variants by splitting via "_" and checking the first part
+                        if (artifact =~ dependency.getName().split('_')[0]) {
                             dependenciesIterator.remove()
                             break
                         }
@@ -732,23 +720,11 @@ class BuildPlugin implements Plugin<Project>  {
         }
     }
 
-    private static void updateVariantPomLocationAndArtifactId(Project project, MavenPublication publication, SparkVariant variant) {
+    private static void updateVariantArtifactId(Project project, MavenPublication publication, SparkVariant variant) {
         // Add variant classifier to the pom file name if required
-        String classifier = variant.shouldClassifySparkVersion() && variant.isDefaultVariant() == false ? "-${variant.getName()}" : ''
         BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class);
-        String filename = "${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}-${project.getVersion()}${classifier}"
-        // Fix the pom name
-        project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom ->
-            if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") {
-                pom.destination = project.provider({"${project.buildDir}/distributions/${filename}.pom"})
-            }
-        }
-        // Fix the artifactId. Note: The publishing task does not like this happening. Hence it is disabled.
-        publication.getPom().withXml { XmlProvider xml ->
-            Node root = xml.asNode()
-            Node artifactId = (root.get('artifactId') as NodeList).get(0) as Node
-            artifactId.setValue("${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}")
-        }
+        // Fix the artifact id
+        publication.setArtifactId("${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}")
     }
 
     /**

dist/build.gradle

@@ -1,3 +1,4 @@
+import jdk.internal.foreign.abi.Binding
 import org.elasticsearch.hadoop.gradle.buildtools.ConcatFilesTask
 import org.elasticsearch.hadoop.gradle.buildtools.DependenciesInfoTask
 import org.elasticsearch.hadoop.gradle.buildtools.DependencyLicensesTask
@@ -132,26 +133,6 @@ javadoc {
     }
 }
 
-publishing {
-    publications {
-        main {
-            getPom().withXml { XmlProvider xml ->
-                Node root = xml.asNode()
-
-                // add clojars repo to pom
-                Node repositories = root.appendNode('repositories')
-                Node repository = repositories.appendNode('repository')
-                repository.appendNode('id', 'clojars.org')
-                repository.appendNode('url', 'https://clojars.org/repo')
-                BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class)
-
-                // Correct the artifact Id, otherwise it is listed as 'dist'
-                root.get('artifactId').get(0).setValue(baseExtension.archivesName.get())
-            }
-        }
-    }
-}
-
 // Name of the directory under the root of the zip file that will contain the zip contents
 String zipContentDir = "elasticsearch-hadoop-${project.version}"
 
@@ -179,8 +160,28 @@ task('distZip', type: Zip) {
 
 distribution {
     dependsOn(distZip)
+
+}
+
+
+publishing {
+    publications {
+        main {
+            artifact tasks.named('distZip')
+            getPom().withXml { XmlProvider xml ->
+                Node root = xml.asNode()
+
+                // add clojars repo to pom
+                Node repositories = root.appendNode('repositories')
+                Node repository = repositories.appendNode('repository')
+                repository.appendNode('id', 'clojars.org')
+                repository.appendNode('url', 'https://clojars.org/repo')
+            }
+        }
+    }
 }
 
+
 // Add a task in the root project that collects all the dependencyReport data for each project
 // Concatenates the dependencies CSV files into a single file
 task generateDependenciesReport(type: ConcatFilesTask) { concatDepsTask ->
@@ -197,3 +198,15 @@ task generateDependenciesReport(type: ConcatFilesTask) { concatDepsTask ->
 project.tasks.named('dependencyLicenses', DependencyLicensesTask) {
     it.dependencies = project.configurations.licenseChecks
 }
+
+
+tasks.register('copyPoms', Copy) {
+    BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class);
+    from(tasks.named('generatePomFileForMainPublication'))
+    into(new File(project.buildDir, 'distributions'))
+    rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom"
+}
+
+tasks.named('distribution').configure {
+    dependsOn 'copyPoms'
+}

hive/build.gradle

@@ -88,3 +88,15 @@ itestJar {
         include "META-INF/services/*"
     }
 }
+
+
+tasks.register('copyPoms', Copy) {
+    BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class);
+    from(tasks.named('generatePomFileForMainPublication'))
+    into(new File(project.buildDir, 'distributions'))
+    rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom"
+}
+
+tasks.named('distribution').configure {
+    dependsOn 'copyPoms'
+}

mr/build.gradle

@@ -137,3 +137,14 @@ eclipse.classpath.file {
         }
     }
 }
+
+tasks.register('copyPoms', Copy) {
+    BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class);
+    from(tasks.named('generatePomFileForMainPublication'))
+    into(new File(project.buildDir, 'distributions'))
+    rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom"
+}
+
+tasks.named('distribution').configure {
+    dependsOn 'copyPoms'
+}

spark/sql-30/build.gradle

@@ -189,3 +189,18 @@ sparkVariants {
         }
     }
 }
+
+
+tasks.register('copyPoms', Copy) {
+    from(tasks.named('generatePomFileForMainPublication')) {
+        rename 'pom-default.xml', "elasticsearch-spark-30_2.13-${project.getVersion()}.pom"
+    }
+    from(tasks.named('generatePomFileForSpark30scala212Publication')) {
+        rename 'pom-default.xml', "elasticsearch-spark-30_2.12-${project.getVersion()}.pom"
+    }
+    into(new File(project.buildDir, 'distributions'))
+}
+
+tasks.named('distribution').configure {
+    dependsOn 'copyPoms'
+}