Expose search shard statistics in search hits.

Original Pull Request #2806
Closes #2605

Author: puppylpg

src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java

@@ -49,6 +49,7 @@
  * {@link org.springframework.data.elasticsearch.core.document.Document}
  *
  * @author Peter-Josef Meisch
+ * @author Haibo Liu
  * @since 4.4
  */
 final class DocumentAdapters {
@@ -73,7 +74,7 @@ public static SearchDocument from(Hit<?> hit, JsonpMapper jsonpMapper) {
 		Map<String, SearchDocumentResponse> innerHits = new LinkedHashMap<>();
 		hit.innerHits().forEach((name, innerHitsResult) -> {
 			// noinspection ReturnOfNull
-			innerHits.put(name, SearchDocumentResponseBuilder.from(innerHitsResult.hits(), null, null, null, null,
+			innerHits.put(name, SearchDocumentResponseBuilder.from(innerHitsResult.hits(), null, null, null, null, null,
 					searchDocument -> null, jsonpMapper));
 		});
 

src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java

@@ -541,7 +541,7 @@ private long timeToLong(Time time) {
 	}
 
 	@Nullable
-	private static ElasticsearchErrorCause toErrorCause(@Nullable ErrorCause errorCause) {
+	static ElasticsearchErrorCause toErrorCause(@Nullable ErrorCause errorCause) {
 
 		if (errorCause != null) {
 			return new ElasticsearchErrorCause( //

src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.data.elasticsearch.client.elc;
 
+import co.elastic.clients.elasticsearch._types.ShardFailure;
+import co.elastic.clients.elasticsearch._types.ShardStatistics;
 import co.elastic.clients.elasticsearch._types.aggregations.Aggregate;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
 import co.elastic.clients.elasticsearch.core.SearchTemplateResponse;
@@ -36,6 +38,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.springframework.data.elasticsearch.core.SearchShardStatistics;
 import org.springframework.data.elasticsearch.core.TotalHitsRelation;
 import org.springframework.data.elasticsearch.core.document.SearchDocument;
 import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
@@ -52,6 +55,7 @@
  * Factory class to create {@link SearchDocumentResponse} instances.
  *
  * @author Peter-Josef Meisch
+ * @author Haibo Liu
  * @since 4.4
  */
 class SearchDocumentResponseBuilder {
@@ -78,8 +82,9 @@ public static <T> SearchDocumentResponse from(ResponseBody<EntityAsMap> response
 		Map<String, Aggregate> aggregations = responseBody.aggregations();
 		Map<String, List<Suggestion<EntityAsMap>>> suggest = responseBody.suggest();
 		var pointInTimeId = responseBody.pitId();
+		var shards = responseBody.shards();
 
-		return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper);
+		return from(hitsMetadata, shards, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper);
 	}
 
 	/**
@@ -98,13 +103,14 @@ public static <T> SearchDocumentResponse from(SearchTemplateResponse<EntityAsMap
 		Assert.notNull(entityCreator, "entityCreator must not be null");
 		Assert.notNull(jsonpMapper, "jsonpMapper must not be null");
 
+		var shards = response.shards();
 		var hitsMetadata = response.hits();
 		var scrollId = response.scrollId();
 		var aggregations = response.aggregations();
 		var suggest = response.suggest();
 		var pointInTimeId = response.pitId();
 
-		return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper);
+		return from(hitsMetadata, shards, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper);
 	}
 
 	/**
@@ -120,8 +126,8 @@ public static <T> SearchDocumentResponse from(SearchTemplateResponse<EntityAsMap
 	 * @param jsonpMapper to map JsonData objects
 	 * @return the {@link SearchDocumentResponse}
 	 */
-	public static <T> SearchDocumentResponse from(HitsMetadata<?> hitsMetadata, @Nullable String scrollId,
-			@Nullable String pointInTimeId, @Nullable Map<String, Aggregate> aggregations,
+	public static <T> SearchDocumentResponse from(HitsMetadata<?> hitsMetadata, @Nullable ShardStatistics shards,
+			@Nullable String scrollId, @Nullable String pointInTimeId, @Nullable Map<String, Aggregate> aggregations,
 			Map<String, List<Suggestion<EntityAsMap>>> suggestES, SearchDocumentResponse.EntityCreator<T> entityCreator,
 			JsonpMapper jsonpMapper) {
 
@@ -155,8 +161,19 @@ public static <T> SearchDocumentResponse from(HitsMetadata<?> hitsMetadata, @Nul
 
 		Suggest suggest = suggestFrom(suggestES, entityCreator);
 
+		SearchShardStatistics shardStatistics = shards != null ? shardsFrom(shards) : null;
+
 		return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, pointInTimeId, searchDocuments,
-				aggregationsContainer, suggest);
+				aggregationsContainer, suggest, shardStatistics);
+	}
+
+	private static SearchShardStatistics shardsFrom(ShardStatistics shards) {
+		List<ShardFailure> failures = shards.failures();
+		List<SearchShardStatistics.Failure> searchFailures = failures.stream()
+				.map(f -> SearchShardStatistics.Failure.of(f.index(), f.node(), f.status(), f.shard(), null,
+						ResponseConverter.toErrorCause(f.reason())))
+				.toList();
+		return SearchShardStatistics.of(shards.failed(), shards.successful(), shards.total(), shards.skipped(), searchFailures);
 	}
 
 	@Nullable

src/main/java/org/springframework/data/elasticsearch/core/SearchHitMapping.java

@@ -46,6 +46,7 @@
  * @author Matt Gilene
  * @author Sascha Woo
  * @author Jakob Hoeper
+ * @author Haibo Liu
  * @since 4.0
  */
 public class SearchHitMapping<T> {
@@ -84,6 +85,7 @@ private SearchHitsImpl<T> mapHitsFromResponse(SearchDocumentResponse searchDocum
 				"Count of documents must match the count of entities");
 
 		long totalHits = searchDocumentResponse.getTotalHits();
+		SearchShardStatistics shardStatistics = searchDocumentResponse.getSearchShardStatistics();
 		float maxScore = searchDocumentResponse.getMaxScore();
 		String scrollId = searchDocumentResponse.getScrollId();
 		String pointInTimeId = searchDocumentResponse.getPointInTimeId();
@@ -103,7 +105,7 @@ private SearchHitsImpl<T> mapHitsFromResponse(SearchDocumentResponse searchDocum
 		mapHitsInCompletionSuggestion(suggest);
 
 		return new SearchHitsImpl<>(totalHits, totalHitsRelation, maxScore, scrollId, pointInTimeId, searchHits,
-				aggregations, suggest);
+				aggregations, suggest, shardStatistics);
 	}
 
 	@SuppressWarnings("unchecked")
@@ -240,7 +242,8 @@ private SearchHits<?> mapInnerDocuments(SearchHits<SearchDocument> searchHits, C
 						searchHits.getPointInTimeId(), //
 						convertedSearchHits, //
 						searchHits.getAggregations(), //
-						searchHits.getSuggest());
+						searchHits.getSuggest(),
+						searchHits.getSearchShardStatistics());
 			}
 		} catch (Exception e) {
 			throw new UncategorizedElasticsearchException("Unable to convert inner hits.", e);

src/main/java/org/springframework/data/elasticsearch/core/SearchHits.java

@@ -27,6 +27,7 @@
  *
  * @param <T> the result data class.
  * @author Sascha Woo
+ * @author Haibo Liu
  * @since 4.0
  */
 public interface SearchHits<T> extends Streamable<SearchHit<T>> {
@@ -108,4 +109,10 @@ default Iterator<SearchHit<T>> iterator() {
 	 */
 	@Nullable
 	String getPointInTimeId();
+
+	/**
+	 * @return shard statistics for the search hit.
+	 */
+	@Nullable
+	SearchShardStatistics getSearchShardStatistics();
 }

src/main/java/org/springframework/data/elasticsearch/core/SearchHitsImpl.java

@@ -29,6 +29,7 @@
  * @param <T> the result data class.
  * @author Peter-Josef Meisch
  * @author Sascha Woo
+ * @author Haibo Liu
  * @since 4.0
  */
 public class SearchHitsImpl<T> implements SearchScrollHits<T> {
@@ -42,6 +43,7 @@ public class SearchHitsImpl<T> implements SearchScrollHits<T> {
 	@Nullable private final AggregationsContainer<?> aggregations;
 	@Nullable private final Suggest suggest;
 	@Nullable private String pointInTimeId;
+	@Nullable private final SearchShardStatistics searchShardStatistics;
 
 	/**
 	 * @param totalHits the number of total hits for the search
@@ -53,7 +55,8 @@ public class SearchHitsImpl<T> implements SearchScrollHits<T> {
 	 */
 	public SearchHitsImpl(long totalHits, TotalHitsRelation totalHitsRelation, float maxScore, @Nullable String scrollId,
 			@Nullable String pointInTimeId, List<? extends SearchHit<T>> searchHits,
-			@Nullable AggregationsContainer<?> aggregations, @Nullable Suggest suggest) {
+			@Nullable AggregationsContainer<?> aggregations, @Nullable Suggest suggest,
+			@Nullable SearchShardStatistics searchShardStatistics) {
 
 		Assert.notNull(searchHits, "searchHits must not be null");
 
@@ -66,6 +69,7 @@ public SearchHitsImpl(long totalHits, TotalHitsRelation totalHitsRelation, float
 		this.aggregations = aggregations;
 		this.suggest = suggest;
 		this.unmodifiableSearchHits = Lazy.of(() -> Collections.unmodifiableList(searchHits));
+		this.searchShardStatistics = searchShardStatistics;
 	}
 
 	// region getter
@@ -118,6 +122,11 @@ public String getPointInTimeId() {
 		return pointInTimeId;
 	}
 
+	@Override
+	public SearchShardStatistics getSearchShardStatistics() {
+		return searchShardStatistics;
+	}
+
 	@Override
 	public String toString() {
 		return "SearchHits{" + //
@@ -128,6 +137,7 @@ public String toString() {
 				", pointInTimeId='" + pointInTimeId + '\'' + //
 				", searchHits={" + searchHits.size() + " elements}" + //
 				", aggregations=" + aggregations + //
+				", shardStatistics=" + searchShardStatistics + //
 				'}';
 	}
 }

src/main/java/org/springframework/data/elasticsearch/core/SearchShardStatistics.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+import java.util.List;
+
+import org.springframework.data.elasticsearch.ElasticsearchErrorCause;
+import org.springframework.lang.Nullable;
+
+/**
+ * @author Haibo Liu
+ * @since 5.3
+ */
+public class SearchShardStatistics {
+	private final Number failed;
+
+	private final Number successful;
+
+	private final Number total;
+
+	@Nullable private final Number skipped;
+
+	private final List<Failure> failures;
+
+	private SearchShardStatistics(Number failed, Number successful, Number total, @Nullable Number skipped,
+			List<Failure> failures) {
+		this.failed = failed;
+		this.successful = successful;
+		this.total = total;
+		this.skipped = skipped;
+		this.failures = failures;
+	}
+
+	public static SearchShardStatistics of(Number failed, Number successful, Number total, @Nullable Number skipped,
+			List<Failure> failures) {
+		return new SearchShardStatistics(failed, successful, total, skipped, failures);
+	}
+
+	public Number getFailed() {
+		return failed;
+	}
+
+	public Number getSuccessful() {
+		return successful;
+	}
+
+	public Number getTotal() {
+		return total;
+	}
+
+	@Nullable
+	public Number getSkipped() {
+		return skipped;
+	}
+
+	public boolean isFailed() {
+		return failed.intValue() > 0;
+	}
+
+	public List<Failure> getFailures() {
+		return failures;
+	}
+
+	public static class Failure {
+		@Nullable private final String index;
+		@Nullable private final String node;
+		@Nullable private final String status;
+		private final int shard;
+		@Nullable private final Exception exception;
+		@Nullable private final ElasticsearchErrorCause elasticsearchErrorCause;
+
+		private Failure(@Nullable String index, @Nullable String node, @Nullable String status, int shard,
+				@Nullable Exception exception, @Nullable ElasticsearchErrorCause elasticsearchErrorCause) {
+			this.index = index;
+			this.node = node;
+			this.status = status;
+			this.shard = shard;
+			this.exception = exception;
+			this.elasticsearchErrorCause = elasticsearchErrorCause;
+		}
+
+		public static SearchShardStatistics.Failure of(@Nullable String index, @Nullable String node,
+				@Nullable String status, int shard, @Nullable Exception exception,
+				@Nullable ElasticsearchErrorCause elasticsearchErrorCause) {
+			return new SearchShardStatistics.Failure(index, node, status, shard, exception, elasticsearchErrorCause);
+		}
+
+		@Nullable
+		public String getIndex() {
+			return index;
+		}
+
+		@Nullable
+		public String getNode() {
+			return node;
+		}
+
+		@Nullable
+		public String getStatus() {
+			return status;
+		}
+
+		@Nullable
+		public Exception getException() {
+			return exception;
+		}
+
+		public int getShard() {
+			return shard;
+		}
+
+		@Nullable
+		public ElasticsearchErrorCause getElasticsearchErrorCause() {
+			return elasticsearchErrorCause;
+		}
+	}
+}

src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchCustomConversions.java

@@ -129,7 +129,7 @@ public Double convert(BigDecimal source) {
 	@WritingConverter
 	enum ByteArrayToBase64Converter implements Converter<byte[], String> {
 
-		INSTANCE,;
+		INSTANCE;
 
 		@Override
 		public String convert(byte[] source) {