Add knn search parameter and remove knn query

src/main/antora/modules/ROOT/pages/migration-guides/migration-guide-5.3-5.4.adoc

@@ -6,6 +6,17 @@ This section describes breaking changes from version 5.3.x to 5.4.x and how remo
 [[elasticsearch-migration-guide-5.3-5.4.breaking-changes]]
 == Breaking Changes
 
+[[elasticsearch-migration-guide-5.3-5.4.breaking-changes.knn-search]]
+=== knn search
+The `withKnnQuery` method in `NativeQueryBuilder` has been replaced with `withKnnSearches` to build a `NativeQuery` with knn search.
+
+`KnnQuery` and `KnnSearch` are two different classes in elasticsearch java client and are used for different queries, with different parameters supported:
+
+- `KnnSearch`: is https://www.elastic.co/guide/en/elasticsearch/reference/8.13/search-search.html#search-api-knn[the top level `knn` query] in the elasticsearch request;
+- `KnnQuery`: is https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-knn-query.html[the `knn` query inside `query` clause];
+
+If `KnnQuery` is still preferable, please be sure to construct it inside `query` clause manually, by means of `withQuery(co.elastic.clients.elasticsearch._types.query_dsl.Query query)` clause in `NativeQueryBuilder`.
+
 [[elasticsearch-migration-guide-5.3-5.4.deprecations]]
 == Deprecations
 

src/main/java/org/springframework/data/elasticsearch/annotations/Field.java

@@ -37,6 +37,7 @@
  * @author Brian Kimmig
  * @author Morgan Lutz
  * @author Sascha Woo
+ * @author Haibo Liu
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target({ ElementType.FIELD, ElementType.ANNOTATION_TYPE, ElementType.METHOD })
@@ -195,6 +196,27 @@
 	 */
 	int dims() default -1;
 
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	String elementType() default FieldElementType.DEFAULT;
+
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	KnnSimilarity knnSimilarity() default KnnSimilarity.DEFAULT;
+
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	KnnIndexOptions[] knnIndexOptions() default {};
+
 	/**
 	 * Controls how Elasticsearch dynamically adds fields to the inner object within the document.<br>
 	 * To be used in combination with {@link FieldType#Object} or {@link FieldType#Nested}

src/main/java/org/springframework/data/elasticsearch/annotations/FieldElementType.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.annotations;
+
+/**
+ * @author Haibo Liu
+ * @since 5.4
+ */
+public final class FieldElementType {
+	public final static String DEFAULT = "";
+	public final static String FLOAT = "float";
+	public final static String BYTE = "byte";
+}

src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java

@@ -29,6 +29,7 @@
  * @author Aleksei Arsenev
  * @author Brian Kimmig
  * @author Morgan Lutz
+ * @author Haibo Liu
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.ANNOTATION_TYPE)
@@ -149,4 +150,25 @@
 	 * @since 4.2
 	 */
 	int dims() default -1;
+
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	String elementType() default FieldElementType.DEFAULT;
+
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	KnnSimilarity knnSimilarity() default KnnSimilarity.DEFAULT;
+
+	/**
+	 * to be used in combination with {@link FieldType#Dense_Vector}
+	 *
+	 * @since 5.4
+	 */
+	KnnIndexOptions[] knnIndexOptions() default {};
 }

src/main/java/org/springframework/data/elasticsearch/annotations/KnnAlgorithmType.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.annotations;
+
+/**
+ * @author Haibo Liu
+ * @since 5.4
+ */
+public enum KnnAlgorithmType {
+	HNSW("hnsw"),
+	INT8_HNSW("int8_hnsw"),
+	FLAT("flat"),
+	INT8_FLAT("int8_flat"),
+	DEFAULT("");
+
+	private final String type;
+
+	KnnAlgorithmType(String type) {
+		this.type = type;
+	}
+
+	public String getType() {
+		return type;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/annotations/KnnIndexOptions.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.annotations;
+
+/**
+ * @author Haibo Liu
+ * @since 5.4
+ */
+public @interface KnnIndexOptions {
+
+	KnnAlgorithmType type() default KnnAlgorithmType.DEFAULT;
+
+	/**
+	 * Only applicable to {@link KnnAlgorithmType#HNSW} and {@link KnnAlgorithmType#INT8_HNSW} index types.
+	 */
+	int m() default -1;
+
+	/**
+	 * Only applicable to {@link KnnAlgorithmType#HNSW} and {@link KnnAlgorithmType#INT8_HNSW} index types.
+	 */
+	int efConstruction() default -1;
+
+	/**
+	 * Only applicable to {@link KnnAlgorithmType#INT8_HNSW} and {@link KnnAlgorithmType#INT8_FLAT} index types.
+	 */
+	float confidenceInterval() default -1F;
+}

src/main/java/org/springframework/data/elasticsearch/annotations/KnnSimilarity.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.annotations;
+
+/**
+ * @author Haibo Liu
+ * @since 5.4
+ */
+public enum KnnSimilarity {
+	L2_NORM("l2_norm"),
+	DOT_PRODUCT("dot_product"),
+	COSINE("cosine"),
+	MAX_INNER_PRODUCT("max_inner_product"),
+	DEFAULT("");
+
+	private final String similarity;
+
+	KnnSimilarity(String similarity) {
+		this.similarity = similarity;
+	}
+
+	public String getSimilarity() {
+		return similarity;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQuery.java

@@ -15,7 +15,6 @@
  */
 package org.springframework.data.elasticsearch.client.elc;
 
-import co.elastic.clients.elasticsearch._types.KnnQuery;
 import co.elastic.clients.elasticsearch._types.KnnSearch;
 import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
@@ -30,7 +29,6 @@
 import java.util.Map;
 
 import org.springframework.data.elasticsearch.core.query.BaseQuery;
-import org.springframework.data.elasticsearch.core.query.ScriptedField;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
@@ -40,6 +38,7 @@
  *
  * @author Peter-Josef Meisch
  * @author Sascha Woo
+ * @author Haibo Liu
  * @since 4.4
  */
 public class NativeQuery extends BaseQuery {
@@ -54,7 +53,6 @@ public class NativeQuery extends BaseQuery {
 	private List<SortOptions> sortOptions = Collections.emptyList();
 
 	private Map<String, JsonData> searchExtensions = Collections.emptyMap();
-	@Nullable private KnnQuery knnQuery;
 	@Nullable private List<KnnSearch> knnSearches = Collections.emptyList();
 
 	public NativeQuery(NativeQueryBuilder builder) {
@@ -72,7 +70,6 @@ public NativeQuery(NativeQueryBuilder builder) {
 					"Cannot add an NativeQuery in a NativeQuery");
 		}
 		this.springDataQuery = builder.getSpringDataQuery();
-		this.knnQuery = builder.getKnnQuery();
 		this.knnSearches = builder.getKnnSearches();
 	}
 
@@ -124,14 +121,6 @@ public void setSpringDataQuery(@Nullable org.springframework.data.elasticsearch.
 		this.springDataQuery = springDataQuery;
 	}
 
-	/**
-	 * @since 5.1
-	 */
-	@Nullable
-	public KnnQuery getKnnQuery() {
-		return knnQuery;
-	}
-
 	/**
 	 * @since 5.3.1
 	 */

src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQueryBuilder.java

@@ -40,6 +40,7 @@
 /**
  * @author Peter-Josef Meisch
  * @author Sascha Woo
+ * @author Haibo Liu
  * @since 4.4
  */
 public class NativeQueryBuilder extends BaseQueryBuilder<NativeQuery, NativeQueryBuilder> {
@@ -213,13 +214,30 @@ public NativeQueryBuilder withQuery(org.springframework.data.elasticsearch.core.
 	}
 
 	/**
-	 * @since 5.1
+	 * @since 5.4
 	 */
-	public NativeQueryBuilder withKnnQuery(KnnQuery knnQuery) {
-		this.knnQuery = knnQuery;
+	public NativeQueryBuilder withKnnSearches(List<KnnSearch> knnSearches) {
+		this.knnSearches = knnSearches;
 		return this;
 	}
 
+	/**
+	 * @since 5.4
+	 */
+	public NativeQueryBuilder withKnnSearches(Function<KnnSearch.Builder, ObjectBuilder<KnnSearch>> fn) {
+
+		Assert.notNull(fn, "fn must not be null");
+
+		return withKnnSearches(fn.apply(new KnnSearch.Builder()).build());
+	}
+
+	/**
+	 * @since 5.4
+	 */
+	public NativeQueryBuilder withKnnSearches(KnnSearch knnSearch) {
+		return withKnnSearches(List.of(knnSearch));
+	}
+
 	public NativeQuery build() {
 		Assert.isTrue(query == null || springDataQuery == null, "Cannot have both a native query and a Spring Data query");
 		return new NativeQuery(this);

src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java

@@ -1377,7 +1377,7 @@ public MsearchRequest searchMsearchRequest(
 	private Function<MultisearchHeader.Builder, ObjectBuilder<MultisearchHeader>> msearchHeaderBuilder(Query query,
 			IndexCoordinates index, @Nullable String routing) {
 		return h -> {
-			var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null
+			var searchType = (query instanceof NativeQuery nativeQuery && !isEmpty(nativeQuery.getKnnSearches())) ? null
 					: searchType(query.getSearchType());
 
 			h //
@@ -1409,7 +1409,7 @@ private <T> void prepareSearchRequest(Query query, @Nullable String routing, @Nu
 
 		ElasticsearchPersistentEntity<?> persistentEntity = getPersistentEntity(clazz);
 
-		var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null
+		var searchType = (query instanceof NativeQuery nativeQuery && !isEmpty(nativeQuery.getKnnSearches())) ? null
 				: searchType(query.getSearchType());
 
 		builder //
@@ -1728,17 +1728,6 @@ private void prepareNativeSearch(NativeQuery query, SearchRequest.Builder builde
 				.sort(query.getSortOptions()) //
 		;
 
-		if (query.getKnnQuery() != null) {
-			var kq = query.getKnnQuery();
-			builder.knn(ksb -> ksb
-					.field(kq.field())
-					.queryVector(kq.queryVector())
-					.numCandidates(kq.numCandidates())
-					.filter(kq.filter())
-					.similarity(kq.similarity()));
-
-		}
-
 		if (!isEmpty(query.getKnnSearches())) {
 			builder.knn(query.getKnnSearches());
 		}
@@ -1760,17 +1749,6 @@ private void prepareNativeSearch(NativeQuery query, MultisearchBody.Builder buil
 				.collapse(query.getFieldCollapse()) //
 				.sort(query.getSortOptions());
 
-		if (query.getKnnQuery() != null) {
-			var kq = query.getKnnQuery();
-			builder.knn(ksb -> ksb
-					.field(kq.field())
-					.queryVector(kq.queryVector())
-					.numCandidates(kq.numCandidates())
-					.filter(kq.filter())
-					.similarity(kq.similarity()));
-
-		}
-
 		if (!isEmpty(query.getKnnSearches())) {
 			builder.knn(query.getKnnSearches());
 		}