From 9d8249ed17e15e073375a27414e74434237a7124 Mon Sep 17 00:00:00 2001 From: mikereiche Date: Fri, 20 Aug 2021 18:21:30 -0700 Subject: [PATCH 1/2] Support derived queries from methods named findDistinctField1AndField2ByField3(). Closes #1200. --- .../ReactiveFindByQueryOperationSupport.java | 5 +- .../data/couchbase/core/query/Query.java | 48 +++++++++++++- .../repository/query/CouchbasePartTree.java | 64 +++++++++++++++++++ .../repository/query/N1qlQueryCreator.java | 6 +- .../query/PartTreeCouchbaseQuery.java | 10 ++- .../couchbase/domain/AirportRepository.java | 16 ++++- ...chbaseRepositoryQueryIntegrationTests.java | 48 +++++++++++--- 7 files changed, 177 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/springframework/data/couchbase/repository/query/CouchbasePartTree.java diff --git a/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java b/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java index 696c335f3..4bc8d962e 100644 --- a/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java +++ b/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java @@ -176,7 +176,7 @@ public Flux all() { }).flatMapMany(ReactiveQueryResult::rowsAsObject).flatMap(row -> { String id = ""; long cas = 0; - if (distinctFields == null) { + if (!query.isDistinct() && distinctFields != null) { if (row.getString(TemplateUtils.SELECT_ID) == null) { return Flux.error(new CouchbaseException( "query did not project " + TemplateUtils.SELECT_ID + ". Either use #{#n1ql.selectEntity} or project " @@ -227,7 +227,8 @@ public Mono exists() { } private String assembleEntityQuery(final boolean count, String[] distinctFields, String collection) { - return query.toN1qlSelectString(template, collection, this.domainType, this.returnType, count, distinctFields); + return query.toN1qlSelectString(template, collection, this.domainType, this.returnType, count, + query.getDistinctFields() != null ? query.getDistinctFields() : distinctFields); } } } diff --git a/src/main/java/org/springframework/data/couchbase/core/query/Query.java b/src/main/java/org/springframework/data/couchbase/core/query/Query.java index 4599d25e5..2677ba8c5 100644 --- a/src/main/java/org/springframework/data/couchbase/core/query/Query.java +++ b/src/main/java/org/springframework/data/couchbase/core/query/Query.java @@ -51,6 +51,8 @@ public class Query { private JsonValue parameters = JsonValue.ja(); private long skip; private int limit; + private boolean distinct; + private String[] distinctFields; private Sort sort = Sort.unsorted(); private QueryScanConsistency queryScanConsistency; private Meta meta; @@ -123,6 +125,46 @@ public Query limit(int limit) { return this; } + /** + * Is this a DISTINCT query? {@code distinct}. + * + * @param distinct + * @return + */ + public Query distinct(boolean distinct) { + this.distinct = distinct; + return this; + } + + /** + * Is this a DISTINCT query? {@code distinct}. + * + * @return distinct + */ + public boolean isDistinct() { + return distinct; + } + + /** + * distinctFields for query (non-null but empty means all fields) ? {@code distinctFields}. + * + * @param distinctFields + * @return + */ + public Query distinct(String[] distinctFields) { + this.distinctFields = distinctFields; + return this; + } + + /** + * distinctFields for query (non-null but empty means all fields) ? {@code distinctFields}. + * + * @return distinctFields + */ + public String[] getDistinctFields() { + return distinctFields; + } + /** * Sets the given pagination information on the {@link Query} instance. Will transparently set {@code skip} and * {@code limit} as well as applying the {@link Sort} instance defined with the {@link Pageable}. @@ -136,7 +178,7 @@ public Query with(final Pageable pageable) { } this.limit = pageable.getPageSize(); this.skip = pageable.getOffset(); - if(!this.sort.equals(pageable.getSort())) + if (!this.sort.equals(pageable.getSort())) this.sort.and(pageable.getSort()); return this; } @@ -176,7 +218,7 @@ public Query with(final Sort sort) { return this; } - public Query withoutSort(){ + public Query withoutSort() { this.sort = Sort.unsorted(); return this; } @@ -294,7 +336,7 @@ public String toN1qlSelectString(ReactiveCouchbaseTemplate template, String coll appendString(statement, n1ql.selectEntity); // select ... appendWhereString(statement, n1ql.filter); // typeKey = typeValue appendWhere(statement, new int[] { 0 }, template.getConverter()); // criteria on this Query - if(!isCount){ + if (!isCount) { appendSort(statement); appendSkipAndLimit(statement); } diff --git a/src/main/java/org/springframework/data/couchbase/repository/query/CouchbasePartTree.java b/src/main/java/org/springframework/data/couchbase/repository/query/CouchbasePartTree.java new file mode 100644 index 000000000..101552182 --- /dev/null +++ b/src/main/java/org/springframework/data/couchbase/repository/query/CouchbasePartTree.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.data.couchbase.repository.query; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.data.mapping.PropertyPath; +import org.springframework.data.repository.query.parser.Part; +import org.springframework.data.repository.query.parser.PartTree; + +/** + * Extend PartTree to parse out distinct fields + * + * @author Michael Reiche + */ +public class CouchbasePartTree extends PartTree { + private static final Pattern DISTINCT_TEMPLATE = Pattern + .compile("^(find|read|get|query|search|stream|count)(Distinct)(\\p{Lu}.*?)(First|Top|By|$)"); + + String[] distinctFields; + + public CouchbasePartTree(String methodName, Class domainType) { + super(methodName, domainType); + maybeInitDistinctFields(methodName, domainType); + } + + String[] getDistinctFields() { + return distinctFields; + } + + private void maybeInitDistinctFields(String methodName, Class domainType) { + if (isDistinct()) { + Matcher grp = DISTINCT_TEMPLATE.matcher(methodName); + if (grp.matches()) { + String grp3 = grp.group(3); + String[] names = grp.group(3).split("And"); + int parameterCount = names.length; + distinctFields = new String[names.length]; + for (int i = 0; i < parameterCount; ++i) { + Part.Type type = Part.Type.fromProperty(names[i]); + PropertyPath path = PropertyPath.from(type.extractProperty(names[i]), domainType); + distinctFields[i] = path.toDotPath(); + } + } else { + distinctFields = new String[0]; + } + } + } +} diff --git a/src/main/java/org/springframework/data/couchbase/repository/query/N1qlQueryCreator.java b/src/main/java/org/springframework/data/couchbase/repository/query/N1qlQueryCreator.java index 66f8c37aa..37cacc2f9 100644 --- a/src/main/java/org/springframework/data/couchbase/repository/query/N1qlQueryCreator.java +++ b/src/main/java/org/springframework/data/couchbase/repository/query/N1qlQueryCreator.java @@ -39,7 +39,6 @@ import org.springframework.data.repository.query.parser.AbstractQueryCreator; import org.springframework.data.repository.query.parser.Part; import org.springframework.data.repository.query.parser.PartTree; -import org.springframework.util.Assert; /** * @author Michael Nitschinger @@ -52,6 +51,7 @@ public class N1qlQueryCreator extends AbstractQueryCreator public static final String META_CAS_PROPERTY = "cas"; public static final String META_EXPIRATION_PROPERTY = "expiration"; + private final PartTree tree; private final ParameterAccessor accessor; private final MappingContext context; private final QueryMethod queryMethod; @@ -61,6 +61,7 @@ public class N1qlQueryCreator extends AbstractQueryCreator public N1qlQueryCreator(final PartTree tree, final ParameterAccessor accessor, final QueryMethod queryMethod, final CouchbaseConverter converter, final String bucketName) { super(tree, accessor); + this.tree = tree; this.accessor = accessor; this.context = converter.getMappingContext(); this.queryMethod = queryMethod; @@ -79,10 +80,11 @@ protected QueryCriteria create(final Part part, final Iterator iterator) public Query createQuery() { Query q = this.createQuery((Optional.of(this.accessor).map(ParameterAccessor::getSort).orElse(Sort.unsorted()))); Pageable pageable = accessor.getPageable(); - if(pageable.isPaged()) { + if (pageable.isPaged()) { q.skip(pageable.getOffset()); q.limit(pageable.getPageSize()); } + q.distinct(tree.isDistinct()); return q; } diff --git a/src/main/java/org/springframework/data/couchbase/repository/query/PartTreeCouchbaseQuery.java b/src/main/java/org/springframework/data/couchbase/repository/query/PartTreeCouchbaseQuery.java index bef95ec1b..e555adea4 100644 --- a/src/main/java/org/springframework/data/couchbase/repository/query/PartTreeCouchbaseQuery.java +++ b/src/main/java/org/springframework/data/couchbase/repository/query/PartTreeCouchbaseQuery.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 the original author or authors. + * Copyright 2020-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ */ public class PartTreeCouchbaseQuery extends AbstractCouchbaseQuery { - private final PartTree tree; + private final CouchbasePartTree tree; private final CouchbaseConverter converter; /** @@ -52,7 +52,7 @@ public PartTreeCouchbaseQuery(CouchbaseQueryMethod method, CouchbaseOperations o super(method, operations, expressionParser, evaluationContextProvider); ResultProcessor processor = method.getResultProcessor(); - this.tree = new PartTree(method.getName(), processor.getReturnedType().getDomainType()); + this.tree = new CouchbasePartTree(method.getName(), processor.getReturnedType().getDomainType()); this.converter = operations.getConverter(); } @@ -79,6 +79,9 @@ protected Query createQuery(ParametersParameterAccessor accessor) { if (tree.isLimiting()) { query.limit(tree.getMaxResults()); } + if (tree.isDistinct()) { + query.distinct(tree.getDistinctFields()); + } return query; } @@ -128,4 +131,5 @@ protected boolean isDeleteQuery() { protected boolean isLimiting() { return tree.isLimiting(); } + } diff --git a/src/test/java/org/springframework/data/couchbase/domain/AirportRepository.java b/src/test/java/org/springframework/data/couchbase/domain/AirportRepository.java index adcbadb62..f7c890ac5 100644 --- a/src/test/java/org/springframework/data/couchbase/domain/AirportRepository.java +++ b/src/test/java/org/springframework/data/couchbase/domain/AirportRepository.java @@ -114,10 +114,10 @@ public interface AirportRepository extends CouchbaseRepository, Long countFancyExpression(@Param("projectIds") List projectIds, @Param("planIds") List planIds, @Param("active") Boolean active); - @Query("SELECT 1 FROM `#{#n1ql.bucket}` WHERE 0 = 1" ) + @Query("SELECT 1 FROM `#{#n1ql.bucket}` WHERE 0 = 1") Long countBad(); - @Query("SELECT count(*) FROM `#{#n1ql.bucket}`" ) + @Query("SELECT count(*) FROM `#{#n1ql.bucket}`") Long countGood(); @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) @@ -126,6 +126,18 @@ Long countFancyExpression(@Param("projectIds") List projectIds, @Param(" @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) Optional findByIdAndIata(String id, String iata); + @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) + List findDistinctIcaoBy(); + + @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) + List findDistinctIcaoAndIataBy(); + + @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) + Long countDistinctIcaoAndIataBy(); + + @ScanConsistency(query = QueryScanConsistency.REQUEST_PLUS) + Long countDistinctIcaoBy(); + @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.METHOD, ElementType.TYPE }) // @Meta diff --git a/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryQueryIntegrationTests.java b/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryQueryIntegrationTests.java index e6dfc6eca..9e8997a8e 100644 --- a/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryQueryIntegrationTests.java +++ b/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryQueryIntegrationTests.java @@ -39,7 +39,6 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; -import com.couchbase.client.java.kv.UpsertOptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -89,6 +88,7 @@ import com.couchbase.client.java.json.JsonArray; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.kv.MutationState; +import com.couchbase.client.java.kv.UpsertOptions; import com.couchbase.client.java.query.QueryOptions; import com.couchbase.client.java.query.QueryScanConsistency; @@ -188,9 +188,9 @@ void annotatedFieldFindName() { Person person2 = personRepository.findById(person.getId().toString()).get(); assertEquals(person.getSalutation(), person2.getSalutation()); // needs fix from datacouch_1184 - //List persons3 = personRepository.findBySalutation("Mrs"); - //assertEquals(1, persons3.size()); - //assertEquals(person.getSalutation(), persons3.get(0).getSalutation()); + // List persons3 = personRepository.findBySalutation("Mrs"); + // assertEquals(1, persons3.size()); + // assertEquals(person.getSalutation(), persons3.get(0).getSalutation()); } finally { personRepository.deleteById(person.getId().toString()); } @@ -476,13 +476,13 @@ void count() { } } - @Test - void badCount(){ + @Test + void badCount() { assertThrows(CouchbaseQueryExecutionException.class, () -> airportRepository.countBad()); } - @Test - void goodCount(){ + @Test + void goodCount() { airportRepository.countGood(); } @@ -528,6 +528,38 @@ void threadSafeParametersTest() throws Exception { } } + @Test + void distinct() { + String[] iatas = { "JFK", "IAD", "SFO", "SJC", "SEA", "LAX", "PHX" }; + String[] icaos = { "ic0", "ic1", "ic0", "ic1", "ic0", "ic1", "ic0" }; + + try { + for (int i = 0; i < iatas.length; i++) { + Airport airport = new Airport("airports::" + iatas[i], iatas[i] /*iata*/, icaos[i] /* icao */); + couchbaseTemplate.insertById(Airport.class).one(airport); + } + + // distinct icao - parser requires 'By' on the end or it does not match pattern. + List airports1 = airportRepository.findDistinctIcaoBy(); + assertEquals(2, airports1.size()); + + List airports2 = airportRepository.findDistinctIcaoAndIataBy(); + assertEquals(7, airports2.size()); + + // count( distinct { iata, icao } ) + long count1 = airportRepository.countDistinctIcaoAndIataBy(); + assertEquals(7, count1); + + // count( distinct { icao } ) + long count2 = airportRepository.countDistinctIcaoBy(); + assertEquals(2, count2); + + } finally { + couchbaseTemplate.removeById() + .all(Arrays.stream(iatas).map((iata) -> "airports::" + iata).collect(Collectors.toSet())); + } + } + @Test void stringQueryTest() throws Exception { Airport airport = new Airport("airports::vie", "vie", "lowx"); From 7c0507fda4b8570ff984357346abe2ac78499dce Mon Sep 17 00:00:00 2001 From: mikereiche Date: Mon, 23 Aug 2021 13:48:35 -0700 Subject: [PATCH 2/2] Support derived queries from methods named findDistinctF1AndF2ByF3(). Closes #1200. --- .../couchbase/core/ReactiveFindByQueryOperationSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java b/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java index 4bc8d962e..ff181e7d5 100644 --- a/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java +++ b/src/main/java/org/springframework/data/couchbase/core/ReactiveFindByQueryOperationSupport.java @@ -176,7 +176,7 @@ public Flux all() { }).flatMapMany(ReactiveQueryResult::rowsAsObject).flatMap(row -> { String id = ""; long cas = 0; - if (!query.isDistinct() && distinctFields != null) { + if (!query.isDistinct() && distinctFields == null) { if (row.getString(TemplateUtils.SELECT_ID) == null) { return Flux.error(new CouchbaseException( "query did not project " + TemplateUtils.SELECT_ID + ". Either use #{#n1ql.selectEntity} or project "