Cannot reuse ProjectionOperation inside $lookup pipeline [DATAMONGO-2615]

[spring-projects-issues]

Daniel Theuke opened DATAMONGO-2615 and commented

If I try to use a ProjectionOperation with andExpression("$foo.bar") inside a custom $lookup document I get the following error:

java.lang.IllegalArgumentException: Invalid reference '$meta.name'!
	at org.springframework.data.mongodb.core.aggregation.ExposedFieldsAggregationOperationContext.getReference(ExposedFieldsAggregationOperationContext.java:114) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.ExposedFieldsAggregationOperationContext.getReference(ExposedFieldsAggregationOperationContext.java:86) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.AggregationExpressionTransformer$AggregationExpressionTransformationContext.getFieldReference(AggregationExpressionTransformer.java:82) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer$CompoundExpressionNodeConversion.convert(SpelExpressionTransformer.java:541) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer.transform(SpelExpressionTransformer.java:113) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.SpelExpressionTransformer.transform(SpelExpressionTransformer.java:105) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.ProjectionOperation$ExpressionProjectionOperationBuilder$ExpressionProjection.toMongoExpression(ProjectionOperation.java:438) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.ProjectionOperation$ExpressionProjectionOperationBuilder$ExpressionProjection.toDocument(ProjectionOperation.java:433) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]
	at org.springframework.data.mongodb.core.aggregation.ProjectionOperation.toDocument(ProjectionOperation.java:261) ~[spring-data-mongodb-3.0.3.RELEASE.jar:3.0.3.RELEASE]

Code:

private static final AggregationOperation EXTRACT_ELEM = replaceRoot(ElemContainer._data);
private static final ProjectionOperation ELEM_AS_METADATA = Aggregation.project()
            .andInclude("id")
            .andInclude("type")
            .andExpression("$meta.name").as("name");
    private static final ProjectionOperation ELEM_AS_NESTED_METADATA = ELEM_AS_METADATA
            // Child ids
            .and(VariableOperators.mapItemsOf(
                    ObjectOperators.valueOf("childIds").toArray())
                    .as("this")
                    .andApply(ctx -> new Document()
                            .append("type", "$$this.k")
                            .append("id", "$$this.v")))
            .as("children"); // Named children to be overwritten later on

    private static final AggregationOperation LOOKUP_CHILDREN = ctx -> new Document()
        .append("$lookup", new Document()
            .append("from", "configuration")
            .append("let", new Document()
                    .append("childIds", "$children"))
            .append("pipeline", asList(
                    new Document() // Join condition
                            .append("$match", new Document()
                                    .append("$expr", new Document()
                                            .append("$in", asList("$_id", "$$childIds")))),
                    EXTRACT_ELEM,
                    ELEM_AS_METADATA.toDocument(ctx.continueOnMissingFieldReference()))) // <-- BOOM
            .append("as", "children"); // Overwrite "children" field

// --------------------------

final TypedAggregation<ElemContainer> aggregation = newAggregate(
                match(criteria),
                EXTRACT_ELEM,
                ELEM_AS_NESTED_METADATA,
                LOOKUP_CHILDREN);
        return this.mongoTemplate.aggregate(aggregation, NestedElementMetadata.class).getMappedResults();

In this case the continueOnMissingFieldReference() doesn't seem to work at all.
(ExposedFieldsAggregationOperationContext doesn't implement that feature at all)

The default implementation should probably be rewritten to return a delegating variant that wraps the related call similar to RelaxedTypeBasedAggregationOperationContext.

Or is there another way to create a nested context?

---

Affects: 3.0.3 (Neumann SR3)

[spring-projects-issues]

Daniel Theuke commented

I can bypass the error if I invoke the nested element with Aggregation.DEFAULT_CONTEXT.

However the continueOnMissingFieldReference() should still be fixed

[blueiceprj]

This feature is really necessary for the performance of query.

Its works for me. But it must be in this package.

org.springframework.data.mongodb.core.aggregation

public class ExcludeFieldsAggregation implements AggregationOperation
{
	private String[] fields;

	public ExcludeFieldsAggregation(String...fields)
	{
		this.fields = fields;
	}
	
	@Override
	public Document toDocument(AggregationOperationContext context)
	{
		Document result = new Document();
		for (String field: fields) {
			result.put(field,0);	
		}
		return new Document("$project", result);
	}

}

public class IncludeFieldsAggregation implements AggregationOperation
{
	private String[] fields;

	public IncludeFieldsAggregation(String...fields)
	{
		this.fields = fields;
	}
	
	@Override
	public Document toDocument(AggregationOperationContext context)
	{
		Document result = new Document();
		for (String field: fields) {
			result.put(field,1);	
		}
		return new Document("$project", result);
	}

}

package org.springframework.data.mongodb.core.aggregation;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import org.bson.Document;
import org.springframework.data.mongodb.core.aggregation.ExposedFields.ExposedField;
import org.springframework.data.mongodb.core.aggregation.FieldsExposingAggregationOperation.InheritsFieldsAggregationOperation;
import org.springframework.data.mongodb.core.query.CriteriaDefinition;
import org.springframework.util.Assert;

import org.springframework.data.mongodb.core.aggregation.ExcludeFieldsAggregation;
import org.springframework.data.mongodb.core.aggregation.IncludeFieldsAggregation;

public class LookupPipelineOperation implements AggregationOperation, FieldsExposingAggregationOperation, InheritsFieldsAggregationOperation
{

	private Field from;
	private Field localField;
	private Field foreignField;
	private ExposedField as;
	
	private Document lets = null;
	private List<AggregationOperation> pipeline = new ArrayList<>();
	private String[] includeFields;
	private String[] excludeFields;

	/**
	 * Creates a new {@link MatchOperation} for the given {@link CriteriaDefinition}.
	 *
	 * @param criteriaDefinition must not be {@literal null}.
	 */
	public LookupPipelineOperation()
	{}

	public LookupPipelineOperation let(String fieldName, String expression)
	{
		Assert.notNull(fieldName, "Field name must not be null!");
		Assert.notNull(expression, "Expression must not be null!");
		if (this.lets==null) {
			this.lets = new Document();
		}
		this.lets.append(fieldName, expression);
		return this;
	}

	public LookupPipelineOperation from(String from)
	{
		Assert.notNull(from, "From must not be null!");
		this.from = Fields.field(from);
		return this;
	}

	public LookupPipelineOperation as(String as)
	{
		Assert.notNull(as, "As must not be null!");
		this.as = new ExposedField(Fields.field(as), true);
		return this;
	}


	public LookupPipelineOperation include(String... fields)
	{
		Assert.notNull(fields, "Fields must not be null!");
		this.includeFields = fields;
		return this;
	}

	public LookupPipelineOperation exclude(String... fields)
	{
		Assert.notNull(fields, "Fields must not be null!");
		this.excludeFields = fields;
		return this;
	}

	public LookupPipelineOperation pipeline(AggregationOperation pipeline)
	{
		Assert.notNull(pipeline, "Pipeline must not be null!");
		if (this.pipeline == null)
		{
			this.pipeline = new ArrayList<>();
		}
		this.pipeline.add(pipeline);
		return this;
	}

	public LookupPipelineOperation pipeline(List<AggregationOperation> pipeline)
	{
		Assert.notNull(pipeline, "Pipeline must not be null!");
		this.pipeline = pipeline;
		return this;
	}

	public LookupPipelineOperation localField(String localField)
	{
		Assert.notNull(localField, "localField must not be null!");
		this.localField = Fields.field(localField);
		return this;
	}

	public LookupPipelineOperation foreignField(String foreignField)
	{
		Assert.notNull(foreignField, "foreignField must not be null!");
		this.foreignField = Fields.field(foreignField);
		return this;
	}

	/*
	 * (non-Javadoc)
	 *
	 * @see org.springframework.data.mongodb.core.aggregation.AggregationOperation#toDocument(org.
	 * springframework.data.mongodb.core.aggregation.AggregationOperationContext)
	 */
	@SuppressWarnings("deprecation")
	@Override
	public Document toDocument(final AggregationOperationContext context)
	{
		context.continueOnMissingFieldReference();
		if (this.excludeFields != null)
		{
			pipeline.add(0, new ExcludeFieldsAggregation(this.excludeFields));
		}
		if (this.includeFields != null)
		{
			pipeline.add(0, new IncludeFieldsAggregation(this.includeFields));
		}
		
		Document lookupDoc = new Document();

		lookupDoc.append("from", from.getTarget());

		if (this.localField!=null)
		{
			lookupDoc.append("localField", localField.getTarget());
		}

		if (this.foreignField!=null)
		{
			lookupDoc.append("foreignField", foreignField.getTarget());
		}

		if (lets != null && !lets.isEmpty())
		{
			lookupDoc.append("let", lets);
		}
		if (this.pipeline != null && this.pipeline.size() > 0)
		{
			lookupDoc.append("pipeline", pipeline.stream().map(op -> op.toDocument(context)).collect(Collectors.toList()));
		}
		if (this.as != null)
		{
			lookupDoc.append("as", as.getTarget());
		}
		return new Document(getOperator(), lookupDoc);
	}

	@Override
	public String getOperator()
	{
		return "$lookup";
	}

	@Override
	public ExposedFields getFields()
	{
		return ExposedFields.from(as);
	}
}

[christophstrobl]

related to #3917

[christophstrobl]

will be fixed by #4328