diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java index 75d7bdedf..7495e3f13 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java @@ -199,8 +199,16 @@ /** * Controls how Elasticsearch dynamically adds fields to the inner object within the document.
* To be used in combination with {@link FieldType#Object} or {@link FieldType#Nested} - * + * * @since 4.3 */ Dynamic dynamic() default Dynamic.INHERIT; + + /** + * marks this field to be excluded from the _source in Elasticsearch + * (https://www.elastic.co/guide/en/elasticsearch/reference/7.15.0/mapping-source-field.html#include-exclude) + * + * @since 4.3 + */ + boolean excludeFromSource() default false; } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java index 7d4cfc4fb..a7f703bd8 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java @@ -21,8 +21,10 @@ import java.io.IOException; import java.lang.annotation.Annotation; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -102,12 +104,12 @@ public class MappingBuilder { private static final String NUMERIC_DETECTION = "numeric_detection"; private static final String DYNAMIC_DATE_FORMATS = "dynamic_date_formats"; private static final String RUNTIME = "runtime"; + private static final String SOURCE = "_source"; + private static final String SOURCE_EXCLUDES = "excludes"; protected final ElasticsearchConverter elasticsearchConverter; private final ObjectMapper objectMapper = new ObjectMapper(); - private boolean writeTypeHints = true; - public MappingBuilder(ElasticsearchConverter elasticsearchConverter) { this.elasticsearchConverter = elasticsearchConverter; } @@ -129,445 +131,479 @@ public String buildPropertyMapping(Class clazz) throws MappingException { protected String buildPropertyMapping(ElasticsearchPersistentEntity entity, @Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) { - try { - - writeTypeHints = entity.writeTypeHints(); - - ObjectNode objectNode = objectMapper.createObjectNode(); + InternalBuilder internalBuilder = new InternalBuilder(); + return internalBuilder.buildPropertyMapping(entity, runtimeFields); + } - // Dynamic templates - addDynamicTemplatesMapping(objectNode, entity); + @Nullable + private org.springframework.data.elasticsearch.core.document.Document getRuntimeFields( + @Nullable ElasticsearchPersistentEntity entity) { - mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null, entity.findAnnotation(DynamicMapping.class), - runtimeFields); + if (entity != null) { + Mapping mappingAnnotation = entity.findAnnotation(Mapping.class); + if (mappingAnnotation != null) { + String runtimeFieldsPath = mappingAnnotation.runtimeFieldsPath(); - return objectMapper.writer().writeValueAsString(objectNode); - } catch (IOException e) { - throw new MappingException("could not build mapping", e); + if (hasText(runtimeFieldsPath)) { + String jsonString = ResourceUtil.readFileFromClasspath(runtimeFieldsPath); + return org.springframework.data.elasticsearch.core.document.Document.parse(jsonString); + } + } } + return null; } - private void writeTypeHintMapping(ObjectNode propertiesNode) throws IOException { + private class InternalBuilder { - if (writeTypeHints) { - propertiesNode.set(TYPEHINT_PROPERTY, objectMapper.createObjectNode() // - .put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) // - .put(FIELD_PARAM_INDEX, false) // - .put(FIELD_PARAM_DOC_VALUES, false)); - } - } + private boolean writeTypeHints = true; + private List excludeFromSource = new ArrayList<>(); + private String nestedPropertyPrefix = ""; - private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity entity, boolean isRootObject, - String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, - @Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping, @Nullable Document runtimeFields) - throws IOException { + protected String buildPropertyMapping(ElasticsearchPersistentEntity entity, + @Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) { - if (entity != null && entity.isAnnotationPresent(Mapping.class)) { - Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class); + try { - if (!mappingAnnotation.enabled()) { - objectNode.put(MAPPING_ENABLED, false); - return; - } + writeTypeHints = entity.writeTypeHints(); - if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) { - objectNode.put(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name())); - } + ObjectNode objectNode = objectMapper.createObjectNode(); - if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) { - objectNode.put(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name())); - } + // Dynamic templates + addDynamicTemplatesMapping(objectNode, entity); - if (mappingAnnotation.dynamicDateFormats().length > 0) { - objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll( - Arrays.stream(mappingAnnotation.dynamicDateFormats()).map(TextNode::valueOf).collect(Collectors.toList())); - } + mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null, + entity.findAnnotation(DynamicMapping.class), runtimeFields); + + if (!excludeFromSource.isEmpty()) { + ObjectNode sourceNode = objectNode.putObject(SOURCE); + ArrayNode excludes = sourceNode.putArray(SOURCE_EXCLUDES); + excludeFromSource.stream().map(TextNode::new).forEach(excludes::add); + } - if (runtimeFields != null) { - objectNode.set(RUNTIME, objectMapper.convertValue(runtimeFields, JsonNode.class)); + return objectMapper.writer().writeValueAsString(objectNode); + } catch (IOException e) { + throw new MappingException("could not build mapping", e); } } - boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField); - if (writeNestedProperties) { - - String type = nestedOrObjectField ? fieldType.toString().toLowerCase() - : FieldType.Object.toString().toLowerCase(); - - ObjectNode nestedObjectNode = objectMapper.createObjectNode(); - nestedObjectNode.put(FIELD_PARAM_TYPE, type); + private void writeTypeHintMapping(ObjectNode propertiesNode) throws IOException { - if (nestedOrObjectField && FieldType.Nested == fieldType && parentFieldAnnotation != null - && parentFieldAnnotation.includeInParent()) { - nestedObjectNode.put(FIELD_INCLUDE_IN_PARENT, true); + if (writeTypeHints) { + propertiesNode.set(TYPEHINT_PROPERTY, objectMapper.createObjectNode() // + .put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) // + .put(FIELD_PARAM_INDEX, false) // + .put(FIELD_PARAM_DOC_VALUES, false)); } - - objectNode.set(nestedObjectFieldName, nestedObjectNode); - // now go on with the nested one - objectNode = nestedObjectNode; } - if (entity != null && entity.dynamic() != Dynamic.INHERIT) { - objectNode.put(TYPE_DYNAMIC, entity.dynamic().name().toLowerCase()); - } else if (dynamicMapping != null) { - objectNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); - } + private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity entity, + boolean isRootObject, String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, + @Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping, + @Nullable Document runtimeFields) throws IOException { - ObjectNode propertiesNode = objectNode.putObject(FIELD_PROPERTIES); + if (entity != null && entity.isAnnotationPresent(Mapping.class)) { + Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class); - writeTypeHintMapping(propertiesNode); + if (!mappingAnnotation.enabled()) { + objectNode.put(MAPPING_ENABLED, false); + return; + } - if (entity != null) { - entity.doWithProperties((PropertyHandler) property -> { - try { - if (property.isAnnotationPresent(Transient.class) || isInIgnoreFields(property, parentFieldAnnotation)) { - return; - } + if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) { + objectNode.put(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name())); + } - if (property.isSeqNoPrimaryTermProperty()) { - if (property.isAnnotationPresent(Field.class)) { - logger.warn("Property {} of {} is annotated for inclusion in mapping, but its type is " + // - "SeqNoPrimaryTerm that is never mapped, so it is skipped", // - property.getFieldName(), entity.getType()); - } - return; - } + if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) { + objectNode.put(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name())); + } - buildPropertyMapping(propertiesNode, isRootObject, property); - } catch (IOException e) { - logger.warn("error mapping property with name {}", property.getName(), e); + if (mappingAnnotation.dynamicDateFormats().length > 0) { + objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll(Arrays.stream(mappingAnnotation.dynamicDateFormats()) + .map(TextNode::valueOf).collect(Collectors.toList())); } - }); - } - } - @Nullable - private org.springframework.data.elasticsearch.core.document.Document getRuntimeFields( - @Nullable ElasticsearchPersistentEntity entity) { + if (runtimeFields != null) { + objectNode.set(RUNTIME, objectMapper.convertValue(runtimeFields, JsonNode.class)); + } + } - if (entity != null) { - Mapping mappingAnnotation = entity.findAnnotation(Mapping.class); - if (mappingAnnotation != null) { - String runtimeFieldsPath = mappingAnnotation.runtimeFieldsPath(); + boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField); + if (writeNestedProperties) { - if (hasText(runtimeFieldsPath)) { - String jsonString = ResourceUtil.readFileFromClasspath(runtimeFieldsPath); - return org.springframework.data.elasticsearch.core.document.Document.parse(jsonString); + String type = nestedOrObjectField ? fieldType.toString().toLowerCase() + : FieldType.Object.toString().toLowerCase(); + + ObjectNode nestedObjectNode = objectMapper.createObjectNode(); + nestedObjectNode.put(FIELD_PARAM_TYPE, type); + + if (nestedOrObjectField && FieldType.Nested == fieldType && parentFieldAnnotation != null + && parentFieldAnnotation.includeInParent()) { + nestedObjectNode.put(FIELD_INCLUDE_IN_PARENT, true); } + + objectNode.set(nestedObjectFieldName, nestedObjectNode); + // now go on with the nested one + objectNode = nestedObjectNode; } - } - return null; - } - private void buildPropertyMapping(ObjectNode propertiesNode, boolean isRootObject, - ElasticsearchPersistentProperty property) throws IOException { + if (entity != null && entity.dynamic() != Dynamic.INHERIT) { + objectNode.put(TYPE_DYNAMIC, entity.dynamic().name().toLowerCase()); + } else if (dynamicMapping != null) { + objectNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); + } - if (property.isAnnotationPresent(Mapping.class)) { + ObjectNode propertiesNode = objectNode.putObject(FIELD_PROPERTIES); - Mapping mapping = property.getRequiredAnnotation(Mapping.class); + writeTypeHintMapping(propertiesNode); - if (mapping.enabled()) { - String mappingPath = mapping.mappingPath(); + if (entity != null) { + entity.doWithProperties((PropertyHandler) property -> { + try { + if (property.isAnnotationPresent(Transient.class) || isInIgnoreFields(property, parentFieldAnnotation)) { + return; + } - if (StringUtils.hasText(mappingPath)) { + if (property.isSeqNoPrimaryTermProperty()) { + if (property.isAnnotationPresent(Field.class)) { + logger.warn("Property {} of {} is annotated for inclusion in mapping, but its type is " + // + "SeqNoPrimaryTerm that is never mapped, so it is skipped", // + property.getFieldName(), entity.getType()); + } + return; + } - ClassPathResource mappings = new ClassPathResource(mappingPath); - if (mappings.exists()) { - propertiesNode.putRawValue(property.getFieldName(), - new RawValue(StreamUtils.copyToString(mappings.getInputStream(), Charset.defaultCharset()))); - return; + buildPropertyMapping(propertiesNode, isRootObject, property); + } catch (IOException e) { + logger.warn("error mapping property with name {}", property.getName(), e); } - } - } else { - applyDisabledPropertyMapping(propertiesNode, property); - return; + }); } } - if (property.isGeoPointProperty()) { - applyGeoPointFieldMapping(propertiesNode, property); - return; - } + private void buildPropertyMapping(ObjectNode propertiesNode, boolean isRootObject, + ElasticsearchPersistentProperty property) throws IOException { - if (property.isGeoShapeProperty()) { - applyGeoShapeMapping(propertiesNode, property); - } + if (property.isAnnotationPresent(Mapping.class)) { - if (property.isJoinFieldProperty()) { - addJoinFieldMapping(propertiesNode, property); - } + Mapping mapping = property.getRequiredAnnotation(Mapping.class); - Field fieldAnnotation = property.findAnnotation(Field.class); - boolean isCompletionProperty = property.isCompletionProperty(); - boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property); - DynamicMapping dynamicMapping = property.findAnnotation(DynamicMapping.class); + if (mapping.enabled()) { + String mappingPath = mapping.mappingPath(); - if (!isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) { + if (StringUtils.hasText(mappingPath)) { - if (fieldAnnotation == null) { + ClassPathResource mappings = new ClassPathResource(mappingPath); + if (mappings.exists()) { + propertiesNode.putRawValue(property.getFieldName(), + new RawValue(StreamUtils.copyToString(mappings.getInputStream(), Charset.defaultCharset()))); + return; + } + } + } else { + applyDisabledPropertyMapping(propertiesNode, property); + return; + } + } + + if (property.isGeoPointProperty()) { + applyGeoPointFieldMapping(propertiesNode, property); return; } - if (isNestedOrObjectProperty) { - Iterator> iterator = property.getPersistentEntityTypes().iterator(); - ElasticsearchPersistentEntity persistentEntity = iterator.hasNext() - ? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next()) - : null; + if (property.isGeoShapeProperty()) { + applyGeoShapeMapping(propertiesNode, property); + } - mapEntity(propertiesNode, persistentEntity, false, property.getFieldName(), true, fieldAnnotation.type(), - fieldAnnotation, dynamicMapping, null); - return; + if (property.isJoinFieldProperty()) { + addJoinFieldMapping(propertiesNode, property); } - } - MultiField multiField = property.findAnnotation(MultiField.class); + String nestedPropertyPath = nestedPropertyPrefix.isEmpty() ? property.getFieldName() + : nestedPropertyPrefix + '.' + property.getFieldName(); - if (isCompletionProperty) { - CompletionField completionField = property.findAnnotation(CompletionField.class); - applyCompletionFieldMapping(propertiesNode, property, completionField); - } + Field fieldAnnotation = property.findAnnotation(Field.class); - if (isRootObject && fieldAnnotation != null && property.isIdProperty()) { - applyDefaultIdFieldMapping(propertiesNode, property); - } else if (multiField != null) { - addMultiFieldMapping(propertiesNode, property, multiField, isNestedOrObjectProperty, dynamicMapping); - } else if (fieldAnnotation != null) { - addSingleFieldMapping(propertiesNode, property, fieldAnnotation, isNestedOrObjectProperty, dynamicMapping); - } - } + if (fieldAnnotation != null && fieldAnnotation.excludeFromSource()) { + excludeFromSource.add(nestedPropertyPath); + } - private boolean hasRelevantAnnotation(ElasticsearchPersistentProperty property) { + boolean isCompletionProperty = property.isCompletionProperty(); + boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property); + DynamicMapping dynamicMapping = property.findAnnotation(DynamicMapping.class); - return property.findAnnotation(Field.class) != null || property.findAnnotation(MultiField.class) != null - || property.findAnnotation(GeoPointField.class) != null - || property.findAnnotation(CompletionField.class) != null; - } + if (!isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) { - private void applyGeoPointFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) - throws IOException { - propertiesNode.set(property.getFieldName(), - objectMapper.createObjectNode().put(FIELD_PARAM_TYPE, TYPE_VALUE_GEO_POINT)); - } + if (fieldAnnotation == null) { + return; + } - private void applyGeoShapeMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) - throws IOException { + if (isNestedOrObjectProperty) { + Iterator> iterator = property.getPersistentEntityTypes().iterator(); + ElasticsearchPersistentEntity persistentEntity = iterator.hasNext() + ? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next()) + : null; - ObjectNode shapeNode = propertiesNode.putObject(property.getFieldName()); - GeoShapeMappingParameters mappingParameters = GeoShapeMappingParameters - .from(property.findAnnotation(GeoShapeField.class)); - mappingParameters.writeTypeAndParametersTo(shapeNode); - } + String currentNestedPropertyPrefix = nestedPropertyPrefix; + nestedPropertyPrefix = nestedPropertyPath; - private void applyCompletionFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property, - @Nullable CompletionField annotation) throws IOException { + mapEntity(propertiesNode, persistentEntity, false, property.getFieldName(), true, fieldAnnotation.type(), + fieldAnnotation, dynamicMapping, null); - ObjectNode completionNode = propertyNode.putObject(property.getFieldName()); - completionNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_COMPLETION); + nestedPropertyPrefix = currentNestedPropertyPrefix; + return; + } + } - if (annotation != null) { - completionNode.put(COMPLETION_MAX_INPUT_LENGTH, annotation.maxInputLength()); - completionNode.put(COMPLETION_PRESERVE_POSITION_INCREMENTS, annotation.preservePositionIncrements()); - completionNode.put(COMPLETION_PRESERVE_SEPARATORS, annotation.preserveSeparators()); + MultiField multiField = property.findAnnotation(MultiField.class); - if (StringUtils.hasLength(annotation.searchAnalyzer())) { - completionNode.put(FIELD_PARAM_SEARCH_ANALYZER, annotation.searchAnalyzer()); + if (isCompletionProperty) { + CompletionField completionField = property.findAnnotation(CompletionField.class); + applyCompletionFieldMapping(propertiesNode, property, completionField); } - if (StringUtils.hasLength(annotation.analyzer())) { - completionNode.put(FIELD_PARAM_INDEX_ANALYZER, annotation.analyzer()); + if (isRootObject && fieldAnnotation != null && property.isIdProperty()) { + applyDefaultIdFieldMapping(propertiesNode, property); + } else if (multiField != null) { + addMultiFieldMapping(propertiesNode, property, multiField, isNestedOrObjectProperty, dynamicMapping); + } else if (fieldAnnotation != null) { + addSingleFieldMapping(propertiesNode, property, fieldAnnotation, isNestedOrObjectProperty, dynamicMapping); } + } - if (annotation.contexts().length > 0) { + private boolean hasRelevantAnnotation(ElasticsearchPersistentProperty property) { - ArrayNode contextsNode = completionNode.putArray(COMPLETION_CONTEXTS); - for (CompletionContext context : annotation.contexts()) { + return property.findAnnotation(Field.class) != null || property.findAnnotation(MultiField.class) != null + || property.findAnnotation(GeoPointField.class) != null + || property.findAnnotation(CompletionField.class) != null; + } - ObjectNode contextNode = contextsNode.addObject(); - contextNode.put(FIELD_CONTEXT_NAME, context.name()); - contextNode.put(FIELD_CONTEXT_TYPE, context.type().name().toLowerCase()); + private void applyGeoPointFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) + throws IOException { + propertiesNode.set(property.getFieldName(), + objectMapper.createObjectNode().put(FIELD_PARAM_TYPE, TYPE_VALUE_GEO_POINT)); + } - if (context.precision().length() > 0) { - contextNode.put(FIELD_CONTEXT_PRECISION, context.precision()); - } + private void applyGeoShapeMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) + throws IOException { - if (StringUtils.hasText(context.path())) { - contextNode.put(FIELD_CONTEXT_PATH, context.path()); - } - } - } + ObjectNode shapeNode = propertiesNode.putObject(property.getFieldName()); + GeoShapeMappingParameters mappingParameters = GeoShapeMappingParameters + .from(property.findAnnotation(GeoShapeField.class)); + mappingParameters.writeTypeAndParametersTo(shapeNode); } - } - private void applyDefaultIdFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property) - throws IOException { - propertyNode.set(property.getFieldName(), objectMapper.createObjectNode()// - .put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) // - .put(FIELD_INDEX, true) // - ); - } + private void applyCompletionFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property, + @Nullable CompletionField annotation) throws IOException { - private void applyDisabledPropertyMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) { + ObjectNode completionNode = propertyNode.putObject(property.getFieldName()); + completionNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_COMPLETION); - try { - Field field = property.getRequiredAnnotation(Field.class); + if (annotation != null) { + completionNode.put(COMPLETION_MAX_INPUT_LENGTH, annotation.maxInputLength()); + completionNode.put(COMPLETION_PRESERVE_POSITION_INCREMENTS, annotation.preservePositionIncrements()); + completionNode.put(COMPLETION_PRESERVE_SEPARATORS, annotation.preserveSeparators()); - if (field.type() != FieldType.Object) { - throw new IllegalArgumentException("Field type must be 'object"); + if (StringUtils.hasLength(annotation.searchAnalyzer())) { + completionNode.put(FIELD_PARAM_SEARCH_ANALYZER, annotation.searchAnalyzer()); + } + + if (StringUtils.hasLength(annotation.analyzer())) { + completionNode.put(FIELD_PARAM_INDEX_ANALYZER, annotation.analyzer()); + } + + if (annotation.contexts().length > 0) { + + ArrayNode contextsNode = completionNode.putArray(COMPLETION_CONTEXTS); + for (CompletionContext context : annotation.contexts()) { + + ObjectNode contextNode = contextsNode.addObject(); + contextNode.put(FIELD_CONTEXT_NAME, context.name()); + contextNode.put(FIELD_CONTEXT_TYPE, context.type().name().toLowerCase()); + + if (context.precision().length() > 0) { + contextNode.put(FIELD_CONTEXT_PRECISION, context.precision()); + } + + if (StringUtils.hasText(context.path())) { + contextNode.put(FIELD_CONTEXT_PATH, context.path()); + } + } + } } + } - propertiesNode.set(property.getFieldName(), objectMapper.createObjectNode() // - .put(FIELD_PARAM_TYPE, field.type().name().toLowerCase()) // - .put(MAPPING_ENABLED, false) // + private void applyDefaultIdFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property) + throws IOException { + propertyNode.set(property.getFieldName(), objectMapper.createObjectNode()// + .put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) // + .put(FIELD_INDEX, true) // ); - - } catch (Exception e) { - throw new MappingException("Could not write enabled: false mapping for " + property.getFieldName(), e); } - } - /** - * Add mapping for @Field annotation - * - * @throws IOException - */ - private void addSingleFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property, - Field annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException { + private void applyDisabledPropertyMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) { - // build the property json, if empty skip it as this is no valid mapping - ObjectNode fieldNode = objectMapper.createObjectNode(); - addFieldMappingParameters(fieldNode, annotation, nestedOrObjectField); + try { + Field field = property.getRequiredAnnotation(Field.class); - if (fieldNode.isEmpty()) { - return; - } + if (field.type() != FieldType.Object) { + throw new IllegalArgumentException("Field type must be 'object"); + } - propertiesNode.set(property.getFieldName(), fieldNode); + propertiesNode.set(property.getFieldName(), objectMapper.createObjectNode() // + .put(FIELD_PARAM_TYPE, field.type().name().toLowerCase()) // + .put(MAPPING_ENABLED, false) // + ); - if (nestedOrObjectField) { - if (annotation.dynamic() != Dynamic.INHERIT) { - fieldNode.put(TYPE_DYNAMIC, annotation.dynamic().name().toLowerCase()); - } else if (dynamicMapping != null) { - fieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); + } catch (Exception e) { + throw new MappingException("Could not write enabled: false mapping for " + property.getFieldName(), e); } } - } - - private void addJoinFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) - throws IOException { - JoinTypeRelation[] joinTypeRelations = property.getRequiredAnnotation(JoinTypeRelations.class).relations(); - if (joinTypeRelations.length == 0) { - logger.warn("Property {}s type is JoinField but its annotation JoinTypeRelation is " + // - "not properly maintained", // - property.getFieldName()); - return; - } + /** + * Add mapping for @Field annotation + * + * @throws IOException + */ + private void addSingleFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property, + Field annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException { - ObjectNode propertyNode = propertiesNode.putObject(property.getFieldName()); - propertyNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_JOIN); + // build the property json, if empty skip it as this is no valid mapping + ObjectNode fieldNode = objectMapper.createObjectNode(); + addFieldMappingParameters(fieldNode, annotation, nestedOrObjectField); - ObjectNode relationsNode = propertyNode.putObject(JOIN_TYPE_RELATIONS); + if (fieldNode.isEmpty()) { + return; + } - for (JoinTypeRelation joinTypeRelation : joinTypeRelations) { - String parent = joinTypeRelation.parent(); - String[] children = joinTypeRelation.children(); + propertiesNode.set(property.getFieldName(), fieldNode); - if (children.length > 1) { - relationsNode.putArray(parent) - .addAll(Arrays.stream(children).map(TextNode::valueOf).collect(Collectors.toList())); - } else if (children.length == 1) { - relationsNode.put(parent, children[0]); + if (nestedOrObjectField) { + if (annotation.dynamic() != Dynamic.INHERIT) { + fieldNode.put(TYPE_DYNAMIC, annotation.dynamic().name().toLowerCase()); + } else if (dynamicMapping != null) { + fieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); + } } } - } - /** - * Add mapping for @MultiField annotation - * - * @throws IOException - */ - private void addMultiFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property, - MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException { + private void addJoinFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) + throws IOException { + JoinTypeRelation[] joinTypeRelations = property.getRequiredAnnotation(JoinTypeRelations.class).relations(); - // main field - ObjectNode mainFieldNode = objectMapper.createObjectNode(); - propertyNode.set(property.getFieldName(), mainFieldNode); + if (joinTypeRelations.length == 0) { + logger.warn("Property {}s type is JoinField but its annotation JoinTypeRelation is " + // + "not properly maintained", // + property.getFieldName()); + return; + } - if (nestedOrObjectField) { - if (annotation.mainField().dynamic() != Dynamic.INHERIT) { - mainFieldNode.put(TYPE_DYNAMIC, annotation.mainField().dynamic().name().toLowerCase()); - } else if (dynamicMapping != null) { - mainFieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); + ObjectNode propertyNode = propertiesNode.putObject(property.getFieldName()); + propertyNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_JOIN); + + ObjectNode relationsNode = propertyNode.putObject(JOIN_TYPE_RELATIONS); + + for (JoinTypeRelation joinTypeRelation : joinTypeRelations) { + String parent = joinTypeRelation.parent(); + String[] children = joinTypeRelation.children(); + + if (children.length > 1) { + relationsNode.putArray(parent) + .addAll(Arrays.stream(children).map(TextNode::valueOf).collect(Collectors.toList())); + } else if (children.length == 1) { + relationsNode.put(parent, children[0]); + } } } - addFieldMappingParameters(mainFieldNode, annotation.mainField(), nestedOrObjectField); + /** + * Add mapping for @MultiField annotation + * + * @throws IOException + */ + private void addMultiFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property, + MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) + throws IOException { + + // main field + ObjectNode mainFieldNode = objectMapper.createObjectNode(); + propertyNode.set(property.getFieldName(), mainFieldNode); + + if (nestedOrObjectField) { + if (annotation.mainField().dynamic() != Dynamic.INHERIT) { + mainFieldNode.put(TYPE_DYNAMIC, annotation.mainField().dynamic().name().toLowerCase()); + } else if (dynamicMapping != null) { + mainFieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase()); + } + } - // inner fields - ObjectNode innerFieldsNode = mainFieldNode.putObject("fields"); + addFieldMappingParameters(mainFieldNode, annotation.mainField(), nestedOrObjectField); - for (InnerField innerField : annotation.otherFields()) { + // inner fields + ObjectNode innerFieldsNode = mainFieldNode.putObject("fields"); - ObjectNode innerFieldNode = innerFieldsNode.putObject(innerField.suffix()); - addFieldMappingParameters(innerFieldNode, innerField, false); + for (InnerField innerField : annotation.otherFields()) { + ObjectNode innerFieldNode = innerFieldsNode.putObject(innerField.suffix()); + addFieldMappingParameters(innerFieldNode, innerField, false); + + } } - } - private void addFieldMappingParameters(ObjectNode fieldNode, Annotation annotation, boolean nestedOrObjectField) - throws IOException { + private void addFieldMappingParameters(ObjectNode fieldNode, Annotation annotation, boolean nestedOrObjectField) + throws IOException { - MappingParameters mappingParameters = MappingParameters.from(annotation); + MappingParameters mappingParameters = MappingParameters.from(annotation); - if (!nestedOrObjectField && mappingParameters.isStore()) { - fieldNode.put(FIELD_PARAM_STORE, true); + if (!nestedOrObjectField && mappingParameters.isStore()) { + fieldNode.put(FIELD_PARAM_STORE, true); + } + mappingParameters.writeTypeAndParametersTo(fieldNode); } - mappingParameters.writeTypeAndParametersTo(fieldNode); - } - /** - * Apply mapping for dynamic templates. - * - * @throws IOException - */ - private void addDynamicTemplatesMapping(ObjectNode objectNode, ElasticsearchPersistentEntity entity) - throws IOException { + /** + * Apply mapping for dynamic templates. + * + * @throws IOException + */ + private void addDynamicTemplatesMapping(ObjectNode objectNode, ElasticsearchPersistentEntity entity) + throws IOException { - if (entity.isAnnotationPresent(DynamicTemplates.class)) { - String mappingPath = entity.getRequiredAnnotation(DynamicTemplates.class).mappingPath(); - if (hasText(mappingPath)) { + if (entity.isAnnotationPresent(DynamicTemplates.class)) { + String mappingPath = entity.getRequiredAnnotation(DynamicTemplates.class).mappingPath(); + if (hasText(mappingPath)) { - String jsonString = ResourceUtil.readFileFromClasspath(mappingPath); - if (hasText(jsonString)) { + String jsonString = ResourceUtil.readFileFromClasspath(mappingPath); + if (hasText(jsonString)) { - JsonNode jsonNode = objectMapper.readTree(jsonString).get("dynamic_templates"); - if (jsonNode != null && jsonNode.isArray()) { - objectNode.set(FIELD_DYNAMIC_TEMPLATES, jsonNode); + JsonNode jsonNode = objectMapper.readTree(jsonString).get("dynamic_templates"); + if (jsonNode != null && jsonNode.isArray()) { + objectNode.set(FIELD_DYNAMIC_TEMPLATES, jsonNode); + } } } } } - } - private boolean isAnyPropertyAnnotatedWithField(@Nullable ElasticsearchPersistentEntity entity) { + private boolean isAnyPropertyAnnotatedWithField(@Nullable ElasticsearchPersistentEntity entity) { - return entity != null && entity.getPersistentProperty(Field.class) != null; - } + return entity != null && entity.getPersistentProperty(Field.class) != null; + } - private boolean isInIgnoreFields(ElasticsearchPersistentProperty property, @Nullable Field parentFieldAnnotation) { + private boolean isInIgnoreFields(ElasticsearchPersistentProperty property, @Nullable Field parentFieldAnnotation) { - if (null != parentFieldAnnotation) { + if (null != parentFieldAnnotation) { - String[] ignoreFields = parentFieldAnnotation.ignoreFields(); - return Arrays.asList(ignoreFields).contains(property.getFieldName()); + String[] ignoreFields = parentFieldAnnotation.ignoreFields(); + return Arrays.asList(ignoreFields).contains(property.getFieldName()); + } + return false; } - return false; - } - private boolean isNestedOrObjectProperty(ElasticsearchPersistentProperty property) { + private boolean isNestedOrObjectProperty(ElasticsearchPersistentProperty property) { - Field fieldAnnotation = property.findAnnotation(Field.class); - return fieldAnnotation != null - && (FieldType.Nested == fieldAnnotation.type() || FieldType.Object == fieldAnnotation.type()); + Field fieldAnnotation = property.findAnnotation(Field.class); + return fieldAnnotation != null + && (FieldType.Nested == fieldAnnotation.type() || FieldType.Object == fieldAnnotation.type()); + } } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index d5d0e31b0..e41edbf6f 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -26,6 +26,7 @@ import java.lang.Object; import java.math.BigDecimal; import java.time.Instant; +import java.time.LocalDate; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -47,13 +48,13 @@ import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.MappingContextBaseTests; import org.springframework.data.elasticsearch.core.SearchHits; -import org.springframework.data.elasticsearch.core.suggest.Completion; import org.springframework.data.elasticsearch.core.geo.GeoPoint; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.IndexQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm; +import org.springframework.data.elasticsearch.core.suggest.Completion; import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.geo.Box; @@ -325,6 +326,16 @@ void shouldWriteRuntimeFields() { } + @Test // #796 + @DisplayName("should write source excludes") + void shouldWriteSourceExcludes() { + + IndexOperations indexOps = operations.indexOps(ExcludedFieldEntity.class); + indexOps.create(); + indexOps.putMapping(); + + } + // region entities @Document(indexName = "ignore-above-index") static class IgnoreAboveEntity { @@ -1172,6 +1183,18 @@ private static class RuntimeFieldEntity { @Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp; } + @Document(indexName = "fields-excluded-from-source") + private static class ExcludedFieldEntity { + @Id @Nullable private String id; + @Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date, + excludeFromSource = true) private LocalDate excludedDate; + @Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity; + } + + private static class NestedExcludedFieldEntity { + @Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText; + } + // endregion } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java index 06c8ded2b..90936e0e8 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java @@ -42,10 +42,10 @@ import org.springframework.data.annotation.Transient; import org.springframework.data.elasticsearch.annotations.*; import org.springframework.data.elasticsearch.core.MappingContextBaseTests; -import org.springframework.data.elasticsearch.core.suggest.Completion; import org.springframework.data.elasticsearch.core.geo.GeoPoint; import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm; +import org.springframework.data.elasticsearch.core.suggest.Completion; import org.springframework.data.geo.Box; import org.springframework.data.geo.Circle; import org.springframework.data.geo.Point; @@ -945,6 +945,49 @@ void shouldWriteRuntimeFields() throws JSONException { assertEquals(expected, mapping, true); } + + @Test // #796 + @DisplayName("should add fields that are excluded from source") + void shouldAddFieldsThatAreExcludedFromSource() throws JSONException { + + String expected = "{\n" + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " },\n" + // + " \"excluded-date\": {\n" + // + " \"type\": \"date\",\n" + // + " \"format\": \"date\"\n" + // + " },\n" + // + " \"nestedEntity\": {\n" + // + " \"type\": \"nested\",\n" + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " },\n" + // + " \"excluded-text\": {\n" + // + " \"type\": \"text\"\n" + // + " }\n" + // + " }\n" + // + " }\n" + // + " },\n" + // + " \"_source\": {\n" + // + " \"excludes\": [\n" + // + " \"excluded-date\",\n" + // + " \"nestedEntity.excluded-text\"\n" + // + " ]\n" + // + " }\n" + // + "}\n"; // + + String mapping = getMappingBuilder().buildPropertyMapping(ExcludedFieldEntity.class); + + assertEquals(expected, mapping, true); + } + // region entities @Document(indexName = "ignore-above-index") @@ -1918,5 +1961,17 @@ private static class RuntimeFieldEntity { @Id @Nullable private String id; @Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp; } + + @Document(indexName = "fields-excluded-from-source") + private static class ExcludedFieldEntity { + @Id @Nullable private String id; + @Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date, + excludeFromSource = true) private LocalDate excludedDate; + @Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity; + } + + private static class NestedExcludedFieldEntity { + @Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText; + } // endregion }