Skip to content

Commit e119483

Browse files
github-actions[bot]stevejgordon
github-actions[bot]
and
stevejgordon
authored
Implement runtime fields on searches (#5259) (#5266)
* Implement runtime fields on searches * Add runtime fields documentation Co-authored-by: Steve Gordon <sgordon@hotmail.co.uk>
1 parent 981b73c commit e119483

File tree

12 files changed

+630
-31
lines changed

12 files changed

+630
-31
lines changed

‎docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,3 +371,59 @@ As demonstrated, by calling `.AutoMap()` inside of the `.Nested<Employee>` mappi
371371
`Employee` nested properties and again, override any inferred mapping from the automapping process,
372372
through manual mapping
373373

374+
[[mapping-runtime-fields]]
375+
==== Mapping runtime fields
376+
377+
A {ref_current}/runtime.html[runtime field] is a field that is evaluated at query time. Runtime fields may
378+
be defined in the mapping of an index.
379+
380+
In this example, we'll define a `CompanyRuntimeFields` class with a single property which we may then use in
381+
the strongly-typed runtime field mapping.
382+
383+
[source,csharp]
384+
----
385+
public class CompanyRuntimeFields
386+
{
387+
public string BirthDayOfWeek { get; set; }
388+
}
389+
390+
var createIndexResponse = _client.Indices.Create("myindex", c => c
391+
.Map<Company>(m => m
392+
.RuntimeFields<CompanyRuntimeFields>(rtf => rtf <1>
393+
.RuntimeField(f => f.BirthDayOfWeek, FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"))) <2>
394+
)
395+
);
396+
----
397+
<1> Use the `CompanyRuntimeFields` class as the generic argument
398+
<2> Use the `BirthDayOfWeek` property as the runtime field name
399+
400+
[source,javascript]
401+
----
402+
{
403+
"mappings": {
404+
"runtime": {
405+
"birthDayOfWeek": {
406+
"type": "keyword",
407+
"script": {
408+
"lang": "painless",
409+
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
410+
}
411+
}
412+
}
413+
}
414+
}
415+
----
416+
417+
It's not necessary to define a type for the runtime field mapping. Runtime fields can optionally be defined
418+
by providing a `string` name.
419+
420+
[source,csharp]
421+
----
422+
createIndexResponse = _client.Indices.Create("myindex", c => c
423+
.Map<Company>(m => m
424+
.RuntimeFields(rtf => rtf
425+
.RuntimeField("birthDayOfWeek", FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))")))
426+
)
427+
);
428+
----
429+

‎docs/high-level.asciidoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ include::search/scrolling-documents.asciidoc[]
211211

212212
include::{output-dir}/covariant-hits/covariant-search-results.asciidoc[]
213213

214+
include::search/searching-runtime-fields.asciidoc[]
215+
214216
[[aggregations]]
215217
== Aggregations
216218

‎docs/search/searching-runtime-fields.asciidoc

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/7.11
2+
3+
:github: https://github.com/elastic/elasticsearch-net
4+
5+
:nuget: https://www.nuget.org/packages
6+
7+
////
8+
IMPORTANT NOTE
9+
==============
10+
This file has been generated from https://github.com/elastic/elasticsearch-net/tree/7.x/src/Tests/Tests/Search/SearchingRuntimeFields.doc.cs.
11+
If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
12+
please modify the original csharp file found at the link and submit the PR with that change. Thanks!
13+
////
14+
15+
[[searching-runtime-fields]]
16+
=== Searching runtime fields
17+
18+
Runtime fields can be returned with search requests by specifying the fields using `.Fields`
19+
on the search request.
20+
21+
[WARNING]
22+
--
23+
This functionality is in beta and is subject to change. The design and code is less mature
24+
than official GA features and is being provided as-is with no warranties. Beta features
25+
are not subject to the support SLA of official GA features.
26+
27+
--
28+
29+
[source,csharp]
30+
----
31+
var searchResponse = _client.Search<Project>(s => s
32+
.Query(q => q
33+
.MatchAll()
34+
)
35+
.Fields<ProjectRuntimeFields>(fs => fs
36+
.Field(f => f.StartedOnDayOfWeek)
37+
.Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
38+
)
39+
);
40+
----
41+
42+
which serializes to the following JSON
43+
44+
[source,javascript]
45+
----
46+
{
47+
"query": {
48+
"match_all": {}
49+
},
50+
"fields": [
51+
"runtime_started_on_day_of_week",
52+
{
53+
"field": "runtime_thirty_days_after_started",
54+
"format": "basic_date"
55+
}
56+
]
57+
}
58+
----
59+
60+
The previous example used the Fluent API to express the query. NEST also exposes an
61+
Object Initializer syntax to compose queries
62+
63+
[source,csharp]
64+
----
65+
var searchRequest = new SearchRequest<Project>
66+
{
67+
Query = new MatchAllQuery(),
68+
Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek) <1>
69+
.And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date) <2>
70+
};
71+
72+
searchResponse = _client.Search<Project>(searchRequest);
73+
----
74+
<1> Here we infer the field name from a property on a POCO class
75+
<2> For runtime fields which return a date, a format may be specified.
76+
77+
==== Defining runtime fields
78+
79+
You may define runtime fields that exist only as part of a query by specifying `.RuntimeFields` on
80+
the search request. You may return this field using `.Fields` or use it for an aggregation.
81+
82+
[source,csharp]
83+
----
84+
var searchResponse = _client.Search<Project>(s => s
85+
.Query(q => q
86+
.MatchAll()
87+
)
88+
.Fields<ProjectRuntimeFields>(fs => fs
89+
.Field(f => f.StartedOnDayOfWeek)
90+
.Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
91+
.Field("search_runtime_field")
92+
)
93+
.RuntimeFields(rtf => rtf.RuntimeField("search_runtime_field", FieldType.Keyword, r => r
94+
.Script("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")))
95+
);
96+
----
97+
98+
which yields the following query JSON
99+
100+
[source,javascript]
101+
----
102+
{
103+
"query": {
104+
"match_all": {}
105+
},
106+
"fields": [
107+
"runtime_started_on_day_of_week",
108+
{
109+
"field": "runtime_thirty_days_after_started",
110+
"format": "basic_date"
111+
},
112+
"search_runtime_field"
113+
],
114+
"runtime_mappings": {
115+
"search_runtime_field": {
116+
"script": {
117+
"lang": "painless",
118+
"source": "if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}"
119+
},
120+
"type": "keyword"
121+
}
122+
}
123+
}
124+
----
125+
126+
The previous example used the Fluent API to express the query. Here is the same query using the
127+
Object Initializer syntax.
128+
129+
[source,csharp]
130+
----
131+
var searchRequest = new SearchRequest<Project>
132+
{
133+
Query = new MatchAllQuery(),
134+
Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek)
135+
.And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date)
136+
.And("search_runtime_field"),
137+
RuntimeFields = new RuntimeFields
138+
{
139+
{ "search_runtime_field", new RuntimeField
140+
{
141+
Type = FieldType.Keyword,
142+
Script = new PainlessScript("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")
143+
}
144+
}
145+
}
146+
};
147+
148+
searchResponse = _client.Search<Project>(searchRequest);
149+
----
150+

‎src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,13 @@ public PutMappingDescriptor<TDocument> RoutingField(Func<RoutingFieldDescriptor<
155155
Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<TDocument>()));
156156

157157
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
158-
public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
159-
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
158+
public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
159+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
160160

161+
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
162+
public PutMappingDescriptor<TDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
163+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
164+
161165
/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
162166
public PutMappingDescriptor<TDocument> FieldNamesField(Func<FieldNamesFieldDescriptor<TDocument>, IFieldNamesField> fieldNamesFieldSelector) =>
163167
Assign(fieldNamesFieldSelector, (a, v) => a.FieldNamesField = v.Invoke(new FieldNamesFieldDescriptor<TDocument>()));

‎src/Nest/Mapping/RuntimeFields/RuntimeFields.cs

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,40 @@
44

55
using System;
66
using System.Collections.Generic;
7+
using System.Linq.Expressions;
78
using Elasticsearch.Net.Utf8Json;
89

910
namespace Nest
1011
{
11-
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, string, IRuntimeField>))]
12-
public interface IRuntimeFields : IIsADictionary<string, IRuntimeField> { }
12+
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, Field, IRuntimeField>))]
13+
public interface IRuntimeFields : IIsADictionary<Field, IRuntimeField> { }
1314

14-
public class RuntimeFields : IsADictionaryBase<string, IRuntimeField>, IRuntimeFields
15+
public class RuntimeFields : IsADictionaryBase<Field, IRuntimeField>, IRuntimeFields
1516
{
1617
public RuntimeFields() { }
1718

18-
public RuntimeFields(IDictionary<string, IRuntimeField> container) : base(container) { }
19+
public RuntimeFields(IDictionary<Field, IRuntimeField> container) : base(container) { }
1920

20-
public RuntimeFields(Dictionary<string, IRuntimeField> container) : base(container) { }
21+
public RuntimeFields(Dictionary<Field, IRuntimeField> container) : base(container) { }
2122

22-
public void Add(string name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
23+
public void Add(Field name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
2324
}
2425

25-
public class RuntimeFieldsDescriptor
26-
: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor, RuntimeFields, string, IRuntimeField>
26+
public class RuntimeFieldsDescriptor<TDocument>
27+
: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor<TDocument>, RuntimeFields, Field, IRuntimeField> where TDocument : class
2728
{
2829
public RuntimeFieldsDescriptor() : base(new RuntimeFields()) { }
2930

30-
public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
31+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
3132
Assign(name, selector?.Invoke(new RuntimeFieldDescriptor(type)));
3233

33-
public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type) =>
34+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
35+
Assign(field, selector?.Invoke(new RuntimeFieldDescriptor(type)));
36+
37+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type) =>
3438
Assign(name, new RuntimeFieldDescriptor(type));
39+
40+
public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type) =>
41+
Assign(field, new RuntimeFieldDescriptor(type));
3542
}
3643
}

‎src/Nest/Mapping/TypeMapping.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -269,9 +269,12 @@ public TypeMappingDescriptor<T> DisableIndexField(bool? disabled = true) =>
269269
public TypeMappingDescriptor<T> RoutingField(Func<RoutingFieldDescriptor<T>, IRoutingField> routingFieldSelector) =>
270270
Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<T>()));
271271

272+
public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor<T>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
273+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<T>())?.Value);
274+
272275
/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
273-
public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
274-
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
276+
public TypeMappingDescriptor<T> RuntimeFields<TDocument>(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TDocument : class =>
277+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
275278

276279
/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
277280
public TypeMappingDescriptor<T> FieldNamesField(Func<FieldNamesFieldDescriptor<T>, IFieldNamesField> fieldNamesFieldSelector) =>

‎src/Nest/Search/Search/SearchRequest.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ public partial interface ISearchRequest : ITypedSearchRequest
3838
[DataMember(Name = "explain")]
3939
bool? Explain { get; set; }
4040

41+
/// <summary>
42+
/// BETA: Allows for retrieving a list of document fields in the search response.
43+
/// <para>This functionality is in beta and is subject to change. </para>
44+
/// </summary>
45+
[DataMember(Name = "fields")]
46+
Fields Fields { get; set; }
47+
4148
/// <summary>
4249
/// The starting from index of the hits to return. Defaults to 0.
4350
/// </summary>
@@ -172,6 +179,12 @@ public partial interface ISearchRequest : ITypedSearchRequest
172179
/// </summary>
173180
[DataMember(Name = "pit")]
174181
IPointInTime PointInTime { get; set; }
182+
183+
/// <summary>
184+
/// Specifies runtime fields which exist only as part of the query.
185+
/// </summary>
186+
[DataMember(Name = "runtime_mappings")]
187+
IRuntimeFields RuntimeFields { get; set; }
175188
}
176189

177190
[ReadAs(typeof(SearchRequest<>))]
@@ -198,6 +211,8 @@ public partial class SearchRequest
198211
/// <inheritdoc />
199212
public bool? Explain { get; set; }
200213
/// <inheritdoc />
214+
public Fields Fields { get; set; }
215+
/// <inheritdoc />
201216
public int? From { get; set; }
202217
/// <inheritdoc />
203218
public IHighlight Highlight { get; set; }
@@ -242,6 +257,8 @@ public partial class SearchRequest
242257
public bool? Version { get; set; }
243258
/// <inheritdoc />
244259
public IPointInTime PointInTime { get; set; }
260+
/// <inheritdoc />
261+
public IRuntimeFields RuntimeFields { get; set; }
245262

246263
protected override HttpMethod HttpMethod =>
247264
RequestState.RequestParameters?.ContainsQueryString("source") == true
@@ -285,6 +302,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
285302
IFieldCollapse ISearchRequest.Collapse { get; set; }
286303
Fields ISearchRequest.DocValueFields { get; set; }
287304
bool? ISearchRequest.Explain { get; set; }
305+
Fields ISearchRequest.Fields { get; set; }
288306
int? ISearchRequest.From { get; set; }
289307
IHighlight ISearchRequest.Highlight { get; set; }
290308
IDictionary<IndexName, double> ISearchRequest.IndicesBoost { get; set; }
@@ -307,6 +325,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
307325
bool? ISearchRequest.TrackTotalHits { get; set; }
308326
bool? ISearchRequest.Version { get; set; }
309327
IPointInTime ISearchRequest.PointInTime { get; set; }
328+
IRuntimeFields ISearchRequest.RuntimeFields { get; set; }
310329

311330
protected sealed override void RequestDefaults(SearchRequestParameters parameters) => TypedKeys();
312331

@@ -335,6 +354,17 @@ public SearchDescriptor<TInferDocument> Source(Func<SourceFilterDescriptor<TInfe
335354
/// <inheritdoc cref="ISearchRequest.Size" />
336355
public SearchDescriptor<TInferDocument> Take(int? take) => Size(take);
337356

357+
/// <inheritdoc cref="ISearchRequest.Fields" />
358+
public SearchDescriptor<TInferDocument> Fields(Func<FieldsDescriptor<TInferDocument>, IPromise<Fields>> fields) =>
359+
Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TInferDocument>())?.Value);
360+
361+
/// <inheritdoc cref="ISearchRequest.Fields" />
362+
public SearchDescriptor<TInferDocument> Fields<TSource>(Func<FieldsDescriptor<TSource>, IPromise<Fields>> fields) where TSource : class =>
363+
Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TSource>())?.Value);
364+
365+
/// <inheritdoc cref="ISearchRequest.Fields" />
366+
public SearchDescriptor<TInferDocument> Fields(Fields fields) => Assign(fields, (a, v) => a.DocValueFields = v);
367+
338368
/// <inheritdoc cref="ISearchRequest.From" />
339369
public SearchDescriptor<TInferDocument> From(int? from) => Assign(from, (a, v) => a.From = v);
340370

@@ -476,6 +506,14 @@ public SearchDescriptor<TInferDocument> PointInTime(string pitId)
476506
public SearchDescriptor<TInferDocument> PointInTime(string pitId, Func<PointInTimeDescriptor, IPointInTime> pit) =>
477507
Assign(pit, (a, v) => a.PointInTime = v?.Invoke(new PointInTimeDescriptor(pitId)));
478508

509+
/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
510+
public SearchDescriptor<TInferDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TInferDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
511+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TInferDocument>())?.Value);
512+
513+
/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
514+
public SearchDescriptor<TInferDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
515+
Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
516+
479517
protected override string ResolveUrl(RouteValues routeValues, IConnectionSettingsValues settings)
480518
{
481519
if (Self.PointInTime is object && !string.IsNullOrEmpty(Self.PointInTime.Id) && routeValues.ContainsKey("index"))

0 commit comments

Comments
 (0)