[Backport 7.x] Implement runtime fields on searches

docs/client-concepts/high-level/mapping/fluent-mapping.asciidoc

@@ -371,3 +371,59 @@ As demonstrated, by calling `.AutoMap()` inside of the `.Nested<Employee>` mappi
 `Employee` nested properties and again, override any inferred mapping from the automapping process,
 through manual mapping
 
+[[mapping-runtime-fields]]
+==== Mapping runtime fields
+
+A {ref_current}/runtime.html[runtime field] is a field that is evaluated at query time. Runtime fields may 
+be defined in the mapping of an index.
+
+In this example, we'll define a `CompanyRuntimeFields` class with a single property which we may then use in 
+the strongly-typed runtime field mapping.
+
+[source,csharp]
+----
+public class CompanyRuntimeFields
+{
+    public string BirthDayOfWeek { get; set; }
+}
+
+var createIndexResponse = _client.Indices.Create("myindex", c => c
+        .Map<Company>(m => m
+            .RuntimeFields<CompanyRuntimeFields>(rtf => rtf <1>
+                .RuntimeField(f => f.BirthDayOfWeek, FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"))) <2>
+        )
+    );
+----
+<1> Use the `CompanyRuntimeFields` class as the generic argument
+<2> Use the `BirthDayOfWeek` property as the runtime field name
+
+[source,javascript]
+----
+{
+  "mappings": {
+    "runtime": {
+      "birthDayOfWeek": {
+        "type": "keyword",
+        "script": {
+          "lang": "painless",
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+        }
+      }
+    }
+  }
+}
+----
+
+It's not necessary to define a type for the runtime field mapping. Runtime fields can optionally be defined
+by providing a `string` name.
+
+[source,csharp]
+----
+createIndexResponse = _client.Indices.Create("myindex", c => c
+    .Map<Company>(m => m
+        .RuntimeFields(rtf => rtf
+            .RuntimeField("birthDayOfWeek", FieldType.Keyword, f => f.Script("emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))")))
+    )
+);
+----
+

docs/high-level.asciidoc

@@ -211,6 +211,8 @@ include::search/scrolling-documents.asciidoc[]
 
 include::{output-dir}/covariant-hits/covariant-search-results.asciidoc[]
 
+include::search/searching-runtime-fields.asciidoc[]
+
 [[aggregations]]
 == Aggregations
 

docs/search/searching-runtime-fields.asciidoc

@@ -0,0 +1,150 @@
+:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/7.11
+
+:github: https://github.com/elastic/elasticsearch-net
+
+:nuget: https://www.nuget.org/packages
+
+////
+IMPORTANT NOTE
+==============
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/7.x/src/Tests/Tests/Search/SearchingRuntimeFields.doc.cs. 
+If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
+please modify the original csharp file found at the link and submit the PR with that change. Thanks!
+////
+
+[[searching-runtime-fields]]
+=== Searching runtime fields
+
+Runtime fields can be returned with search requests by specifying the fields using `.Fields`
+on the search request.
+
+[WARNING]
+--
+This functionality is in beta and is subject to change. The design and code is less mature 
+than official GA features and is being provided as-is with no warranties. Beta features 
+are not subject to the support SLA of official GA features.
+
+--
+
+[source,csharp]
+----
+var searchResponse = _client.Search<Project>(s => s
+    .Query(q => q
+        .MatchAll()
+    )
+    .Fields<ProjectRuntimeFields>(fs => fs
+        .Field(f => f.StartedOnDayOfWeek)
+        .Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
+    )
+);
+----
+
+which serializes to the following JSON
+
+[source,javascript]
+----
+{
+  "query": {
+    "match_all": {}
+  },
+  "fields": [
+    "runtime_started_on_day_of_week",
+    {
+      "field": "runtime_thirty_days_after_started",
+      "format": "basic_date"
+    }
+  ]
+}
+----
+
+The previous example used the Fluent API to express the query. NEST also exposes an
+Object Initializer syntax to compose queries
+
+[source,csharp]
+----
+var searchRequest = new SearchRequest<Project>
+{
+    Query = new MatchAllQuery(),
+    Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek) <1>
+        .And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date) <2>
+};
+
+searchResponse = _client.Search<Project>(searchRequest);
+----
+<1> Here we infer the field name from a property on a POCO class
+<2> For runtime fields which return a date, a format may be specified.
+
+==== Defining runtime fields
+
+You may define runtime fields that exist only as part of a query by specifying `.RuntimeFields` on
+the search request. You may return this field using `.Fields` or use it for an aggregation.
+
+[source,csharp]
+----
+var searchResponse = _client.Search<Project>(s => s
+    .Query(q => q
+        .MatchAll()
+    )
+    .Fields<ProjectRuntimeFields>(fs => fs
+        .Field(f => f.StartedOnDayOfWeek)
+        .Field(f => f.ThirtyDaysFromStarted, format: DateFormat.basic_date)
+        .Field("search_runtime_field")
+    )
+    .RuntimeFields(rtf => rtf.RuntimeField("search_runtime_field", FieldType.Keyword, r => r
+        .Script("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")))
+);
+----
+
+which yields the following query JSON
+
+[source,javascript]
+----
+{
+  "query": {
+    "match_all": {}
+  },
+  "fields": [
+    "runtime_started_on_day_of_week",
+    {
+      "field": "runtime_thirty_days_after_started",
+      "format": "basic_date"
+    },
+    "search_runtime_field"
+  ],
+  "runtime_mappings": {
+    "search_runtime_field": {
+      "script": {
+        "lang": "painless",
+        "source": "if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}"
+      },
+      "type": "keyword"
+    }
+  }
+}
+----
+
+The previous example used the Fluent API to express the query. Here is the same query using the
+Object Initializer syntax.
+
+[source,csharp]
+----
+var searchRequest = new SearchRequest<Project>
+{
+    Query = new MatchAllQuery(),
+    Fields = Infer.Field<ProjectRuntimeFields>(p => p.StartedOnDayOfWeek)
+        .And<ProjectRuntimeFields>(p => p.ThirtyDaysFromStarted, format: DateFormat.basic_date)
+        .And("search_runtime_field"),
+    RuntimeFields = new RuntimeFields
+    {
+        { "search_runtime_field", new RuntimeField
+            {
+                Type = FieldType.Keyword,
+                Script = new PainlessScript("if (doc['type'].size() != 0) {emit(doc['type'].value.toUpperCase())}")
+            }
+        }
+    }
+};
+
+searchResponse = _client.Search<Project>(searchRequest);
+----
+

src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs

@@ -155,9 +155,13 @@ public PutMappingDescriptor<TDocument> RoutingField(Func<RoutingFieldDescriptor<
 			Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<TDocument>()));
 
 		/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
-		public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
-			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
+		public PutMappingDescriptor<TDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
 
+		/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
+		public PutMappingDescriptor<TDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
+
 		/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
 		public PutMappingDescriptor<TDocument> FieldNamesField(Func<FieldNamesFieldDescriptor<TDocument>, IFieldNamesField> fieldNamesFieldSelector) =>
 			Assign(fieldNamesFieldSelector, (a, v) => a.FieldNamesField = v.Invoke(new FieldNamesFieldDescriptor<TDocument>()));

src/Nest/Mapping/RuntimeFields/RuntimeFields.cs

@@ -4,33 +4,40 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq.Expressions;
 using Elasticsearch.Net.Utf8Json;
 
 namespace Nest
 {
-	[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, string, IRuntimeField>))]
-	public interface IRuntimeFields : IIsADictionary<string, IRuntimeField> { }
+	[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<RuntimeFields, IRuntimeFields, Field, IRuntimeField>))]
+	public interface IRuntimeFields : IIsADictionary<Field, IRuntimeField> { }
 
-	public class RuntimeFields : IsADictionaryBase<string, IRuntimeField>, IRuntimeFields
+	public class RuntimeFields : IsADictionaryBase<Field, IRuntimeField>, IRuntimeFields
 	{
 		public RuntimeFields() { }
 
-		public RuntimeFields(IDictionary<string, IRuntimeField> container) : base(container) { }
+		public RuntimeFields(IDictionary<Field, IRuntimeField> container) : base(container) { }
 
-		public RuntimeFields(Dictionary<string, IRuntimeField> container) : base(container) { }
+		public RuntimeFields(Dictionary<Field, IRuntimeField> container) : base(container) { }
 
-		public void Add(string name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
+		public void Add(Field name, IRuntimeField runtimeField) => BackingDictionary.Add(name, runtimeField);
 	}
 
-	public class RuntimeFieldsDescriptor
-		: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor, RuntimeFields, string, IRuntimeField>
+	public class RuntimeFieldsDescriptor<TDocument>
+		: IsADictionaryDescriptorBase<RuntimeFieldsDescriptor<TDocument>, RuntimeFields, Field, IRuntimeField> where TDocument : class
 	{
 		public RuntimeFieldsDescriptor() : base(new RuntimeFields()) { }
 
-		public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
+		public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
 			Assign(name, selector?.Invoke(new RuntimeFieldDescriptor(type)));
 
-		public RuntimeFieldsDescriptor RuntimeField(string name, FieldType type) =>
+		public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type, Func<RuntimeFieldDescriptor, IRuntimeField> selector) =>
+			Assign(field, selector?.Invoke(new RuntimeFieldDescriptor(type)));
+
+		public RuntimeFieldsDescriptor<TDocument> RuntimeField(string name, FieldType type) =>
 			Assign(name, new RuntimeFieldDescriptor(type));
+
+		public RuntimeFieldsDescriptor<TDocument> RuntimeField(Expression<Func<TDocument, Field>> field, FieldType type) =>
+			Assign(field, new RuntimeFieldDescriptor(type));
 	}
 }

src/Nest/Mapping/TypeMapping.cs

@@ -269,9 +269,12 @@ public TypeMappingDescriptor<T> DisableIndexField(bool? disabled = true) =>
 		public TypeMappingDescriptor<T> RoutingField(Func<RoutingFieldDescriptor<T>, IRoutingField> routingFieldSelector) =>
 			Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor<T>()));
 
+		public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor<T>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<T>())?.Value);
+
 		/// <inheritdoc cref="ITypeMapping.RuntimeFields" />
-		public TypeMappingDescriptor<T> RuntimeFields(Func<RuntimeFieldsDescriptor, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
-			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor())?.Value);
+		public TypeMappingDescriptor<T> RuntimeFields<TDocument>(Func<RuntimeFieldsDescriptor<TDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TDocument : class =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TDocument>())?.Value);
 
 		/// <inheritdoc cref="ITypeMapping.FieldNamesField" />
 		public TypeMappingDescriptor<T> FieldNamesField(Func<FieldNamesFieldDescriptor<T>, IFieldNamesField> fieldNamesFieldSelector) =>

src/Nest/Search/Search/SearchRequest.cs

@@ -38,6 +38,13 @@ public partial interface ISearchRequest : ITypedSearchRequest
 		[DataMember(Name = "explain")]
 		bool? Explain { get; set; }
 
+		/// <summary>
+		/// BETA: Allows for retrieving a list of document fields in the search response.
+		/// <para>This functionality is in beta and is subject to change. </para>
+		/// </summary>
+		[DataMember(Name = "fields")]
+		Fields Fields { get; set; }
+
 		/// <summary>
 		/// The starting from index of the hits to return. Defaults to 0.
 		/// </summary>
@@ -172,6 +179,12 @@ public partial interface ISearchRequest : ITypedSearchRequest
 		/// </summary>
 		[DataMember(Name = "pit")]
 		IPointInTime PointInTime { get; set; }
+
+		/// <summary>
+		/// Specifies runtime fields which exist only as part of the query.
+		/// </summary>
+		[DataMember(Name = "runtime_mappings")]
+		IRuntimeFields RuntimeFields { get; set; }
 	}
 
 	[ReadAs(typeof(SearchRequest<>))]
@@ -198,6 +211,8 @@ public partial class SearchRequest
 		/// <inheritdoc />
 		public bool? Explain { get; set; }
 		/// <inheritdoc />
+		public Fields Fields { get; set; }
+		/// <inheritdoc />
 		public int? From { get; set; }
 		/// <inheritdoc />
 		public IHighlight Highlight { get; set; }
@@ -242,6 +257,8 @@ public partial class SearchRequest
 		public bool? Version { get; set; }
 		/// <inheritdoc />
 		public IPointInTime PointInTime { get; set; }
+		/// <inheritdoc />
+		public IRuntimeFields RuntimeFields { get; set; }
 
 		protected override HttpMethod HttpMethod =>
 			RequestState.RequestParameters?.ContainsQueryString("source") == true
@@ -285,6 +302,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
 		IFieldCollapse ISearchRequest.Collapse { get; set; }
 		Fields ISearchRequest.DocValueFields { get; set; }
 		bool? ISearchRequest.Explain { get; set; }
+		Fields ISearchRequest.Fields { get; set; }
 		int? ISearchRequest.From { get; set; }
 		IHighlight ISearchRequest.Highlight { get; set; }
 		IDictionary<IndexName, double> ISearchRequest.IndicesBoost { get; set; }
@@ -307,6 +325,7 @@ public partial class SearchDescriptor<TInferDocument> where TInferDocument : cla
 		bool? ISearchRequest.TrackTotalHits { get; set; }
 		bool? ISearchRequest.Version { get; set; }
 		IPointInTime ISearchRequest.PointInTime { get; set; }
+		IRuntimeFields ISearchRequest.RuntimeFields { get; set; }
 
 		protected sealed override void RequestDefaults(SearchRequestParameters parameters) => TypedKeys();
 
@@ -335,6 +354,17 @@ public SearchDescriptor<TInferDocument> Source(Func<SourceFilterDescriptor<TInfe
 		/// <inheritdoc cref="ISearchRequest.Size" />
 		public SearchDescriptor<TInferDocument> Take(int? take) => Size(take);
 
+		/// <inheritdoc cref="ISearchRequest.Fields" />
+		public SearchDescriptor<TInferDocument> Fields(Func<FieldsDescriptor<TInferDocument>, IPromise<Fields>> fields) =>
+			Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TInferDocument>())?.Value);
+
+		/// <inheritdoc cref="ISearchRequest.Fields" />
+		public SearchDescriptor<TInferDocument> Fields<TSource>(Func<FieldsDescriptor<TSource>, IPromise<Fields>> fields) where TSource : class =>
+			Assign(fields, (a, v) => a.Fields = v?.Invoke(new FieldsDescriptor<TSource>())?.Value);
+
+		/// <inheritdoc cref="ISearchRequest.Fields" />
+		public SearchDescriptor<TInferDocument> Fields(Fields fields) => Assign(fields, (a, v) => a.DocValueFields = v);
+
 		/// <inheritdoc cref="ISearchRequest.From" />
 		public SearchDescriptor<TInferDocument> From(int? from) => Assign(from, (a, v) => a.From = v);
 
@@ -476,6 +506,14 @@ public SearchDescriptor<TInferDocument> PointInTime(string pitId)
 		public SearchDescriptor<TInferDocument> PointInTime(string pitId, Func<PointInTimeDescriptor, IPointInTime> pit) =>
 			Assign(pit, (a, v) => a.PointInTime = v?.Invoke(new PointInTimeDescriptor(pitId)));
 
+		/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
+		public SearchDescriptor<TInferDocument> RuntimeFields(Func<RuntimeFieldsDescriptor<TInferDocument>, IPromise<IRuntimeFields>> runtimeFieldsSelector) =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TInferDocument>())?.Value);
+
+		/// <inheritdoc cref="ISearchRequest.RuntimeFields" />
+		public SearchDescriptor<TInferDocument> RuntimeFields<TSource>(Func<RuntimeFieldsDescriptor<TSource>, IPromise<IRuntimeFields>> runtimeFieldsSelector) where TSource : class =>
+			Assign(runtimeFieldsSelector, (a, v) => a.RuntimeFields = v?.Invoke(new RuntimeFieldsDescriptor<TSource>())?.Value);
+
 		protected override string ResolveUrl(RouteValues routeValues, IConnectionSettingsValues settings)
 		{
 			if (Self.PointInTime is object && !string.IsNullOrEmpty(Self.PointInTime.Id) && routeValues.ContainsKey("index"))