Add metadata header on requests (#5190)

Implement meta header for client requests

Author: stevejgordon

docs/client-concepts/connection/configuration-options.asciidoc

@@ -64,6 +64,10 @@ Ensures the response bytes are always available on the `ElasticsearchResponse<T>
 +
 IMPORTANT: Depending on the registered serializer, this may cause the response to be buffered in memory first, potentially affecting performance.
 
+`DisableMetaHeader`::
+
+Disables the meta header which is included on all requests by default. This header contains lightweight information  about the client and runtime.
+
 `DisablePing`::
 
 When a node is used for the very first time or when it's used for the first time after it has been marked dead a ping with a very low timeout is send to the node to make sure that when it's still dead it reports it as fast as possible. You can disable these pings globally here if you rather have it fail on the possible slower original request

src/Elasticsearch.Net/Api/RequestParameters/RequestParametersExtensions.cs

@@ -0,0 +1,24 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+
+namespace Elasticsearch.Net
+{
+	internal static class RequestParametersExtensions
+	{
+		internal static void SetRequestMetaData(this IRequestParameters parameters, RequestMetaData requestMetaData)
+		{
+			if (parameters is null)
+				throw new ArgumentNullException(nameof(parameters));
+
+			if (requestMetaData is null)
+				throw new ArgumentNullException(nameof(requestMetaData));
+
+			parameters.RequestConfiguration ??= new RequestConfiguration();
+
+			parameters.RequestConfiguration.RequestMetaData = requestMetaData;
+		}
+	}
+}

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -33,36 +33,7 @@ public class ConnectionConfiguration : ConnectionConfiguration<ConnectionConfigu
 		/// As the old curl based handler is known to bleed TCP connections:
 		/// <para>https://github.com/dotnet/runtime/issues/22366</para>
 		/// </summary>
-        private static bool UsingCurlHandler
-		{
-			get
-			{
-#if !DOTNETCORE
-				return false;
-#else
-				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
-				if (!curlHandlerExists) return false;
-
-				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
-				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
-				// Must be using CurlHandler.
-				if (!socketsHandlerExists) return true;
-
-				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
-					return !isEnabled;
-
-				var environmentVariable =
-					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
-
-				// SocketsHandler exists and no environment variable exists to disable it.
-				// Must be using SocketsHandler and not CurlHandler
-				if (environmentVariable == null) return false;
-
-				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
-					environmentVariable.Equals("0");
-#endif
-			}
-		}
+		private static bool UsingCurlHandler => ConnectionInfo.UsingCurlHandler;
 
 		/// <summary>
 		/// The default ping timeout. Defaults to 2 seconds
@@ -153,7 +124,6 @@ public ConnectionConfiguration(IConnectionPool connectionPool, IElasticsearchSer
 		/// <param name="serializer">A serializer implementation used to serialize requests and deserialize responses</param>
 		public ConnectionConfiguration(IConnectionPool connectionPool, IConnection connection, IElasticsearchSerializer serializer)
 			: base(connectionPool, connection, serializer) { }
-
 	}
 
 	[Browsable(false)]
@@ -176,6 +146,7 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private TimeSpan? _deadTimeout;
 		private bool _disableAutomaticProxyDetection;
 		private bool _disableDirectStreaming;
+		private bool _disableMetaHeader;
 		private bool _disablePings;
 		private bool _enableHttpCompression;
 		private bool _enableHttpPipelining = true;
@@ -207,7 +178,7 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private bool _enableThreadPoolStats;
 
 		private string _userAgent = ConnectionConfiguration.DefaultUserAgent;
-		private Func<HttpMethod, int, bool> _statusCodeToResponseSuccess;
+		private readonly Func<HttpMethod, int, bool> _statusCodeToResponseSuccess;
 
 		protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection connection, IElasticsearchSerializer requestResponseSerializer)
 		{
@@ -234,7 +205,6 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 				_apiKeyAuthCredentials = cloudPool.ApiKeyCredentials;
 				_enableHttpCompression = true;
 			}
-
 		}
 
 		protected IElasticsearchSerializer UseThisRequestResponseSerializer { get; set; }
@@ -248,6 +218,7 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		TimeSpan? IConnectionConfigurationValues.DeadTimeout => _deadTimeout;
 		bool IConnectionConfigurationValues.DisableAutomaticProxyDetection => _disableAutomaticProxyDetection;
 		bool IConnectionConfigurationValues.DisableDirectStreaming => _disableDirectStreaming;
+		bool IConnectionConfigurationValues.DisableMetaHeader => _disableMetaHeader;
 		bool IConnectionConfigurationValues.DisablePings => _disablePings;
 		bool IConnectionConfigurationValues.EnableHttpCompression => _enableHttpCompression;
 		NameValueCollection IConnectionConfigurationValues.Headers => _headers;
@@ -286,6 +257,8 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		bool IConnectionConfigurationValues.TransferEncodingChunked => _transferEncodingChunked;
 		bool IConnectionConfigurationValues.EnableTcpStats => _enableTcpStats;
 		bool IConnectionConfigurationValues.EnableThreadPoolStats => _enableThreadPoolStats;
+		
+		MetaHeaderProvider IConnectionConfigurationValues.MetaHeaderProvider { get; } = new MetaHeaderProvider();
 
 		void IDisposable.Dispose() => DisposeManagedResources();
 
@@ -368,6 +341,12 @@ public T SniffOnConnectionFault(bool sniffsOnConnectionFault = true) =>
 		/// </summary>
 		public T DisableAutomaticProxyDetection(bool disable = true) => Assign(disable, (a, v) => a._disableAutomaticProxyDetection = v);
 
+		/// <summary>
+		/// Disables the meta header which is included on all requests by default. This header contains lightweight information 
+		/// about the client and runtime.
+		/// </summary>
+		public T DisableMetaHeader(bool disable = true) => Assign(disable, (a, v) => a._disableMetaHeader = v);
+
 		/// <summary>
 		/// Instead of following a c/go like error checking on response.IsValid do throw an exception (except when <see cref="IApiCallDetails.SuccessOrKnownError"/> is false)
 		/// on the client when a call resulted in an exception on either the client or the Elasticsearch server.
@@ -432,11 +411,11 @@ public T SniffOnConnectionFault(bool sniffsOnConnectionFault = true) =>
 
 		/// <summary>
 		/// DnsRefreshTimeout for the connections. Defaults to 5 minutes.
-		#if DOTNETCORE
+#if DOTNETCORE
 		/// <para>Will create new instances of <see cref="System.Net.Http.HttpClient"/> after this timeout to force DNS updates</para>
-		#else
+#else
 		/// <para>Will set both <see cref="System.Net.ServicePointManager.DnsRefreshTimeout"/> and <see cref="System.Net.ServicePointManager.ConnectionLeaseTimeout "/>
-		#endif
+#endif
 		/// </summary>
 		public T DnsRefreshTimeout(TimeSpan timeout) => Assign(timeout, (a, v) => a._dnsRefreshTimeout = v);
 

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -77,6 +77,8 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// </summary>
 		bool DisableDirectStreaming { get; }
 
+		bool DisableMetaHeader { get; }
+
 		/// <summary>
 		/// This signals that we do not want to send initial pings to unknown/previously dead nodes
 		/// and just send the call straightaway
@@ -273,5 +275,10 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// Enable statistics about thread pools to be collected when making a request
 		/// </summary>
 		bool EnableThreadPoolStats { get; }
+
+		/// <summary>
+		/// Produces the client meta header for a request.
+		/// </summary>
+		MetaHeaderProvider MetaHeaderProvider { get; }
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -126,6 +126,11 @@ public interface IRequestConfiguration
 
 		/// <inheritdoc cref="IConnectionConfigurationValues.EnableThreadPoolStats"/>
 		bool? EnableThreadPoolStats { get; set; }
+
+		/// <summary>
+		/// Holds additional meta data about the request.
+		/// </summary>
+		RequestMetaData RequestMetaData { get; set; }
 	}
 
 	public class RequestConfiguration : IRequestConfiguration
@@ -172,6 +177,8 @@ public class RequestConfiguration : IRequestConfiguration
 		public bool? EnableTcpStats { get; set; }
 		/// <inheritdoc />
 		public bool? EnableThreadPoolStats { get; set; }
+		/// <inheritdoc />
+		public RequestMetaData RequestMetaData { get; set; }
 	}
 
 	public class RequestConfigurationDescriptor : IRequestConfiguration
@@ -223,6 +230,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 		NameValueCollection IRequestConfiguration.Headers { get; set; }
 		bool? IRequestConfiguration.EnableTcpStats { get; set; }
 		bool? IRequestConfiguration.EnableThreadPoolStats { get; set; }
+		RequestMetaData IRequestConfiguration.RequestMetaData { get; set; }
 
 		/// <summary>
 		/// Submit the request on behalf in the context of a different shield user
@@ -406,5 +414,12 @@ public RequestConfigurationDescriptor EnableThreadPoolStats(bool? enableThreadPo
 			Self.EnableThreadPoolStats = enableThreadPoolStats;
 			return this;
 		}
+
+		/// <inheritdoc cref="IRequestConfiguration.RequestMetaData" />
+		internal RequestConfigurationDescriptor RequestMetaData(RequestMetaData metaData)
+		{
+			Self.RequestMetaData = metaData;
+			return this;
+		}
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfigurationExtensions.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+
+namespace Elasticsearch.Net
+{
+	internal static class RequestConfigurationExtensions
+	{
+		internal static void SetRequestMetaData(this IRequestConfiguration requestConfiguration, RequestMetaData requestMetaData)
+		{
+			if (requestConfiguration is null)
+				throw new ArgumentNullException(nameof(requestConfiguration));
+
+			if (requestMetaData is null)
+				throw new ArgumentNullException(nameof(requestMetaData));
+
+			requestConfiguration.RequestMetaData = requestMetaData;
+		}
+	}
+}

src/Elasticsearch.Net/Configuration/RequestMetaData.cs

@@ -0,0 +1,34 @@
+using System.Collections.Generic;
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// Holds meta data about a client request.
+	/// </summary>
+	public sealed class RequestMetaData
+	{
+		/// <summary>
+		/// Reserved key for a meta data entry which identifies the helper which produced the request.
+		/// </summary>
+		internal const string HelperKey = "helper";
+
+		private Dictionary<string, string> _metaDataItems;
+
+		internal bool TryAddMetaData (string key, string value)
+		{
+			_metaDataItems ??= new Dictionary<string, string>();
+
+#if NETSTANDARD2_1
+			return _metaDataItems.TryAdd(key, value);
+#else
+			if (_metaDataItems.ContainsKey(key))
+				return false;
+
+			_metaDataItems.Add(key, value);
+			return true;
+#endif
+		}		
+
+		public IReadOnlyDictionary<string, string> Items => _metaDataItems ?? EmptyReadOnly<string, string>.Dictionary;
+	}
+}

src/Elasticsearch.Net/Connection/ConnectionInfo.cs

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+#if DOTNETCORE
+using System.Net.Http;
+#endif
+
+namespace Elasticsearch.Net
+{
+	public static class ConnectionInfo
+	{
+		public static bool UsingCurlHandler
+		{
+			get
+			{
+#if !DOTNETCORE
+				return false;
+#else
+				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
+				if (!curlHandlerExists)
+					return false;
+
+				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
+				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
+				// Must be using CurlHandler.
+				if (!socketsHandlerExists)
+					return true;
+
+				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
+					return !isEnabled;
+
+				var environmentVariable =
+					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
+
+				// SocketsHandler exists and no environment variable exists to disable it.
+				// Must be using SocketsHandler and not CurlHandler
+				if (environmentVariable == null)
+					return false;
+
+				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
+					environmentVariable.Equals("0");
+#endif
+			}
+		}
+	}
+}

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -37,7 +37,6 @@ internal class WebProxy : IWebProxy
 		public bool IsBypassed(Uri host) => host.IsLoopback;
 	}
 
-
 	/// <summary> The default IConnection implementation. Uses <see cref="HttpClient" />.</summary>
 	public class HttpConnection : IConnection
 	{
@@ -139,6 +138,7 @@ public virtual async Task<TResponse> RequestAsync<TResponse>(RequestData request
 			IDisposable receive = DiagnosticSources.SingletonDisposable;
 			ReadOnlyDictionary<TcpState, int> tcpStats = null;
 			ReadOnlyDictionary<string, ThreadPoolStatistics> threadPoolStats = null;
+			requestData.IsAsync = true;
 
 			try
 			{
@@ -333,6 +333,14 @@ protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestDat
 			if (!requestData.RunAs.IsNullOrEmpty())
 				requestMessage.Headers.Add(RequestData.RunAsSecurityHeader, requestData.RunAs);
 
+			if (requestData.MetaHeaderProvider is object)
+			{
+				var value = requestData.MetaHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+					requestMessage.Headers.TryAddWithoutValidation(requestData.MetaHeaderProvider.HeaderName, value);
+			}
+
 			return requestMessage;
 		}
 

src/Elasticsearch.Net/Connection/HttpWebRequestConnection.cs

@@ -111,6 +111,7 @@ CancellationToken cancellationToken
 
 			try
 			{
+				requestData.IsAsync = true;
 				var data = requestData.PostData;
 				var request = CreateHttpWebRequest(requestData);
 				using (cancellationToken.Register(() => request.Abort()))
@@ -238,6 +239,14 @@ protected virtual HttpWebRequest CreateWebRequest(RequestData requestData)
 			if (requestData.Headers != null && requestData.Headers.HasKeys())
 				request.Headers.Add(requestData.Headers);
 
+			if (requestData.MetaHeaderProvider is object)
+			{
+				var value = requestData.MetaHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+					request.Headers.Add(requestData.MetaHeaderProvider.HeaderName, requestData.MetaHeaderProvider.ProduceHeaderValue(requestData));
+			}
+
 			var timeout = (int)requestData.RequestTimeout.TotalMilliseconds;
 			request.Timeout = timeout;
 			request.ReadWriteTimeout = timeout;