[Backport 6.x] Implement meta header for client requests (#5302)

* Implement meta header for client requests

(cherry picked (and adapted) from commit fec16ef)

* Skip nested sort test prior to 6.1

Author: stevejgordon

docs/client-concepts/connection/configuration-options.asciidoc

@@ -64,6 +64,10 @@ Ensures the response bytes are always available on the `ElasticsearchResponse<T>
 +
 IMPORTANT: Depending on the registered serializer, this may cause the response to be buffered in memory first, potentially affecting performance.
 
+`DisableMetaHeader`::
+
+Disables the meta header which is included on all requests by default. This header contains lightweight information  about the client and runtime.
+
 `DisablePing`::
 
 When a node is used for the very first time or when it's used for the first time after it has been marked dead a ping with a very low timeout is send to the node to make sure that when it's still dead it reports it as fast as possible. You can disable these pings globally here if you rather have it fail on the possible slower original request

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -28,36 +28,7 @@ public class ConnectionConfiguration : ConnectionConfiguration<ConnectionConfigu
 		/// As the old curl based handler is known to bleed TCP connections:
 		/// <para>https://github.com/dotnet/runtime/issues/22366</para>
 		/// </summary>
-        private static bool UsingCurlHandler
-		{
-			get
-			{
-#if !DOTNETCORE
-				return false;
-#else
-				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
-				if (!curlHandlerExists) return false;
-
-				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
-				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
-				// Must be using CurlHandler.
-				if (!socketsHandlerExists) return true;
-
-				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
-					return !isEnabled;
-
-				var environmentVariable =
-					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
-
-				// SocketsHandler exists and no environment variable exists to disable it.
-				// Must be using SocketsHandler and not CurlHandler
-				if (environmentVariable == null) return false;
-
-				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
-					environmentVariable.Equals("0");
-#endif
-			}
-		}
+        private static bool UsingCurlHandler => ConnectionInfo.UsingCurlHandler;
 
 		/// <summary>
 		/// The default ping timeout. Defaults to 2 seconds
@@ -174,6 +145,7 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private bool _disableAutomaticProxyDetection = false;
 
 		private bool _disableDirectStreaming = false;
+		private bool _disableMetaHeader;
 		private bool _disablePings;
 		private bool _enableHttpCompression;
 		private bool _enableHttpPipelining = true;
@@ -235,6 +207,7 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		TimeSpan? IConnectionConfigurationValues.DeadTimeout => _deadTimeout;
 		bool IConnectionConfigurationValues.DisableAutomaticProxyDetection => _disableAutomaticProxyDetection;
 		bool IConnectionConfigurationValues.DisableDirectStreaming => _disableDirectStreaming;
+		bool IConnectionConfigurationValues.DisableMetaHeader => _disableMetaHeader;
 		bool IConnectionConfigurationValues.DisablePings => _disablePings;
 		bool IConnectionConfigurationValues.EnableHttpCompression => _enableHttpCompression;
 		NameValueCollection IConnectionConfigurationValues.Headers => _headers;
@@ -270,6 +243,8 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		ElasticsearchUrlFormatter IConnectionConfigurationValues.UrlFormatter => _urlFormatter;
 		string IConnectionConfigurationValues.UserAgent => _userAgent;
 
+		MetaHeaderProvider IConnectionConfigurationValues.MetaHeaderProvider { get; } = new MetaHeaderProvider();
+
 		void IDisposable.Dispose() => DisposeManagedResources();
 
 		private static void DefaultCompletedRequestHandler(IApiCallDetails response) { }
@@ -362,6 +337,12 @@ public T SniffOnConnectionFault(bool sniffsOnConnectionFault = true) =>
 		/// </summary>
 		public T DisableAutomaticProxyDetection(bool disable = true) => Assign(disable, (a, v) => a._disableAutomaticProxyDetection = v);
 
+		/// <summary>
+		/// Disables the meta header which is included on all requests by default. This header contains lightweight information 
+		/// about the client and runtime.
+		/// </summary>
+		public T DisableMetaHeader(bool disable = true) => Assign(disable, (a, v) => a._disableMetaHeader = v);
+
 		/// <summary>
 		/// Instead of following a c/go like error checking on response.IsValid always throw an exception
 		/// on the client when a call resulted in an exception on either the client or the Elasticsearch server.

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -61,6 +61,11 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// </summary>
 		bool DisableDirectStreaming { get; }
 
+		/// <summary>
+		/// When set to true will disable sending the meta header on requests. Defaults to  false
+		/// </summary>
+		bool DisableMetaHeader { get; }
+
 		/// <summary>
 		/// This signals that we do not want to send initial pings to unknown/previously dead nodes
 		/// and just send the call straightaway
@@ -229,5 +234,10 @@ public interface IConnectionConfigurationValues : IDisposable
 		#endif
 		/// </summary>
 		TimeSpan DnsRefreshTimeout { get; }
+
+		/// <summary>
+		/// Produces the client meta header for a request.
+		/// </summary>
+		MetaHeaderProvider MetaHeaderProvider { get; }
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -93,6 +93,11 @@ public interface IRequestConfiguration
 		/// <para>Reasons for such exceptions could be search parser errors, index missing exceptions, etc...</para>
 		/// </summary>
 		bool ThrowExceptions { get; set; }
+
+		/// <summary>
+		/// Holds additional meta data about the request.
+		/// </summary>
+		RequestMetaData RequestMetaData { get; set; }
 	}
 
 	public class RequestConfiguration : IRequestConfiguration
@@ -113,6 +118,7 @@ public class RequestConfiguration : IRequestConfiguration
 		public string OpaqueId { get; set; }
 		public TimeSpan? PingTimeout { get; set; }
 		public TimeSpan? RequestTimeout { get; set; }
+		public RequestMetaData RequestMetaData { get; set; }
 
 		/// <summary>
 		/// Submit the request on behalf in the context of a different user
@@ -162,6 +168,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 		string IRequestConfiguration.RunAs { get; set; }
 		private IRequestConfiguration Self => this;
 		bool IRequestConfiguration.ThrowExceptions { get; set; }
+		RequestMetaData IRequestConfiguration.RequestMetaData { get; set; }
 
 		/// <summary>
 		/// Submit the request on behalf in the context of a different shield user
@@ -284,5 +291,12 @@ public RequestConfigurationDescriptor ClientCertificate(X509Certificate certific
 		/// <summary> Use the following client certificate to authenticate this request to Elasticsearch </summary>
 		public RequestConfigurationDescriptor ClientCertificate(string certificatePath) =>
 			ClientCertificates(new X509Certificate2Collection { new X509Certificate(certificatePath) });
+
+		/// <inheritdoc cref="IRequestConfiguration.RequestMetaData" />
+		public RequestConfigurationDescriptor RequestMetaData(RequestMetaData metaData)
+		{
+			Self.RequestMetaData = metaData;
+			return this;
+		}
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfigurationExtensions.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+
+namespace Elasticsearch.Net
+{
+	public static class RequestConfigurationExtensions
+	{
+		public static void SetRequestMetaData(this IRequestConfiguration requestConfiguration, RequestMetaData requestMetaData)
+		{
+			if (requestConfiguration is null)
+				throw new ArgumentNullException(nameof(requestConfiguration));
+
+			if (requestMetaData is null)
+				throw new ArgumentNullException(nameof(requestMetaData));
+
+			requestConfiguration.RequestMetaData = requestMetaData;
+		}
+	}
+}

src/Elasticsearch.Net/Configuration/RequestMetaData.cs

@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// Holds meta data about a client request.
+	/// </summary>
+	public sealed class RequestMetaData
+	{
+		/// <summary>
+		/// Reserved key for a meta data entry which identifies the helper which produced the request.
+		/// </summary>
+		public const string HelperKey = "helper";
+
+		private Dictionary<string, string> _metaDataItems;
+
+		public bool TryAddMetaData (string key, string value)
+		{
+			_metaDataItems ??= new Dictionary<string, string>();
+
+			if (_metaDataItems.ContainsKey(key))
+				return false;
+
+			_metaDataItems.Add(key, value);
+			return true;
+		}
+
+		public IReadOnlyDictionary<string, string> Items => _metaDataItems ?? EmptyReadOnly<string, string>.Dictionary;
+	}
+}

src/Elasticsearch.Net/Connection/ConnectionInfo.cs

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+#if DOTNETCORE
+using System.Net.Http;
+#endif
+
+namespace Elasticsearch.Net
+{
+	public static class ConnectionInfo
+	{
+		public static bool UsingCurlHandler
+		{
+			get
+			{
+#if !DOTNETCORE
+				return false;
+#else
+				var curlHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.CurlHandler") != null;
+				if (!curlHandlerExists)
+					return false;
+
+				var socketsHandlerExists = typeof(HttpClientHandler).Assembly.GetType("System.Net.Http.SocketsHttpHandler") != null;
+				// running on a .NET core version with CurlHandler, before the existence of SocketsHttpHandler.
+				// Must be using CurlHandler.
+				if (!socketsHandlerExists)
+					return true;
+
+				if (AppContext.TryGetSwitch("System.Net.Http.UseSocketsHttpHandler", out var isEnabled))
+					return !isEnabled;
+
+				var environmentVariable =
+					Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER");
+
+				// SocketsHandler exists and no environment variable exists to disable it.
+				// Must be using SocketsHandler and not CurlHandler
+				if (environmentVariable == null)
+					return false;
+
+				return environmentVariable.Equals("false", StringComparison.OrdinalIgnoreCase) ||
+					environmentVariable.Equals("0");
+#endif
+			}
+		}
+	}
+}

src/Elasticsearch.Net/Connection/HttpConnection-CoreFx.cs

@@ -99,6 +99,7 @@ public virtual async Task<TResponse> RequestAsync<TResponse>(RequestData request
 			Stream responseStream = null;
 			Exception ex = null;
 			string mimeType = null;
+			requestData.IsAsync = true;
 			try
 			{
 				var requestMessage = CreateHttpRequestMessage(requestData);
@@ -233,6 +234,13 @@ protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestDat
 				stream.Position = 0;
 			}
 
+			if (requestData.MetaHeaderProvider is object) {
+				var value = requestData.MetaHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+					requestMessage.Headers.TryAddWithoutValidation(requestData.MetaHeaderProvider.HeaderName, value);
+			}
+
 			return requestMessage;
 		}
 

src/Elasticsearch.Net/Connection/HttpWebRequestConnection.cs

@@ -82,6 +82,7 @@ CancellationToken cancellationToken
 			Stream responseStream = null;
 			Exception ex = null;
 			string mimeType = null;
+			requestData.IsAsync = true;
 			try
 			{
 				var data = requestData.PostData;
@@ -195,6 +196,13 @@ protected virtual HttpWebRequest CreateWebRequest(RequestData requestData)
 			if (requestData.Headers != null && requestData.Headers.HasKeys())
 				request.Headers.Add(requestData.Headers);
 
+			if (requestData.MetaHeaderProvider is object) {
+				var value = requestData.MetaHeaderProvider.ProduceHeaderValue(requestData);
+
+				if (!string.IsNullOrEmpty(value))
+					request.Headers.Add(requestData.MetaHeaderProvider.HeaderName, requestData.MetaHeaderProvider.ProduceHeaderValue(requestData));
+			}
+
 			var timeout = (int)requestData.RequestTimeout.TotalMilliseconds;
 			request.Timeout = timeout;
 			request.ReadWriteTimeout = timeout;

src/Elasticsearch.Net/Connection/MetaData/ClientVersionInfo.cs

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Diagnostics;
+using System.Reflection;
+using System.Text.RegularExpressions;
+
+namespace Elasticsearch.Net
+{
+	internal sealed class ClientVersionInfo : VersionInfo
+	{
+		private static readonly Regex VersionRegex = new Regex(@"(\d+\.)(\d+\.)(\d)");
+
+		public static readonly ClientVersionInfo Empty = new ClientVersionInfo { Version = new Version(0, 0, 0), IsPrerelease = false };
+
+		private ClientVersionInfo() { }
+
+		public static ClientVersionInfo Create<T>()
+		{
+			var fullVersion = DetermineClientVersion(typeof(T));
+
+			var clientVersion = new ClientVersionInfo();
+			clientVersion.StoreVersion(fullVersion);
+			return clientVersion;
+		}
+
+		private static string DetermineClientVersion(Type type)
+		{
+			try
+			{
+				var productVersion = FileVersionInfo.GetVersionInfo(type.GetTypeInfo().Assembly.Location)?.ProductVersion ?? EmptyVersion;
+
+				if (productVersion == EmptyVersion)
+					productVersion = Assembly.GetAssembly(type).GetName().Version.ToString();
+
+				var match = VersionRegex.Match(productVersion);
+
+				return match.Success ? match.Value : EmptyVersion;
+			}
+			catch
+			{
+				// ignore failures and fall through
+			}
+
+			return EmptyVersion;
+		}
+	}
+}

src/Elasticsearch.Net/Connection/MetaData/MetaDataHeader.cs

@@ -0,0 +1,44 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text;
+
+namespace Elasticsearch.Net
+{
+	internal sealed class MetaDataHeader
+	{
+		private const char _separator = ',';
+
+		private readonly string _headerValue;
+
+		public MetaDataHeader(VersionInfo version, string serviceIdentifier, bool isAsync)
+		{
+			ClientVersion = version.ToString();
+			RuntimeVersion = new RuntimeVersionInfo().ToString();
+			ServiceIdentifier = serviceIdentifier;
+
+			// This code is expected to be called infrequently so we're not concerns with over optimising this
+
+			_headerValue = new StringBuilder(64)
+				.Append(serviceIdentifier).Append("=").Append(ClientVersion).Append(_separator)
+				.Append("a=").Append(isAsync ? "1" : "0").Append(_separator)
+				.Append("net=").Append(RuntimeVersion).Append(_separator)
+				.Append(_httpClientIdentifier).Append("=").Append(RuntimeVersion)
+				.ToString();
+		}
+
+		private static readonly string _httpClientIdentifier =
+#if DOTNETCORE
+			ConnectionInfo.UsingCurlHandler ? "cu" : "so";
+#else
+			"wr";
+#endif
+
+		public string ServiceIdentifier { get; private set; }
+		public string ClientVersion { get; private set; }
+		public string RuntimeVersion { get; private set; }
+
+		public override string ToString() => _headerValue;
+	}
+}