[OTel instrumentation] Add path params and endpoint in opts to perform_request

.github/workflows/main.yml

@@ -40,4 +40,4 @@ jobs:
     - name: elasticsearch
       run: cd elasticsearch && bundle exec rake test:all
     - name: elasticsearch-api
-      run: cd elasticsearch-api && bundle exec rake test:spec test:platinum:unit
+      run: rake elasticsearch:download_artifacts[8.11.0-SNAPSHOT] && cd elasticsearch-api && bundle exec rake test:spec test:platinum:unit

docs/basic-config.asciidoc

@@ -8,29 +8,30 @@ can use.
 [cols="<,<,<"]
 |===
 
-| **Parameter**        | **Data type**   | **Description**
-| `adapter`            | Symbol          | A specific adapter for Faraday (for example, `:patron`).
-| `api_key`            | String, Hash    | For API key Authentication. Either the base64 encoding of `id` and `api_key` joined by a colon as a string, or a hash with the `id` and `api_key` values.
-| `compression`        | Boolean         | Whether to compress requests. Gzip compression is used. Defaults to `false`. Responses are automatically inflated if they are compressed. If a custom transport object is used, it must handle the request compression and response inflation.
-| `enable_meta_header` | Boolean         | Whether to enable sending the meta data header to Cloud. Defaults to `true`.
-| `hosts`              | String, Array   | Single host passed as a string or hash, or multiple hosts passed as an array; `host` or `url` keys are also valid.
-| `log`                | Boolean         | Whether to use the default logger. Disabled by default.
-| `logger`             | Object          | An instance of a Logger-compatible object.
-| `opaque_id_prefix`   | String          | Sets a prefix for X-Opaque-Id when initializing the client. This is prepended to the id you set before each request if you're using X-Opaque-Id.
-| `randomize_hosts`    | Boolean         | Whether to shuffle connections on initialization and reload. Defaults to `false`.
-| `reload_connections` | Boolean, Number | Whether to reload connections after X requests. Defaults to `false`.
-| `reload_on_failure`  | Boolean         | Whether to reload connections after failure. Defaults to `false`.
-| `request_timeout`    | Integer         | The request timeout to be passed to transport in options.
-| `resurrect_after`    | Integer         | Specifies after how many seconds a dead connection should be tried again.
-| `retry_on_failure`   | Boolean, Number | Whether to retry X times when request fails before raising and exception. Defaults to `false`.
-| `retry_on_status`    | Array, Number   | Specifies which status code needs to be returned to retry.
-| `selector`           | Constant        | An instance of selector strategy implemented with {Elastic::Transport::Transport::Connections::Selector::Base}.
-| `send_get_body_as`   | String          | Specifies the HTTP method to use for GET requests with a body. Defaults to `GET`.
-| `serializer_class`   | Constant        | Specifies a serializer class to use. It is initialized by the transport and passed the transport instance.
-| `sniffer_timeout`    | Integer         | Specifieds the timeout for reloading connections in seconds. Defaults to `1`.
-| `trace`              | Boolean         | Whether to use the default tracer. Disabled by default.
-| `tracer`             | Object          | Specifies an instance of a Logger-compatible object.
-| `transport`          | Object          | Specifies a transport instance.
-| `transport_class`    | Constant        | Specifies a transport class to use. It is initialized by the client and passed hosts and all arguments.
-| `transport_options`  | Hash            | Specifies the options to be passed to the `Faraday::Connection` constructor.
+| **Parameter**                   | **Data type**                          | **Description**
+| `adapter`                       | Symbol                                 | A specific adapter for Faraday (for example, `:patron`).
+| `api_key`                       | String, Hash                           | For API key Authentication. Either the base64 encoding of `id` and `api_key` joined by a colon as a string, or a hash with the `id` and `api_key` values.
+| `compression`                   | Boolean                                | Whether to compress requests. Gzip compression is used. Defaults to `false`. Responses are automatically inflated if they are compressed. If a custom transport object is used, it must handle the request compression and response inflation.
+| `enable_meta_header`            | Boolean                                | Whether to enable sending the meta data header to Cloud. Defaults to `true`.
+| `hosts`                         | String, Array                          | Single host passed as a string or hash, or multiple hosts passed as an array; `host` or `url` keys are also valid.
+| `log`                           | Boolean                                | Whether to use the default logger. Disabled by default.
+| `logger`                        | Object                                 | An instance of a Logger-compatible object.
+| `opaque_id_prefix`              | String                                 | Sets a prefix for X-Opaque-Id when initializing the client. This is prepended to the id you set before each request if you're using X-Opaque-Id.
+| `opentelemetry_tracer_provider` | `OpenTelemetry::Trace::TracerProvider` | An explicit TracerProvider to use instead of the global one with OpenTelemetry. This enables better dependency injection and simplifies testing.
+| `randomize_hosts`               | Boolean                                | Whether to shuffle connections on initialization and reload. Defaults to `false`.
+| `reload_connections`            | Boolean, Number                        | Whether to reload connections after X requests. Defaults to `false`.
+| `reload_on_failure`             | Boolean                                | Whether to reload connections after failure. Defaults to `false`.
+| `request_timeout`               | Integer                                | The request timeout to be passed to transport in options.
+| `resurrect_after`               | Integer                                | Specifies after how many seconds a dead connection should be tried again.
+| `retry_on_failure`              | Boolean, Number                        | Whether to retry X times when request fails before raising and exception. Defaults to `false`.
+| `retry_on_status`               | Array, Number                          | Specifies which status code needs to be returned to retry.
+| `selector`                      | Constant                               | An instance of selector strategy implemented with {Elastic::Transport::Transport::Connections::Selector::Base}.
+| `send_get_body_as`              | String                                 | Specifies the HTTP method to use for GET requests with a body. Defaults to `GET`.
+| `serializer_class`              | Constant                               | Specifies a serializer class to use. It is initialized by the transport and passed the transport instance.
+| `sniffer_timeout`               | Integer                                | Specifies the timeout for reloading connections in seconds. Defaults to `1`.
+| `trace`                         | Boolean                                | Whether to use the default tracer. Disabled by default.
+| `tracer`                        | Object                                 | Specifies an instance of a Logger-compatible object.
+| `transport`                     | Object                                 | Specifies a transport instance.
+| `transport_class`               | Constant                               | Specifies a transport class to use. It is initialized by the client and passed hosts and all arguments.
+| `transport_options`             | Hash                                   | Specifies the options to be passed to the `Faraday::Connection` constructor.
 |===

docs/open-telemetry.asciidoc

@@ -0,0 +1,92 @@
+[[opentelemetry]]
+=== Using OpenTelemetry
+
+You can use https://opentelemetry.io/[OpenTelemetry] to monitor the performance and behavior of your {es} requests through the Ruby Client.
+The Ruby Client comes with built-in OpenTelemetry instrumentation that emits https://www.elastic.co/guide/en/apm/guide/current/apm-distributed-tracing.html[distributed tracing spans] by default.
+With that, applications https://opentelemetry.io/docs/instrumentation/ruby/manual/[instrumented with OpenTelemetry] or using the https://opentelemetry.io/docs/instrumentation/ruby/automatic/[OpenTelemetry Ruby SDK] are inherently enriched with additional spans that contain insightful information about the execution of the {es} requests.
+
+The native instrumentation in the Ruby Client follows the https://opentelemetry.io/docs/specs/semconv/database/elasticsearch/[OpenTelemetry Semantic Conventions for {es}]. In particular, the instrumentation in the client covers the logical layer of {es} requests. A single span per request is created that is processed by the service through the Ruby Client. The following image shows a trace that records the handling of two different {es} requests: a `ping` request and a `search` request.
+
+[role="screenshot"]
+image::images/otel-waterfall-without-http.png[alt="Distributed trace with Elasticsearch spans",align="center"]
+
+Usually, OpenTelemetry auto-instrumentation modules come with instrumentation support for HTTP-level communication. In this case, in addition to the logical {es} client requests, spans will be captured for the physical HTTP requests emitted by the client. The following image shows a trace with both, {es} spans (in blue) and the corresponding HTTP-level spans (in red):
+
+[role="screenshot"]
+image::images/otel-waterfall-with-http.png[alt="Distributed trace with Elasticsearch spans",align="center"]
+
+Advanced Ruby Client behavior such as nodes round-robin and request retries are revealed through the combination of logical {es} spans and the physical HTTP spans. The following example shows a `search` request in a scenario with two nodes:
+
+[role="screenshot"]
+image::images/otel-waterfall-retry.png[alt="Distributed trace with Elasticsearch spans",align="center"]
+
+The first node is unavailable and results in an HTTP error, while the retry to the second node succeeds. Both HTTP requests are subsumed by the logical {es} request span (in blue).
+
+[discrete]
+==== Setup the OpenTelemetry instrumentation
+
+When using the https://opentelemetry.io/docs/instrumentation/ruby/manual[OpenTelemetry Ruby SDK manually] or using the https://opentelemetry.io/docs/instrumentation/ruby/automatic/[OpenTelemetry Ruby Auto-Instrumentations], the Ruby Client's OpenTelemetry instrumentation is enabled by default and uses the global OpenTelemetry SDK with the global tracer provider. You can provide a tracer provider via the Ruby Client configuration option `opentelemetry_tracer_provider` when instantiating the client. This is sometimes useful for testing or other specific use cases.
+
+[source,ruby]
+------------------------------------
+client = Elasticsearch::Client.new(
+  cloud_id: '<CloudID>',
+  api_key: '<ApiKey>',
+  opentelemetry_tracer_provider: tracer_provider
+)
+------------------------------------
+
+[discrete]
+==== Configuring the OpenTelemetry instrumentation
+
+You can configure the OpenTelemetry instrumentation through Environment Variables.
+The following configuration options are available.
+
+[discrete]
+[[opentelemetry-config-enable]]
+===== Enable / Disable the OpenTelemetry instrumentation
+
+With this configuration option you can enable (default) or disable the built-in OpenTelemetry instrumentation.
+
+**Default:** `true`
+
+|============
+| Environment Variable | `OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_ENABLED`
+|============
+
+[discrete]
+===== Capture search request bodies
+
+Per default, the built-in OpenTelemetry instrumentation does not capture request bodies due to data privacy considerations. You can use this option to enable capturing of search queries from the request bodies of {es} search requests in case you wish to gather this information regardless. The options are to capture the raw search query, sanitize the query with a default list of sensitive keys, or not capture it at all.
+
+**Default:** `omit`
+
+**Valid Options:** `omit`, `sanitize`, `raw`
+
+|============
+| Environment Variable | `OTEL_INSTRUMENTATION_ELASTICSEARCH_CAPTURE_SEARCH_QUERY`
+|============
+
+[discrete]
+===== Sanitize the {es} search request body
+
+You can configure the list of keys whose values are redacted when the search query is captured. Values must be comma-separated.
+
+**Default:** `nil`
+
+|============
+| Environment Variable | `OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_SEARCH_QUERY_SANITIZE_KEYS`
+|============
+
+Example:
+
+```bash
+OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_SEARCH_QUERY_SANITIZE_KEYS='sensitive-key,other-sensitive-key'
+```
+
+[discrete]
+==== Overhead
+
+The OpenTelemetry instrumentation (as any other monitoring approach) may come with a slight overhead on CPU, memory, and/or latency. The overhead may only occur when the instrumentation is enabled (default) and an OpenTelemetry SDK is active in the target application. When the instrumentation is disabled or no OpenTelemetry SDK is active within the target application, monitoring overhead is not expected when using the client.
+
+Even in cases where the instrumentation is enabled and is actively used (by an OpenTelemetry SDK), the overhead is minimal and negligible in the vast majority of cases. In edge cases where there is a noticeable overhead, the <<opentelemetry-config-enable,instrumentation can be explicitly disabled>> to eliminate any potential impact on performance.

elasticsearch-api/lib/elasticsearch/api/actions/async_search/delete.rb

@@ -30,6 +30,14 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
         #
         def delete(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "async_search.delete" }
+
+          defined_params = [:id].inject({}) do |set_variables, variable|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+            set_variables
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
           raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
 
           arguments = arguments.clone
@@ -44,7 +52,7 @@ def delete(arguments = {})
           params = {}
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end

elasticsearch-api/lib/elasticsearch/api/actions/async_search/get.rb

@@ -33,6 +33,14 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
         #
         def get(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "async_search.get" }
+
+          defined_params = [:id].inject({}) do |set_variables, variable|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+            set_variables
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
           raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
 
           arguments = arguments.clone
@@ -47,7 +55,7 @@ def get(arguments = {})
           params = Utils.process_params(arguments)
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end

elasticsearch-api/lib/elasticsearch/api/actions/async_search/status.rb

@@ -30,6 +30,14 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
         #
         def status(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "async_search.status" }
+
+          defined_params = [:id].inject({}) do |set_variables, variable|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+            set_variables
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
           raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
 
           arguments = arguments.clone
@@ -44,7 +52,7 @@ def status(arguments = {})
           params = {}
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end

elasticsearch-api/lib/elasticsearch/api/actions/async_search/submit.rb

@@ -72,6 +72,14 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html
         #
         def submit(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "async_search.submit" }
+
+          defined_params = [:index].inject({}) do |set_variables, variable|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+            set_variables
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
           arguments = arguments.clone
           headers = arguments.delete(:headers) || {}
 
@@ -88,7 +96,7 @@ def submit(arguments = {})
           params = Utils.process_params(arguments)
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end

elasticsearch-api/lib/elasticsearch/api/actions/autoscaling/delete_autoscaling_policy.rb

@@ -30,6 +30,14 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/autoscaling-delete-autoscaling-policy.html
         #
         def delete_autoscaling_policy(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "autoscaling.delete_autoscaling_policy" }
+
+          defined_params = [:name].inject({}) do |set_variables, variable|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+            set_variables
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
           raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
 
           arguments = arguments.clone
@@ -44,7 +52,7 @@ def delete_autoscaling_policy(arguments = {})
           params = {}
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end

elasticsearch-api/lib/elasticsearch/api/actions/autoscaling/get_autoscaling_capacity.rb

@@ -29,6 +29,8 @@ module Actions
         # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/autoscaling-get-autoscaling-capacity.html
         #
         def get_autoscaling_capacity(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || "autoscaling.get_autoscaling_capacity" }
+
           arguments = arguments.clone
           headers = arguments.delete(:headers) || {}
 
@@ -39,7 +41,7 @@ def get_autoscaling_capacity(arguments = {})
           params = {}
 
           Elasticsearch::API::Response.new(
-            perform_request(method, path, params, body, headers)
+            perform_request(method, path, params, body, headers, request_opts)
           )
         end
       end