From 55e3ffe570740100d2ed69df441f1c8bf95ec154 Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Mon, 21 Apr 2025 10:27:46 -0500 Subject: [PATCH 1/2] fix various syntax and rendering issues --- docs/reference/api-reference.md | 1 + docs/reference/client-helpers.md | 182 +++++++++++++++++++++++++++---- docs/reference/connecting.md | 3 +- docs/reference/observability.md | 55 ++++++++-- 4 files changed, 212 insertions(+), 29 deletions(-) diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index 7381b5ff2..5c691ea5c 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -980,6 +980,7 @@ PUT my-index-000001/_doc/1?version=2&version_type=external "id": "elkbee" } } +``` In this example, the operation will succeed since the supplied version of 2 is higher than the current document version of 1. If the document was already updated and its version was set to 2 or higher, the indexing command will fail and result in a conflict (409 HTTP status code). diff --git a/docs/reference/client-helpers.md b/docs/reference/client-helpers.md index c80562db4..731df31cc 100644 --- a/docs/reference/client-helpers.md +++ b/docs/reference/client-helpers.md @@ -51,18 +51,120 @@ console.log(result) To create a new instance of the Bulk helper, access it as shown in the example above, the configuration options are: -| | | -| --- | --- | -| `datasource` | An array, async generator or a readable stream with the data you need to index/create/update/delete. It can be an array of strings or objects, but also a stream of json strings or JavaScript objects.
If it is a stream, we recommend to use the [`split2`](https://www.npmjs.com/package/split2) package, that splits the stream on new lines delimiters.
This parameter is mandatory.

```js
const { createReadStream } = require('fs')
const split = require('split2')
const b = client.helpers.bulk({
// if you just use split(), the data will be used as array of strings
datasource: createReadStream('./dataset.ndjson').pipe(split())
// if you need to manipulate the data, you can pass JSON.parse to split
datasource: createReadStream('./dataset.ndjson').pipe(split(JSON.parse))
})
```
| -| `onDocument` | A function that is called for each document of the datasource. Inside this function you can manipulate the document and you must return the operation you want to execute with the document. Look at the [Bulk API documentation](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) to see the supported operations.
This parameter is mandatory.

```js
const b = client.helpers.bulk({
onDocument (doc) {
return {
index: { _index: 'my-index' }
}
}
})
```
| -| `onDrop` | A function that is called for everytime a document can’t be indexed and it has reached the maximum amount of retries.

```js
const b = client.helpers.bulk({
onDrop (doc) {
console.log(doc)
}
})
```
| -| `onSuccess` | A function that is called for each successful operation in the bulk request, which includes the result from Elasticsearch along with the original document that was sent, or `null` for delete operations.

```js
const b = client.helpers.bulk({
onSuccess ({ result, document }) {
console.log(`SUCCESS: Document ${result.index._id} indexed to ${result.index._index}`)
}
})
```
| -| `flushBytes` | The size of the bulk body in bytes to reach before to send it. Default of 5MB.
*Default:* `5000000`

```js
const b = client.helpers.bulk({
flushBytes: 1000000
})
```
| -| `flushInterval` | How much time (in milliseconds) the helper waits before flushing the body from the last document read.
*Default:* `30000`

```js
const b = client.helpers.bulk({
flushInterval: 30000
})
```
| -| `concurrency` | How many request is executed at the same time.
*Default:* `5`

```js
const b = client.helpers.bulk({
concurrency: 10
})
```
| -| `retries` | How many times a document is retried before to call the `onDrop` callback.
*Default:* Client max retries.

```js
const b = client.helpers.bulk({
retries: 3
})
```
| -| `wait` | How much time to wait before retries in milliseconds.
*Default:* 5000.

```js
const b = client.helpers.bulk({
wait: 3000
})
```
| -| `refreshOnCompletion` | If `true`, at the end of the bulk operation it runs a refresh on all indices or on the specified indices.
*Default:* false.

```js
const b = client.helpers.bulk({
refreshOnCompletion: true
// or
refreshOnCompletion: 'index-name'
})
```
| + +`datasource` +: An array, async generator or a readable stream with the data you need to index/create/update/delete. It can be an array of strings or objects, but also a stream of json strings or JavaScript objects. + If it is a stream, we recommend to use the [`split2`](https://www.npmjs.com/package/split2) package, that splits the stream on new lines delimiters. + This parameter is mandatory. + + ```js + const { createReadStream } = require('fs') + const split = require('split2') + const b = client.helpers.bulk({ + // if you just use split(), the data will be used as array of strings + datasource: createReadStream('./dataset.ndjson').pipe(split()) + // if you need to manipulate the data, you can pass JSON.parse to split + datasource: createReadStream('./dataset.ndjson').pipe(split(JSON.parse)) + }) + ``` + +`onDocument` +: A function that is called for each document of the datasource. Inside this function you can manipulate the document and you must return the operation you want to execute with the document. Look at the [Bulk API documentation](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) to see the supported operations. + This parameter is mandatory. + + ```js + const b = client.helpers.bulk({ + onDocument (doc) { + return { + index: { _index: 'my-index' } + } + } + }) + ``` + +`onDrop` +: A function that is called for everytime a document can’t be indexed and it has reached the maximum amount of retries. + + ```js + const b = client.helpers.bulk({ + onDrop (doc) { + console.log(doc) + } + }) + ``` + +`onSuccess` +: A function that is called for each successful operation in the bulk request, which includes the result from Elasticsearch along with the original document that was sent, or `null` for delete operations. + + ```js + const b = client.helpers.bulk({ + onSuccess ({ result, document }) { + console.log(`SUCCESS: Document ${result.index._id} indexed to ${result.index._index}`) + } + }) + ``` + +`flushBytes` +: The size of the bulk body in bytes to reach before to send it. Default of 5MB. + *Default:* `5000000` + + ```js + const b = client.helpers.bulk({ + flushBytes: 1000000 + }) + ``` + +`flushInterval` +: How much time (in milliseconds) the helper waits before flushing the body from the last document read. + *Default:* `30000` + + ```js + const b = client.helpers.bulk({ + flushInterval: 30000 + }) + ``` + +`concurrency` +: How many request is executed at the same time. + *Default:* `5` + + ```js + const b = client.helpers.bulk({ + concurrency: 10 + }) + ``` + +`retries` +: How many times a document is retried before to call the `onDrop` callback. + *Default:* Client max retries. + + ```js + const b = client.helpers.bulk({ + retries: 3 + }) + ``` + +`wait` +: How much time to wait before retries in milliseconds. + *Default:* 5000. + + ```js + const b = client.helpers.bulk({ + wait: 3000 + }) + ``` + +`refreshOnCompletion` +: If `true`, at the end of the bulk operation it runs a refresh on all indices or on the specified indices. + *Default:* false. + + ```js + const b = client.helpers.bulk({ + refreshOnCompletion: true + // or + refreshOnCompletion: 'index-name' + }) + ``` ### Supported operations [_supported_operations] @@ -255,13 +357,55 @@ m.search( To create a new instance of the multi search (msearch) helper, you should access it as shown in the example above, the configuration options are: -| | | -| --- | --- | -| `operations` | How many search operations should be sent in a single msearch request.
*Default:* `5`

```js
const m = client.helpers.msearch({
operations: 10
})
```
| -| `flushInterval` | How much time (in milliseconds) the helper waits before flushing the operations from the last operation read.
*Default:* `500`

```js
const m = client.helpers.msearch({
flushInterval: 500
})
```
| -| `concurrency` | How many request is executed at the same time.
*Default:* `5`

```js
const m = client.helpers.msearch({
concurrency: 10
})
```
| -| `retries` | How many times an operation is retried before to resolve the request. An operation is retried only in case of a 429 error.
*Default:* Client max retries.

```js
const m = client.helpers.msearch({
retries: 3
})
```
| -| `wait` | How much time to wait before retries in milliseconds.
*Default:* 5000.

```js
const m = client.helpers.msearch({
wait: 3000
})
```
| +`operations` +: How many search operations should be sent in a single msearch request. + *Default:* `5` + + ```js + const m = client.helpers.msearch({ + operations: 10 + }) + ``` + +`flushInterval` +: How much time (in milliseconds) the helper waits before flushing the operations from the last operation read. + *Default:* `500` + + ```js + const m = client.helpers.msearch({ + flushInterval: 500 + }) + ``` + +`concurrency` +: How many request is executed at the same time. + *Default:* `5` + + ```js + const m = client.helpers.msearch({ + concurrency: 10 + }) + ``` + +`retries` +: How many times an operation is retried before to resolve the request. An operation is retried only in case of a 429 error. + *Default:* Client max retries. + + ```js + const m = client.helpers.msearch({ + retries: 3 + }) + ``` + +`wait` +: How much time to wait before retries in milliseconds. + *Default:* 5000. + + ```js + const m = client.helpers.msearch({ + wait: 3000 + }) + ``` ### Stopping the msearch helper [_stopping_the_msearch_helper] diff --git a/docs/reference/connecting.md b/docs/reference/connecting.md index 887dc587d..9fd3e654a 100644 --- a/docs/reference/connecting.md +++ b/docs/reference/connecting.md @@ -461,9 +461,8 @@ console.log(errors) You can find the errors exported by the client in the table below. -| | | | -| --- | --- | --- | | **Error** | **Description** | **Properties** | +| --- | --- | --- | | `ElasticsearchClientError` | Every error inherits from this class, it is the basic error generated by the client. | * `name` - `string`
* `message` - `string`
| | `TimeoutError` | Generated when a request exceeds the `requestTimeout` option. | * `name` - `string`
* `message` - `string`
* `meta` - `object`, contains all the information about the request
| | `ConnectionError` | Generated when an error occurs during the request, it can be a connection error or a malformed stream of data. | * `name` - `string`
* `message` - `string`
* `meta` - `object`, contains all the information about the request
| diff --git a/docs/reference/observability.md b/docs/reference/observability.md index 38f8c332d..bb77e7a4f 100644 --- a/docs/reference/observability.md +++ b/docs/reference/observability.md @@ -67,14 +67,53 @@ client.diagnostic.on('response', (err, result) => { The client emits the following events: -| | | -| --- | --- | -| `serialization` | Emitted before starting serialization and compression. If you want to measure this phase duration, you should measure the time elapsed between this event and `request`.

```js
client.diagnostic.on('serialization', (err, result) => {
console.log(err, result)
})
```
| -| `request` | Emitted before sending the actual request to {{es}} *(emitted multiple times in case of retries)*.

```js
client.diagnostic.on('request', (err, result) => {
console.log(err, result)
})
```
| -| `deserialization` | Emitted before starting deserialization and decompression. If you want to measure this phase duration, you should measure the time elapsed between this event and `response`. *(This event might not be emitted in certain situations)*.

```js
client.diagnostic.on('deserialization', (err, result) => {
console.log(err, result)
})
```
| -| `response` | Emitted once {{es}} response has been received and parsed.

```js
client.diagnostic.on('response', (err, result) => {
console.log(err, result)
})
```
| -| `sniff` | Emitted when the client ends a sniffing request.

```js
client.diagnostic.on('sniff', (err, result) => {
console.log(err, result)
})
```
| -| `resurrect` | Emitted if the client is able to resurrect a dead node.

```js
client.diagnostic.on('resurrect', (err, result) => {
console.log(err, result)
})
```
| +`serialization` +: Emitted before starting serialization and compression. If you want to measure this phase duration, you should measure the time elapsed between this event and `request`. + ```js + client.diagnostic.on('serialization', (err, result) => { + console.log(err, result) + }) + ``` + +`request` +: Emitted before sending the actual request to {{es}} *(emitted multiple times in case of retries)*. + ```js + client.diagnostic.on('request', (err, result) => { + console.log(err, result) + }) + ``` + +`deserialization` +: Emitted before starting deserialization and decompression. If you want to measure this phase duration, you should measure the time elapsed between this event and `response`. *(This event might not be emitted in certain situations)*. + ```js + client.diagnostic.on('deserialization', (err, result) => { + console.log(err, result) + }) + ``` + +`response` +: Emitted once {{es}} response has been received and parsed. + ```js + client.diagnostic.on('response', (err, result) => { + console.log(err, result) + }) + ``` + +`sniff` +: Emitted when the client ends a sniffing request. + ```js + client.diagnostic.on('sniff', (err, result) => { + console.log(err, result) + }) + ``` + +`resurrect` +: Emitted if the client is able to resurrect a dead node. + ```js + client.diagnostic.on('resurrect', (err, result) => { + console.log(err, result) + }) + ``` The values of `result` in `serialization`, `request`, `deserialization`, `response` and `sniff` are: From 06c8ff137aa3406522c6d3de9d9522cb745b8cfe Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Mon, 21 Apr 2025 12:16:51 -0500 Subject: [PATCH 2/2] more fixes --- docs/reference/basic-config.md | 56 ------------------------------- docs/reference/connecting.md | 9 ++--- docs/reference/getting-started.md | 8 ++--- 3 files changed, 7 insertions(+), 66 deletions(-) diff --git a/docs/reference/basic-config.md b/docs/reference/basic-config.md index 7b523cbeb..3363ac007 100644 --- a/docs/reference/basic-config.md +++ b/docs/reference/basic-config.md @@ -82,8 +82,6 @@ auth: { } ``` ---- - ### `maxRetries` Type: `number`
@@ -91,8 +89,6 @@ Default: `3` Max number of retries for each request. ---- - ### `requestTimeout` Type: `number`
@@ -100,8 +96,6 @@ Default: `No value` Max request timeout in milliseconds for each request. ---- - ### `pingTimeout` Type: `number`
@@ -109,8 +103,6 @@ Default: `3000` Max ping request timeout in milliseconds for each request. ---- - ### `sniffInterval` Type: `number, boolean`
@@ -122,8 +114,6 @@ Perform a sniff operation every `n` milliseconds. Sniffing might not be the best solution. Before using the various `sniff` options, review this [blog post](https://www.elastic.co/blog/elasticsearch-sniffing-best-practices-what-when-why-how). ::: ---- - ### `sniffOnStart` Type: `boolean`
@@ -131,8 +121,6 @@ Default: `false` Perform a sniff once the client is started. Be sure to review the sniffing best practices [blog post](https://www.elastic.co/blog/elasticsearch-sniffing-best-practices-what-when-why-how). ---- - ### `sniffEndpoint` Type: `string`
@@ -140,8 +128,6 @@ Default: `'_nodes/_all/http'` Endpoint to ping during a sniff. Be sure to review the sniffing best practices [blog post](https://www.elastic.co/blog/elasticsearch-sniffing-best-practices-what-when-why-how). ---- - ### `sniffOnConnectionFault` Type: `boolean`
@@ -149,8 +135,6 @@ Default: `false` Perform a sniff on connection fault. Be sure to review the sniffing best practices [blog post](https://www.elastic.co/blog/elasticsearch-sniffing-best-practices-what-when-why-how). ---- - ### `resurrectStrategy` Type: `string`
@@ -159,8 +143,6 @@ Default: `'ping'` Configure the node resurrection strategy.
Options: `'ping'`, `'optimistic'`, `'none'` ---- - ### `suggestCompression` Type: `boolean`
@@ -168,8 +150,6 @@ Default: `false` Adds an `accept-encoding` header to every request. ---- - ### `compression` Type: `string, boolean`
@@ -178,8 +158,6 @@ Default: `false` Enables gzip request body compression.
Options: `'gzip'`, `false` ---- - ### `tls` Type: `http.SecureContextOptions`
@@ -187,8 +165,6 @@ Default: `null` The [tls configuraton](https://nodejs.org/api/tls.html). ---- - ### `proxy` Type: `string, URL`
@@ -208,8 +184,6 @@ const client = new Client({ }) ``` ---- - ### `agent` Type: `http.AgentOptions, function`
@@ -237,8 +211,6 @@ const client = new Client({ }) ``` ---- - ### `nodeFilter` Type: `function` @@ -257,8 +229,6 @@ function defaultNodeFilter (node) { } ``` ---- - ### `nodeSelector` Type: `function`
@@ -276,8 +246,6 @@ function nodeSelector (connections) { } ``` ---- - ### `generateRequestId` Type: `function`
@@ -294,8 +262,6 @@ function generateRequestId (params, options) { } ``` ---- - ### `name` Type: `string, symbol`
@@ -303,8 +269,6 @@ Default: `elasticsearch-js` The name to identify the client instance in the events. ---- - ### `opaqueIdPrefix` Type: `string`
@@ -313,8 +277,6 @@ Default: `null` A string that will be use to prefix any `X-Opaque-Id` header. See [`X-Opaque-Id` support](/reference/observability.md#_x_opaque_id_support) for more details. ---- - ### `headers` Type: `object`
@@ -322,8 +284,6 @@ Default: `{}` A set of custom headers to send in every request. ---- - ### `context` Type: `object`
@@ -331,8 +291,6 @@ Default: `null` A custom object that you can use for observability in your events. It will be merged with the API level context option. ---- - ### `enableMetaHeader` Type: `boolean`
@@ -340,8 +298,6 @@ Default: `true` If true, adds an header named `'x-elastic-client-meta'`, containing some minimal telemetry data, such as the client and platform version. ---- - ### `cloud` Type: `object`
@@ -363,16 +319,12 @@ const client = new Client({ }) ``` ---- - ### `disablePrototypePoisoningProtection` Default: `true` `boolean`, `'proto'`, `'constructor'` - The client can protect you against prototype poisoning attacks. For more information, refer to [Square Brackets are the Enemy](https://web.archive.org/web/20200319091159/https://hueniverse.com/square-brackets-are-the-enemy-ff5b9fd8a3e8?gi=184a27ee2a08). If needed, you can enable prototype poisoning protection entirely (`false`) or one of the two checks (`'proto'` or `'constructor'`). For performance reasons, it is disabled by default. To learn more, refer to the [`secure-json-parse` documentation](https://github.com/fastify/secure-json-parse). ---- - ### `caFingerprint` Type: `string`
@@ -380,8 +332,6 @@ Default: `null` If configured, verify that the fingerprint of the CA certificate that has signed the certificate of the server matches the supplied fingerprint. Only accepts SHA256 digest fingerprints. ---- - ### `maxResponseSize` Type: `number`
@@ -389,8 +339,6 @@ Default: `null` When configured, `maxResponseSize` verifies that the uncompressed response size is lower than the configured number. If it’s higher, the request will be canceled. The `maxResponseSize` cannot be higher than the value of `buffer.constants.MAX_STRING_LENGTH`. ---- - ### `maxCompressedResponseSize` Type: `number`
@@ -398,8 +346,6 @@ Default: `null` When configured, `maxCompressedResponseSize` verifies that the compressed response size is lower than the configured number. If it’s higher, the request will be canceled. The `maxCompressedResponseSize` cannot be higher than the value of `buffer.constants.MAX_STRING_LENGTH`. ---- - ### `redaction` Type: `object`
@@ -411,8 +357,6 @@ Options for how to redact potentially sensitive data from metadata attached to ` [Read about redaction](/reference/advanced-config.md#redaction) for more details :::: ---- - ### `serverMode` Type: `string`
diff --git a/docs/reference/connecting.md b/docs/reference/connecting.md index 9fd3e654a..82a121584 100644 --- a/docs/reference/connecting.md +++ b/docs/reference/connecting.md @@ -332,21 +332,22 @@ The supported request specific options are: | Option | Description | | --- | ----------- | | `ignore` | `number[]` -  HTTP status codes which should not be considered errors for this request.
*Default:* `null` | -| `requestTimeout` | `number` or `string` - Max request timeout for the request in milliseconds. This overrides the client default, which is to not time out at all. See [Elasticsearch best practices for HTML clients](elasticsearch://reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration) for more info.
_Default:* No timeout | +| `requestTimeout` | `number` or `string` - Max request timeout for the request in milliseconds. This overrides the client default, which is to not time out at all. See [Elasticsearch best practices for HTML clients](elasticsearch://reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration) for more info.
_Default:_ No timeout | | `retryOnTimeout` | `boolean` - Retry requests that have timed out.*Default:* `false` | | `maxRetries` | `number` - Max number of retries for the request, it overrides the client default.
*Default:* `3` | | `compression` | `string` or `boolean` - Enables body compression for the request.
*Options:* `false`, `'gzip'`
*Default:* `false` | | `asStream` | `boolean` - Instead of getting the parsed body back, you get the raw Node.js stream of data.
*Default:* `false` | | `headers` | `object` - Custom headers for the request.
*Default:* `null` | -|`querystring` | `object` - Custom querystring for the request.
*Default:* `null` | +| `querystring` | `object` - Custom querystring for the request.
*Default:* `null` | | `id` | `any` - Custom request ID. *(overrides the top level request id generator)*
*Default:* `null` | | `context` | `any` - Custom object per request. *(you can use it to pass data to the clients events)*
*Default:* `null` | -| `opaqueId` | `string` - Set the `X-Opaque-Id` HTTP header. See [X-Opaque-Id HTTP header](elasticsearch://reference/elasticsearch/rest-apis/api-conventions.md#x-opaque-id) *Default:* `null` | +| `opaqueId` | `string` - Set the `X-Opaque-Id` HTTP header. See [X-Opaque-Id HTTP header](elasticsearch://reference/elasticsearch/rest-apis/api-conventions.md#x-opaque-id)
*Default:* `null` | | `maxResponseSize` | `number` - When configured, it verifies that the uncompressed response size is lower than the configured number, if it’s higher it will abort the request. It cannot be higher than buffer.constants.MAX_STRING_LENTGH
*Default:* `null` | | `maxCompressedResponseSize` | `number` - When configured, it verifies that the compressed response size is lower than the configured number, if it’s higher it will abort the request. It cannot be higher than buffer.constants.MAX_LENTGH
*Default:* `null` | | `signal` | `AbortSignal` - The AbortSignal instance to allow request abortion.
*Default:* `null` | | `meta` | `boolean` - Rather than returning the body, return an object containing `body`, `statusCode`, `headers` and `meta` keys
*Default*: `false` | -| `redaction` | `object` - Options for redacting potentially sensitive data from error metadata. See [Redaction of potentially sensitive data](/reference/advanced-config.md#redaction). | `retryBackoff` | +| `redaction` | `object` - Options for redacting potentially sensitive data from error metadata. See [Redaction of potentially sensitive data](/reference/advanced-config.md#redaction). | +| `retryBackoff` | `(min: number, max: number, attempt: number) => number;` - A function that calculates how long to sleep, in seconds, before the next request retry
_Default:_ A built-in function that uses exponential backoff with jitter. | ## Using the Client in a Function-as-a-Service Environment [client-faas-env] diff --git a/docs/reference/getting-started.md b/docs/reference/getting-started.md index 1420d6c4c..61f2dabfb 100644 --- a/docs/reference/getting-started.md +++ b/docs/reference/getting-started.md @@ -45,15 +45,11 @@ const client = new Client({ Your Elasticsearch endpoint can be found on the **My deployment** page of your deployment: -:::{image} images/es-endpoint.jpg -:alt: Finding Elasticsearch endpoint -::: +![Finding Elasticsearch endpoint](images/es-endpoint.jpg) You can generate an API key on the **Management** page under Security. -:::{image} images/create-api-key.png -:alt: Create API key -::: +![Create API key](images/create-api-key.png) For other connection options, refer to the [*Connecting*](/reference/connecting.md) section.