Merge remote-tracking branch 'origin/main' into Update-Inference-specification-for-Hugging-Face's-rerank-task

Author: Evgenii-Kazannik

.backportrc.json

@@ -0,0 +1,6 @@
+{
+  "repoOwner": "elastic",
+  "repoName": "elasticsearch-specification",
+  "targetBranchChoices": ["9.0", "8.19", "8.18", "8.17"],
+  "fork": false
+}

.github/download-artifacts/index.js

@@ -17,23 +17,23 @@
  * under the License.
  */
 
-'use strict'
-
-const core = require('@actions/core')
-const { join } = require('path')
-const minimist = require('minimist')
-const stream = require('stream')
-const { promisify } = require('util')
-const { createWriteStream, promises } = require('fs')
-const rimraf = require('rimraf')
-const fetch = require('node-fetch')
-const crossZip = require('cross-zip')
+import core from '@actions/core'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import minimist from 'minimist'
+import stream from 'stream'
+import { promisify } from 'util'
+import { createWriteStream, promises } from 'fs'
+import { rimraf } from 'rimraf'
+import fetch from 'node-fetch'
+import crossZip from 'cross-zip'
 
 const { mkdir, rename, readdir, unlink } = promises
 const pipeline = promisify(stream.pipeline)
 const unzip = promisify(crossZip.unzip)
-const rm = promisify(rimraf)
 
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
 const esFolder = join(__dirname, '..', '..', 'artifacts')
 const zipFolder = join(esFolder, 'artifacts.zip')
 const downloadedSpec = join(esFolder, 'rest-api-spec', 'api')
@@ -57,8 +57,8 @@ async function downloadArtifacts (opts) {
   core.info(`Resolved artifact URL for ${resolved.commit_url}`)
 
   core.info('Cleanup')
-  await rm(esFolder)
-  await rm(specFolder)
+  await rimraf(esFolder)
+  await rimraf(specFolder)
   await mkdir(esFolder, { recursive: true })
   await mkdir(specFolder, { recursive: true })
 
@@ -74,7 +74,7 @@ async function downloadArtifacts (opts) {
   await unzip(zipFolder, esFolder)
 
   core.info('Cleanup')
-  await rm(zipFolder)
+  await rimraf(zipFolder)
 
   core.info('Moving files')
   const files = await readdir(downloadedSpec)

.github/download-artifacts/package.json

@@ -2,6 +2,7 @@
   "name": "download-artifacts",
   "version": "1.0.0",
   "description": "",
+  "type": "module",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -13,7 +14,7 @@
     "@actions/core": "^1.6.0",
     "cross-zip": "^4.0.0",
     "minimist": "^1.2.5",
-    "node-fetch": "^2.6.7",
-    "rimraf": "^3.0.2"
+    "node-fetch": "^3",
+    "rimraf": "^6"
   }
 }

.github/validate-pr/index.js

@@ -26,7 +26,7 @@ import assert from 'assert'
 import * as core from '@actions/core'
 import { copyFile } from 'fs/promises'
 import * as github from '@actions/github'
-import specification from '../../output/schema/schema.json' assert { type: 'json' }
+import specification from '../../output/schema/schema.json' with { type: 'json' }
 import { run as getReport } from '../../../clients-flight-recorder/scripts/types-validator/index.js'
 import {
   getNamespace,
@@ -90,7 +90,12 @@ async function run() {
     if (file.startsWith('specification/_spec_utils')) continue
     if (file.startsWith('specification/_doc_ids')) continue
     if (file.startsWith('specification/_json_spec')) continue
-    if (file === 'specification/tsconfig.json') continue
+    if (file.startsWith('specification/node_modules')) continue
+    if (file.endsWith('tsconfig.json')) continue
+    if (file.endsWith('eslint.config.js')) continue
+    if (file.endsWith('package.json')) continue
+    if (file.endsWith('package-lock.json')) continue
+    if (file.endsWith('.md')) continue
     if (getApi(file).endsWith('_types')) {
       const apis = specification.endpoints
         .filter(endpoint => endpoint.name.split('.').filter(s => !privateNames.includes(s))[0] === getApi(file).split('.')[0])

.github/validate-pr/package.json

@@ -13,6 +13,6 @@
   "dependencies": {
     "@actions/core": "^1.6.0",
     "@actions/github": "^5.0.0",
-    "zx": "^4.3.0"
+    "zx": "^8"
   }
 }

.github/workflows/code-format.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Use Node.js 20
+    - name: Use Node.js 22
       uses: actions/setup-node@v4
       with:
-        node-version: 20
+        node-version: 22
 
     - name: Install
       run: |

.github/workflows/compiler-rs.yml

@@ -14,10 +14,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Use Node.js 20
+      - name: Use Node.js 22
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
 
       - name: Install deps
         run: |

.github/workflows/compiler.test.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Use Node.js 20
+      - name: Use Node.js 22
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
 
       - name: Install deps
         run: |

.github/workflows/generate.yml

@@ -34,10 +34,10 @@ jobs:
         token: ${{ secrets.PAT }}
         persist-credentials: true
 
-    - name: Setup Node 20
+    - name: Setup Node 22
       uses: actions/setup-node@v4
       with:
-        node-version: 20
+        node-version: 22
         cache: npm
         cache-dependency-path: '**/package-lock.json'
 

.github/workflows/linter.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Use Node.js 20
+    - name: Use Node.js 22
       uses: actions/setup-node@v4
       with:
-        node-version: 20
+        node-version: 22
 
     - name: Install
       run: |

.github/workflows/update-rest-api-json.yml

@@ -20,10 +20,10 @@ jobs:
         with:
           ref: ${{ matrix.branch }}
 
-      - name: Use Node.js 20
+      - name: Use Node.js 22
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
 
       - name: Install deps
         run: |

.github/workflows/validate-pr.yml

@@ -27,10 +27,10 @@ jobs:
           token: ${{ secrets.PAT }}
           ref: main
 
-      - name: Use Node.js 20
+      - name: Use Node.js 22
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
 
       - name: Install deps 1/2
         working-directory: ./clients-flight-recorder

.nvmrc

@@ -1 +1 @@
-20
+22

Makefile

@@ -2,6 +2,7 @@ SHELL := /bin/bash
 
 validate: ## Validate a given endpoint request or response
 	@node compiler/run-validations.js --api $(api) --type $(type) --branch $(branch)
+	@npm run lint --prefix specification
 
 validate-no-cache: ## Validate a given endpoint request or response without local cache
 	@node compiler/run-validations.js --api $(api) --type $(type) --branch $(branch) --no-cache
@@ -39,6 +40,8 @@ setup:	## Install dependencies for contrib target
 	@make clean-dep
 	@npm install --prefix compiler
 	@npm install --prefix typescript-generator
+	@npm install --prefix validator
+	@npm install --prefix specification
 	@npm install @redocly/cli
 
 clean-dep:	## Clean npm dependencies
@@ -52,6 +55,9 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --output output/openapi/elasticsearch-openapi.json
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
+transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --output output/openapi/elasticsearch-openapi-docs.json
+
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
 

compiler-rs/Cargo.lock

@@ -264,7 +264,7 @@ pub struct Availability {
     pub visibility: Option<Visibility>,
 }
 
-/// The availability of an
+/// The availability of an endpoint, field or parameter
 pub type Availabilities = IndexMap<Flavor, Availability>;
 
 pub trait AvailabilityFilter: Fn(&Option<Availabilities>) -> bool {}
@@ -494,6 +494,7 @@ impl TypeDefinition {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaExample {
     pub summary: Option<String>,
+    pub method_request: Option<String>,
     pub description: Option<String>,
     pub value: Option<String>,
     pub external_value: Option<String>,

compiler-rs/clients_schema/src/lib.rs

@@ -10,7 +10,7 @@ clients_schema = {path="../clients_schema"}
 argh = { workspace = true }
 derive_more = { version = "2", features = ["from_str"] }
 serde_json = { workspace = true }
-serde_ignored = { workspace = true }
+itertools = { workspace = true }
 icu_segmenter = { workspace = true }
 openapiv3 = { workspace = true }
 anyhow = { workspace = true }

compiler-rs/clients_schema_to_openapi/Cargo.toml

@@ -20,13 +20,17 @@ pub struct Cli {
     #[argh(option, default = "SchemaFlavor::All")]
     pub flavor: SchemaFlavor,
 
-    /// add enum descriptions to property descriptions [default = true]
-    #[argh(option, default = "true")]
-    pub lift_enum_descriptions: bool,
-
     /// generate only this namespace (can be repeated)
     #[argh(option)]
     pub namespace: Vec<String>,
+
+    /// add enum descriptions to property descriptions [default = true]
+    #[argh(switch)]
+    pub lift_enum_descriptions: bool,
+
+    /// merge endpoints with multiple paths into a single OpenAPI operation [default = false]
+    #[argh(switch)]
+    pub merge_multipath_endpoints: bool,
 }
 
 use derive_more::FromStr;
@@ -42,20 +46,21 @@ pub enum SchemaFlavor {
 }
 
 impl From<Cli> for Configuration {
-    fn from(val: Cli) -> Configuration {
-        let flavor = match val.flavor {
+    fn from(cli: Cli) -> Configuration {
+        let flavor = match cli.flavor {
             SchemaFlavor::All => None,
             SchemaFlavor::Serverless => Some(Flavor::Serverless),
             SchemaFlavor::Stack => Some(Flavor::Stack),
         };
 
         Configuration {
             flavor,
-            lift_enum_descriptions: val.lift_enum_descriptions,
-            namespaces: if val.namespace.is_empty() {
+            lift_enum_descriptions: cli.lift_enum_descriptions,
+            merge_multipath_endpoints: cli.merge_multipath_endpoints,
+            namespaces: if cli.namespace.is_empty() {
                 None
             } else {
-                Some(val.namespace)
+                Some(cli.namespace)
             },
         }
     }

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -23,15 +23,24 @@ pub mod cli;
 
 use indexmap::IndexMap;
 
-use clients_schema::{Availabilities, Flavor, IndexedModel, Stability, Visibility};
+use clients_schema::{Availabilities, Availability, Flavor, IndexedModel, Stability, Visibility};
 use openapiv3::{Components, OpenAPI};
+use serde_json::Value;
 use clients_schema::transform::ExpandConfig;
 use crate::components::TypesAndComponents;
 
 pub struct Configuration {
     pub flavor: Option<Flavor>,
     pub namespaces: Option<Vec<String>>,
+
+    /// If a property value is an enumeration, the description of possible values will be copied in the
+    /// property's description (also works for arrays of enums).
     pub lift_enum_descriptions: bool,
+
+    /// Will output endpoints having multiple paths into a single operation. The operation's path will
+    /// be the longest one (with values for all optional parameters), and the other paths will be added
+    /// at the beginning of the operation's description.
+    pub merge_multipath_endpoints: bool,
 }
 
 /// Convert an API model into an OpenAPI v3 schema, optionally filtered for a given flavor
@@ -149,30 +158,37 @@ fn info(model: &IndexedModel) -> openapiv3::Info {
     }
 }
 
-pub fn availability_as_extensions(availabilities: &Option<Availabilities>) -> IndexMap<String, serde_json::Value> {
+pub fn availability_as_extensions(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>) -> IndexMap<String, serde_json::Value> {
     let mut result = IndexMap::new();
+    convert_availabilities(availabilities, flavor, &mut result);
+    result
+}
 
+pub fn convert_availabilities(availabilities: &Option<Availabilities>, flavor: &Option<Flavor>, result: &mut IndexMap<String, Value>) {
     if let Some(avails) = availabilities {
-        // We may have several availabilities, but since generally exists only on stateful (stack)
-        for (_, availability) in avails {
-            if let Some(stability) = &availability.stability {
-                match stability {
+        if let Some(flav) = flavor {
+            if let Some(availability) = avails.get(flav) {
+                let Availability {since,stability,..} = &availability;
+                let stab = stability.clone().unwrap_or(Stability::Stable);
+                let mut since_str = "".to_string();
+                if let Some(since) = since {
+                    since_str = format!("; Added in {since}");
+                }
+                match stab {
                     Stability::Beta => {
-                        result.insert("x-beta".to_string(), serde_json::Value::Bool(true));
+                        let beta_since = format!("Beta{since_str}");
+                        result.insert("x-state".to_string(), Value::String(beta_since));
                     }
                     Stability::Experimental => {
-                        result.insert("x-state".to_string(), serde_json::Value::String("Technical preview".to_string()));
+                        let exp_since = format!("Technical preview{since_str}");
+                        result.insert("x-state".to_string(), Value::String(exp_since));
                     }
-                    Stability::Stable => {
-                        if let Some(since) = &availability.since {
-                            let stable_since = "Added in ".to_string() + since;
-                            result.insert("x-state".to_string(), serde_json::Value::String(stable_since));
-                        }
+                    Stability::Stable => { 
+                        let stable_since = format!("Generally available{since_str}");
+                        result.insert("x-state".to_string(), Value::String(stable_since));
                     }
                 }
             }
         }
     }
-
-    result
 }