Add yaml test runner project

.ci/run-elasticsearch.ps1

@@ -217,7 +217,7 @@ $volumes = @(
 "--volume", "${volume_name}:/usr/share/elasticsearch/data"
 )
 
-if (-not ($version -contains "oss")) {
+if (-not ($version -match "oss")) {
   $environment += @(
   "--env", "ELASTIC_PASSWORD=`"$ELASTIC_PASSWORD`"",
   "--env", "xpack.license.self_generated.type=trial",
@@ -241,7 +241,7 @@ if (-not ($version -contains "oss")) {
 }
 
 $url="http://$NODE_NAME"
-if (-not ($version -contains "oss")) {
+if (-not ($version -match "oss")) {
   $url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME"
 }
 

.gitignore

@@ -5,4 +5,7 @@ Cargo.lock
 .idea
 .vscode/
 *.log
-yaml_test_runner/
+yaml_test_runner/yaml/
+yaml_test_runner/tests/oss
+yaml_test_runner/tests/xpack
+yaml_test_runner/tests/mod.rs

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
     "api_generator",
-    "elasticsearch"
+    "elasticsearch",
+    "yaml_test_runner"
 ]

api_generator/src/main.rs → api_generator/src/bin/run.rs

@@ -16,36 +16,28 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+extern crate api_generator;
 extern crate dialoguer;
 
-#[macro_use]
-extern crate lazy_static;
-
-#[macro_use]
-extern crate quote;
-
+use api_generator::{generator, rest_spec};
 use dialoguer::Input;
 use std::path::PathBuf;
 use std::{
     fs::{self, File},
     io::Write,
-    path::Path,
 };
 
-mod api_generator;
-mod error;
-mod rest_spec;
-
-fn main() {
+fn main() -> Result<(), failure::Error> {
     // This must be run from the src root directory, with cargo run -p api_generator
-    let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs")).unwrap();
-    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src/generated")).unwrap();
-    let last_downloaded_version = "./api_generator/last_downloaded_version";
+    let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
+    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src/generated"))?;
+    let last_downloaded_version =
+        PathBuf::from("./api_generator/rest_specs/last_downloaded_version");
 
     let mut download_specs = false;
     let mut answer = String::new();
-    let default_branch = if Path::new(last_downloaded_version).exists() {
-        fs::read_to_string(last_downloaded_version).expect("Could not read branch into string")
+    let default_branch = if last_downloaded_version.exists() {
+        fs::read_to_string(&last_downloaded_version)?
     } else {
         String::from("master")
     };
@@ -76,13 +68,9 @@ fn main() {
             .interact()
             .unwrap();
 
-        fs::remove_dir_all(&download_dir).unwrap();
-        rest_spec::download_specs(&branch, &download_dir);
-
-        File::create(last_downloaded_version)
-            .expect("failed to create last_downloaded_version file")
-            .write_all(branch.as_bytes())
-            .expect("unable to write branch to last_downloaded_version file");
+        fs::remove_dir_all(&download_dir)?;
+        rest_spec::download_specs(&branch, &download_dir)?;
+        File::create(&last_downloaded_version)?.write_all(branch.as_bytes())?;
     }
 
     // only offer to generate if there are downloaded specs
@@ -109,12 +97,13 @@ fn main() {
         if generate_code {
             // delete existing generated files if the exist
             if generated_dir.exists() {
-                fs::remove_dir_all(&generated_dir).unwrap();
+                fs::remove_dir_all(&generated_dir)?;
             }
 
-            fs::create_dir_all(&generated_dir).unwrap();
-
-            api_generator::generate(&branch, &download_dir, &generated_dir).unwrap();
+            fs::create_dir_all(&generated_dir)?;
+            generator::generate(&branch, &download_dir, &generated_dir)?;
         }
     }
+
+    Ok(())
 }

api_generator/src/api_generator/code_gen/mod.rs → api_generator/src/generator/code_gen/mod.rs

@@ -22,7 +22,7 @@ pub mod request;
 pub mod root;
 pub mod url;
 
-use crate::api_generator::TypeKind;
+use crate::generator::TypeKind;
 use inflector::Inflector;
 use quote::Tokens;
 use std::str;
@@ -73,7 +73,7 @@ pub fn parse_expr(input: quote::Tokens) -> syn::Expr {
 }
 
 /// Ensures that the name generated is one that is valid for Rust
-fn valid_name(s: &str) -> &str {
+pub fn valid_name(s: &str) -> &str {
     match s {
         "type" => "ty",
         s => s,
@@ -116,7 +116,7 @@ impl GetPath for syn::Ty {
     fn get_path(&self) -> &syn::Path {
         match *self {
             syn::Ty::Path(_, ref p) => &p,
-            _ => panic!("Only path types are supported."),
+            ref p => panic!(format!("Expected syn::Ty::Path, but found {:?}", p)),
         }
     }
 }
@@ -138,21 +138,42 @@ impl<T: GetPath> GetIdent for T {
 }
 
 /// Gets the Ty syntax token for a TypeKind
-fn typekind_to_ty(name: &str, kind: &TypeKind, required: bool) -> syn::Ty {
+/// TODO: This function is serving too many purposes. Refactor it
+fn typekind_to_ty(name: &str, kind: &TypeKind, required: bool, fn_arg: bool) -> syn::Ty {
     let mut v = String::new();
     if !required {
         v.push_str("Option<");
     }
 
     let str_type = "&'b str";
     match kind {
-        TypeKind::None => v.push_str(str_type),
-        TypeKind::List => v.push_str(format!("&'b [{}]", str_type).as_ref()),
-        TypeKind::Enum => v.push_str(name.to_pascal_case().as_str()),
+        TypeKind::Unknown(_) => v.push_str(str_type),
+        TypeKind::List => {
+            v.push_str("&'b [");
+            v.push_str(str_type);
+            v.push_str("]");
+        }
+        TypeKind::Enum => match name {
+            // opened https://github.com/elastic/elasticsearch/issues/53212
+            // to discuss whether this really should be a collection
+            "expand_wildcards" => {
+                // Expand wildcards should
+                v.push_str("&'b [");
+                v.push_str(name.to_pascal_case().as_str());
+                v.push_str("]");
+            }
+            _ => v.push_str(name.to_pascal_case().as_str()),
+        },
         TypeKind::String => v.push_str(str_type),
         TypeKind::Text => v.push_str(str_type),
         TypeKind::Boolean => match name {
-            "track_total_hits" => v.push_str("TrackTotalHits"),
+            "track_total_hits" => {
+                if fn_arg {
+                    v.push_str(format!("Into<{}>", name.to_pascal_case()).as_str())
+                } else {
+                    v.push_str(name.to_pascal_case().as_str())
+                }
+            }
             _ => v.push_str("bool"),
         },
         TypeKind::Number => v.push_str("i64"),

api_generator/src/api_generator/code_gen/namespace_clients.rs → api_generator/src/generator/code_gen/namespace_clients.rs

@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-use crate::api_generator::*;
-use crate::api_generator::code_gen::request::request_builder::RequestBuilder;
-use crate::api_generator::code_gen::*;
+use crate::generator::code_gen::request::request_builder::RequestBuilder;
+use crate::generator::code_gen::*;
+use crate::generator::*;
 use inflector::Inflector;
 use quote::Tokens;
 use std::path::PathBuf;

api_generator/src/api_generator/code_gen/params.rs → api_generator/src/generator/code_gen/params.rs

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-use crate::api_generator::*;
+use crate::generator::*;
 use inflector::Inflector;
 use quote::Tokens;
 use regex::Regex;

api_generator/src/api_generator/code_gen/request/request_builder.rs → api_generator/src/generator/code_gen/request/request_builder.rs

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-use crate::api_generator::{
+use crate::generator::{
     code_gen, code_gen::url::enum_builder::EnumBuilder, code_gen::*, ApiEndpoint, HttpMethod, Type,
     TypeKind,
 };
@@ -25,7 +25,7 @@ use quote::{ToTokens, Tokens};
 use reqwest::Url;
 use std::path::PathBuf;
 use std::{collections::BTreeMap, fs, str};
-use syn::{Field, FieldValue, ImplItem};
+use syn::{Field, FieldValue, ImplItem, TraitBoundModifier, TyParamBound};
 
 /// Builder that generates the AST for a request builder struct
 pub struct RequestBuilder<'a> {
@@ -126,7 +126,8 @@ impl<'a> RequestBuilder<'a> {
             let struct_fields = endpoint_params.iter().map(|(param_name, param_type)| {
                 let field = Self::create_struct_field((param_name, param_type));
                 let field_rename = lit(param_name);
-                if param_type.ty == TypeKind::List {
+                // TODO: we special case expand_wildcards here to be a list, but this should be fixed upstream
+                if param_type.ty == TypeKind::List || param_name == "expand_wildcards" {
                     let serialize_with = lit("crate::client::serialize_coll_qs");
                     quote! {
                         #[serde(rename = #field_rename, serialize_with = #serialize_with)]
@@ -352,10 +353,36 @@ impl<'a> RequestBuilder<'a> {
     /// Creates the AST for a builder fn for a builder impl
     fn create_impl_fn(f: (&String, &Type)) -> syn::ImplItem {
         let name = valid_name(&f.0).to_lowercase();
+        let (ty, value_ident, fn_generics) = {
+            let ty = typekind_to_ty(&f.0, &f.1.ty, true, true);
+            match ty {
+                syn::Ty::Path(ref _q, ref p) => {
+                    if p.get_ident().as_ref() == "Into" {
+                        let ty = syn::parse_type("T").unwrap();
+                        let ident = code_gen::ident(format!("{}.into()", &name));
+                        let ty_param = syn::TyParam {
+                            ident: code_gen::ident("T"),
+                            default: None,
+                            attrs: vec![],
+                            bounds: vec![TyParamBound::Trait(
+                                syn::PolyTraitRef {
+                                    trait_ref: p.clone(),
+                                    bound_lifetimes: vec![],
+                                },
+                                TraitBoundModifier::None,
+                            )],
+                        };
+                        let generics = generics(vec![], vec![ty_param]);
+                        (ty, ident, generics)
+                    } else {
+                        (ty, ident(&name), generics_none())
+                    }
+                }
+                _ => (ty, ident(&name), generics_none()),
+            }
+        };
         let impl_ident = ident(&name);
         let field_ident = ident(&name);
-        let value_ident = ident(&name);
-        let ty = typekind_to_ty(&f.0, &f.1.ty, true);
         let doc_attr = match &f.1.description {
             Some(docs) => vec![doc(docs)],
             _ => vec![],
@@ -382,7 +409,7 @@ impl<'a> RequestBuilder<'a> {
                         output: syn::FunctionRetTy::Ty(code_gen::ty("Self")),
                         variadic: false,
                     },
-                    generics: generics_none(),
+                    generics: fn_generics,
                 },
                 // generates a fn body of the form
                 // --------
@@ -409,13 +436,11 @@ impl<'a> RequestBuilder<'a> {
         enum_builder: &EnumBuilder,
         accepts_nd_body: bool,
     ) -> Tokens {
-        // TODO: lazy_static! for this?
         let mut common_fields: Vec<Field> = common_params
             .iter()
             .map(Self::create_struct_field)
             .collect();
 
-        // TODO: lazy_static! for this?
         let mut common_builder_fns: Vec<ImplItem> =
             common_params.iter().map(Self::create_impl_fn).collect();
 
@@ -667,7 +692,7 @@ impl<'a> RequestBuilder<'a> {
             ident: Some(ident(valid_name(&f.0).to_lowercase())),
             vis: syn::Visibility::Inherited,
             attrs: vec![],
-            ty: typekind_to_ty(&f.0, &f.1.ty, false),
+            ty: typekind_to_ty(&f.0, &f.1.ty, false, false),
         }
     }
 

api_generator/src/api_generator/code_gen/root.rs → api_generator/src/generator/code_gen/root.rs

@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-use crate::api_generator::*;
-use crate::api_generator::code_gen::request::request_builder::RequestBuilder;
-use crate::api_generator::code_gen::*;
+use crate::generator::code_gen::request::request_builder::RequestBuilder;
+use crate::generator::code_gen::*;
+use crate::generator::*;
 use inflector::Inflector;
 use quote::Tokens;
 use std::path::PathBuf;

api_generator/src/api_generator/code_gen/url/enum_builder.rs → api_generator/src/generator/code_gen/url/enum_builder.rs

@@ -32,8 +32,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-use crate::api_generator::code_gen::url::url_builder::{IntoExpr, UrlBuilder};
-use crate::api_generator::{code_gen::*, ApiEndpoint, Path};
+use crate::generator::code_gen::url::url_builder::{IntoExpr, UrlBuilder};
+use crate::generator::{code_gen::*, ApiEndpoint, Path};
 use inflector::Inflector;
 
 /// Builder for request url parts enum
@@ -139,7 +139,7 @@ impl<'a> EnumBuilder<'a> {
                             ident: None,
                             vis: syn::Visibility::Inherited,
                             attrs: vec![],
-                            ty: typekind_to_ty(p, ty, true),
+                            ty: typekind_to_ty(p, ty, true, false),
                         }
                     })
                     .collect(),
@@ -306,15 +306,16 @@ mod tests {
     #![cfg_attr(rustfmt, rustfmt_skip)]
 
     use super::*;
-    use crate::api_generator::{Url, Path, HttpMethod, Body, Deprecated, Type, TypeKind, Documentation, ast_eq};
+    use crate::generator::{Url, Path, HttpMethod, Body, Deprecated, Type, TypeKind, Documentation, ast_eq};
     use std::collections::BTreeMap;
-    use crate::api_generator::code_gen::url::url_builder::PathString;
+    use crate::generator::code_gen::url::url_builder::PathString;
 
     #[test]
     fn generate_parts_enum_from_endpoint() {
         let endpoint = (
             "search".to_string(),
             ApiEndpoint {
+                full_name: Some("search".to_string()),
                 documentation: Documentation {
                     description: None,
                     url: None,