[Backport 7.x] Implement wipe cluster settings for YAML tests (#5531)

* Implement wipe cluster settings for YAML tests (#5527)

* Implement wipe cluster settings
* Unskip cluster settings tests
* Remove redundant skip
* Update skip comment
* Reuse settings for return
* Unskip cluster voting config
* Update skip comment
* Cleanup file
* Unskip a test and add extra comments
* Remove ML skip
* Remove skip which now passes
* Remove skip where file does not exist
* Cleanup

(cherry picked from commit ddc2801)

* Fix namespace

Author: stevejgordon

tests/Tests.YamlRunner/SkipList.fs

@@ -10,32 +10,24 @@ type SkipFile = SkipFile of string
 
 let SkipList = dict<SkipFile,SkipSection> [
 
-    // Need to implement cluster settings cleanup
-    SkipFile "cluster.put_settings/10_basic.yml", All
-    
+    // TODO: Needs investigation. "Setting upgrade mode to disabled from enabled" runs in isolation but not with the entire test file
     SkipFile "ml/set_upgrade_mode.yml", All
     SkipFile "ml/inference_crud.yml", Section "Test update model alias with model id referring to missing model"
+
+    // Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
+    // resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
     SkipFile "ml/start_stop_datafeed.yml", All
+
     SkipFile "ml/post_data.yml", All
-    
-    // These send empty strings for required parameters
-    // TODO i THINK this is now supported
-    SkipFile "ml/explain_data_frame_analytics.yml", Section "Test neither job id nor body"
-    
+
     // funny looking dispatch /_security/privilege/app?name
     SkipFile "privileges/10_basic.yml", All
-    
-    // 7.x only
-    // We skip the generation of this API till one of the later minors
-    SkipFile "indices.upgrade/10_basic.yml", All
+
     // Sets a dictionary to null, we need to see if we can backport this from master
     SkipFile "search.aggregation/240_max_buckets.yml", All
     SkipFile "search.aggregation/180_percentiles_tdigest_metric.yml", Section "Invalid params test"
     SkipFile "search.aggregation/190_percentiles_hdr_metric.yml", Section "Invalid params test"
 
-    // - Failed: Assert operation NumericAssert Length invalidated_api_keys "Long" Reason: Expected 2.000000 = 3.000000        
-    SkipFile "api_key/11_invalidation.yml", Section "Test invalidate api key by realm name"
-
     // Test looks for "testnode.crt", but "ca.crt" is returned first
     SkipFile "ssl/10_basic.yml", Section "Test get SSL certificates"
 
@@ -58,7 +50,6 @@ let SkipList = dict<SkipFile,SkipSection> [
 
     // Missing refreshes in the test
     SkipFile "data_frame/transforms_start_stop.yml", All
-    SkipFile "ml/index_layout.yml", All
 
     // Todo investigate
     SkipFile "transform/transforms_start_stop.yml", Sections [
@@ -70,12 +61,14 @@ let SkipList = dict<SkipFile,SkipSection> [
         "Test get multiple transform stats"
         "Test get multiple transform stats where one does not have a task"
     ]
+
     // More QA tests than API tests
     SkipFile "data_frame/transforms_stats.yml", Sections [
         "Test get multiple transform stats"
         "Test get transform stats on missing transform"
         "Test get multiple transform stats where one does not have a task"
     ]
+
     // Invalid license makes subsequent tests fail
     SkipFile "license/20_put_license.yml", All
 
@@ -103,18 +96,15 @@ let SkipList = dict<SkipFile,SkipSection> [
         "Test open and close with non-existent job id"
     ]
     // Failed: Actions custom Setup actions Reason: Setup
-    SkipFile "ml/stop_data_frame_analytics.yml", Section "Test stop given missing config and allow_no_match is true"
-    SkipFile "ml/start_stop_datafeed.yml", Section "Test stop given expression"
+    SkipFile "ml/stop_data_frame_analytics.yml", Sections [
+        "Test stop given missing config and allow_no_match is true"
+        "Test stop given missing config and allow_no_match is false"
+    ]
+
     SkipFile "transform/transforms_start_stop.yml", Sections [
         "Test start transform"  
         "Verify start transform reuses destination index"
     ]
-    // Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
-    // resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
-    SkipFile "ml/start_stop_datafeed.yml", Sections [
-        "Test start datafeed when persistent task allocation disabled"
-        "Test start given field without mappings"
-    ]
     // Indexing step doesn't appear to work (getting total.hits=0)
     SkipFile "monitoring/bulk/10_basic.yml",
         Section "Bulk indexing of monitoring data on closed indices should throw an export exception"
@@ -130,8 +120,7 @@ let SkipList = dict<SkipFile,SkipSection> [
     ]
     // TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
     SkipFile "snapshot/10_basic.yml", Section "Create a source only snapshot and then restore it"
-    // illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
-    SkipFile "users/10_basic.yml", Section "Test put user with password hash"
+
     // Slash in index name is not escaped (BUG)
     SkipFile "security/authz/13_index_datemath.yml", Section "Test indexing documents with datemath, when permitted"
     // Possibly a cluster health color mismatch...
@@ -149,9 +138,6 @@ let SkipList = dict<SkipFile,SkipSection> [
     ]
     // Cannot connect to Docker IP
     SkipFile "watcher/execute_watch/60_http_input.yml", All
-    // Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
-    // Also test does too much within a single test, so has to be disabled as whole, unfortunately.
-    SkipFile "xpack/15_basic.yml", All
 
     // Snapshot testing requires local filesystem access
     SkipFile "snapshot.create/10_basic.yml", All
@@ -161,24 +147,16 @@ let SkipList = dict<SkipFile,SkipSection> [
     SkipFile "snapshot.status/10_basic.yml", All
 
     // Datastreams are currently experimental
-    SkipFile "indices.data_stream/10_basic.yml", All
+    //SkipFile "indices.data_stream/10_basic.yml", All
 
     // uses $stashed id in match with object
     SkipFile "cluster.reroute/11_explain.yml", Sections [
         "Explain API for non-existent node & shard"
     ]
-    
-    //These are ignored because they were flagged on a big PR.
-    
-    //additional enters in regex
-    SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ]
-    
-    //Replace stashed value in body that is passed as string json
-    SkipFile "api_key/10_basic.yml", Section "Test get api key"
 
-    //new API TODO remove when we regenerate
-    SkipFile "cluster.voting_config_exclusions/10_basic.yml", All
-    
+    // Additional entries in regex: Failed cat.templates 10_basic.yml: Assert operation Match $body RegexAssertion
+    SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ]
+
     //TODO has dates without strings which trips up our yaml parser
     SkipFile "runtime_fields/40_date.yml", All
     // double / int in object comparison
@@ -194,7 +172,7 @@ let SkipList = dict<SkipFile,SkipSection> [
 
     SkipFile "ml/inference_processor.yml", Section "Test simulate"
 
-    // TODO investigate post 7.11.0
+    // TODO: Review again soon once zip includes updated test file from https://github.com/elastic/elasticsearch/pull/71084
     SkipFile "nodes.info/10_basic.yml", Section "node_info role test"
 
     // TODO investigate

tests/Tests.YamlRunner/TestSuiteBootstrap.fs

@@ -11,6 +11,7 @@ open Elasticsearch.Net.Specification.CatApi
 open Elasticsearch.Net.Specification.IndicesApi
 open Tests.YamlRunner.Models
 open System.Collections.Generic
+open Elasticsearch.Net.Specification.ClusterApi
 
 let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->
     let firstFailure (responses:DynamicResponse seq) =
@@ -135,8 +136,17 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->
     let wipeTemplateForXPack () = deleteTemplates() @ deleteComponentTemplates()
 
     let wipeClusterSettings () =
-        let settings = client.Cluster.GetSettings<DynamicResponse>()
-        settings
+        let settings = client.Cluster.GetSettings<DynamicResponse>(ClusterGetSettingsRequestParameters(FlatSettings=true))
+        let payload =
+            [ 
+                "transient", dict [ for v in settings.Get<DynamicDictionary>("transient").Keys -> v, null ];
+                "persistent", dict [ for v in settings.Get<DynamicDictionary>("persistent").Keys -> v, null ];
+            ]
+            |> dict
+        if payload.["transient"].Values.Count > 0 || payload.["transient"].Values.Count > 0 then
+            client.Cluster.PutSettings<DynamicResponse>(PostData.Serializable(payload))
+        else 
+            settings
 
     let deleteAllILMPolicies () =
         let preserved = [
@@ -248,7 +258,6 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->
 
         yield! stopTransforms()
 
-            
         if suite = Platinum then
             let data = PostData.String @"{""password"":""x-pack-test-password"", ""roles"":[""superuser""]}"
             yield client.Security.PutUser<DynamicResponse>("x_pack_rest_user", data)