From 68feec9d680e6d45592dc09cb98da36c5975300c Mon Sep 17 00:00:00 2001 From: Steve Gordon Date: Tue, 6 Apr 2021 16:45:47 +0100 Subject: [PATCH 1/2] Implement wipe cluster settings for YAML tests (#5527) * Implement wipe cluster settings * Unskip cluster settings tests * Remove redundant skip * Update skip comment * Reuse settings for return * Unskip cluster voting config * Update skip comment * Cleanup file * Unskip a test and add extra comments * Remove ML skip * Remove skip which now passes * Remove skip where file does not exist * Cleanup (cherry picked from commit ddc280194c1dd122f05f980aabc774c25325a14e) --- tests/Tests.YamlRunner/SkipList.fs | 62 +++++++------------- tests/Tests.YamlRunner/TestSuiteBootstrap.fs | 14 ++++- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/tests/Tests.YamlRunner/SkipList.fs b/tests/Tests.YamlRunner/SkipList.fs index f406166db53..45fb3720bfc 100644 --- a/tests/Tests.YamlRunner/SkipList.fs +++ b/tests/Tests.YamlRunner/SkipList.fs @@ -10,32 +10,24 @@ type SkipFile = SkipFile of string let SkipList = dict [ - // Need to implement cluster settings cleanup - SkipFile "cluster.put_settings/10_basic.yml", All - + // TODO: Needs investigation. "Setting upgrade mode to disabled from enabled" runs in isolation but not with the entire test file SkipFile "ml/set_upgrade_mode.yml", All SkipFile "ml/inference_crud.yml", Section "Test update model alias with model id referring to missing model" + + // Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened + // resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist SkipFile "ml/start_stop_datafeed.yml", All + SkipFile "ml/post_data.yml", All - - // These send empty strings for required parameters - // TODO i THINK this is now supported - SkipFile "ml/explain_data_frame_analytics.yml", Section "Test neither job id nor body" - + // funny looking dispatch /_security/privilege/app?name SkipFile "privileges/10_basic.yml", All - - // 7.x only - // We skip the generation of this API till one of the later minors - SkipFile "indices.upgrade/10_basic.yml", All + // Sets a dictionary to null, we need to see if we can backport this from master SkipFile "search.aggregation/240_max_buckets.yml", All SkipFile "search.aggregation/180_percentiles_tdigest_metric.yml", Section "Invalid params test" SkipFile "search.aggregation/190_percentiles_hdr_metric.yml", Section "Invalid params test" - // - Failed: Assert operation NumericAssert Length invalidated_api_keys "Long" Reason: Expected 2.000000 = 3.000000 - SkipFile "api_key/11_invalidation.yml", Section "Test invalidate api key by realm name" - // Test looks for "testnode.crt", but "ca.crt" is returned first SkipFile "ssl/10_basic.yml", Section "Test get SSL certificates" @@ -58,7 +50,6 @@ let SkipList = dict [ // Missing refreshes in the test SkipFile "data_frame/transforms_start_stop.yml", All - SkipFile "ml/index_layout.yml", All // Todo investigate SkipFile "transform/transforms_start_stop.yml", Sections [ @@ -70,12 +61,14 @@ let SkipList = dict [ "Test get multiple transform stats" "Test get multiple transform stats where one does not have a task" ] + // More QA tests than API tests SkipFile "data_frame/transforms_stats.yml", Sections [ "Test get multiple transform stats" "Test get transform stats on missing transform" "Test get multiple transform stats where one does not have a task" ] + // Invalid license makes subsequent tests fail SkipFile "license/20_put_license.yml", All @@ -103,18 +96,15 @@ let SkipList = dict [ "Test open and close with non-existent job id" ] // Failed: Actions custom Setup actions Reason: Setup - SkipFile "ml/stop_data_frame_analytics.yml", Section "Test stop given missing config and allow_no_match is true" - SkipFile "ml/start_stop_datafeed.yml", Section "Test stop given expression" + SkipFile "ml/stop_data_frame_analytics.yml", Sections [ + "Test stop given missing config and allow_no_match is true" + "Test stop given missing config and allow_no_match is false" + ] + SkipFile "transform/transforms_start_stop.yml", Sections [ "Test start transform" "Verify start transform reuses destination index" ] - // Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened - // resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist - SkipFile "ml/start_stop_datafeed.yml", Sections [ - "Test start datafeed when persistent task allocation disabled" - "Test start given field without mappings" - ] // Indexing step doesn't appear to work (getting total.hits=0) SkipFile "monitoring/bulk/10_basic.yml", Section "Bulk indexing of monitoring data on closed indices should throw an export exception" @@ -130,8 +120,7 @@ let SkipList = dict [ ] // TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR SkipFile "snapshot/10_basic.yml", Section "Create a source only snapshot and then restore it" - // illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT] - SkipFile "users/10_basic.yml", Section "Test put user with password hash" + // Slash in index name is not escaped (BUG) SkipFile "security/authz/13_index_datemath.yml", Section "Test indexing documents with datemath, when permitted" // Possibly a cluster health color mismatch... @@ -149,9 +138,6 @@ let SkipList = dict [ ] // Cannot connect to Docker IP SkipFile "watcher/execute_watch/60_http_input.yml", All - // Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API. - // Also test does too much within a single test, so has to be disabled as whole, unfortunately. - SkipFile "xpack/15_basic.yml", All // Snapshot testing requires local filesystem access SkipFile "snapshot.create/10_basic.yml", All @@ -161,24 +147,16 @@ let SkipList = dict [ SkipFile "snapshot.status/10_basic.yml", All // Datastreams are currently experimental - SkipFile "indices.data_stream/10_basic.yml", All + //SkipFile "indices.data_stream/10_basic.yml", All // uses $stashed id in match with object SkipFile "cluster.reroute/11_explain.yml", Sections [ "Explain API for non-existent node & shard" ] - - //These are ignored because they were flagged on a big PR. - - //additional enters in regex - SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ] - - //Replace stashed value in body that is passed as string json - SkipFile "api_key/10_basic.yml", Section "Test get api key" - //new API TODO remove when we regenerate - SkipFile "cluster.voting_config_exclusions/10_basic.yml", All - + // Additional entries in regex: Failed cat.templates 10_basic.yml: Assert operation Match $body RegexAssertion + SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ] + //TODO has dates without strings which trips up our yaml parser SkipFile "runtime_fields/40_date.yml", All // double / int in object comparison @@ -194,7 +172,7 @@ let SkipList = dict [ SkipFile "ml/inference_processor.yml", Section "Test simulate" - // TODO investigate post 7.11.0 + // TODO: Review again soon once zip includes updated test file from https://github.com/elastic/elasticsearch/pull/71084 SkipFile "nodes.info/10_basic.yml", Section "node_info role test" // TODO investigate diff --git a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs index 8c5635ad96d..9b2381c094c 100644 --- a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs +++ b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs @@ -135,8 +135,17 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) -> let wipeTemplateForXPack () = deleteTemplates() @ deleteComponentTemplates() let wipeClusterSettings () = - let settings = client.Cluster.GetSettings() - settings + let settings = client.Cluster.GetSettings(ClusterGetSettingsRequestParameters(FlatSettings=true)) + let payload = + [ + "transient", dict [ for v in settings.Get("transient").Keys -> v, null ]; + "persistent", dict [ for v in settings.Get("persistent").Keys -> v, null ]; + ] + |> dict + if payload.["transient"].Values.Count > 0 || payload.["transient"].Values.Count > 0 then + client.Cluster.PutSettings(PostData.Serializable(payload)) + else + settings let deleteAllILMPolicies () = let preserved = [ @@ -248,7 +257,6 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) -> yield! stopTransforms() - if suite = Platinum then let data = PostData.String @"{""password"":""x-pack-test-password"", ""roles"":[""superuser""]}" yield client.Security.PutUser("x_pack_rest_user", data) From a2887f150a2290a608c609092a7e5b856804fc5a Mon Sep 17 00:00:00 2001 From: Steve Gordon Date: Thu, 8 Apr 2021 09:57:46 +0100 Subject: [PATCH 2/2] Fix namespace --- tests/Tests.YamlRunner/TestSuiteBootstrap.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs index 9b2381c094c..78ce43aaee8 100644 --- a/tests/Tests.YamlRunner/TestSuiteBootstrap.fs +++ b/tests/Tests.YamlRunner/TestSuiteBootstrap.fs @@ -11,6 +11,7 @@ open Elasticsearch.Net.Specification.CatApi open Elasticsearch.Net.Specification.IndicesApi open Tests.YamlRunner.Models open System.Collections.Generic +open Elasticsearch.Net.Specification.ClusterApi let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) -> let firstFailure (responses:DynamicResponse seq) =