Skip to content

[Backport 7.x] Implement wipe cluster settings for YAML tests #5531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 8, 2021
Merged

[Backport 7.x] Implement wipe cluster settings for YAML tests #5531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
68feec9
Implement wipe cluster settings for YAML tests (#5527)
stevejgordon Apr 6, 2021
a2887f1
Fix namespace
stevejgordon Apr 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 20 additions & 42 deletions tests/Tests.YamlRunner/SkipList.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,32 +10,24 @@ type SkipFile = SkipFile of string

let SkipList = dict<SkipFile,SkipSection> [

// Need to implement cluster settings cleanup
SkipFile "cluster.put_settings/10_basic.yml", All

// TODO: Needs investigation. "Setting upgrade mode to disabled from enabled" runs in isolation but not with the entire test file
SkipFile "ml/set_upgrade_mode.yml", All
SkipFile "ml/inference_crud.yml", Section "Test update model alias with model id referring to missing model"

// Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
// resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
SkipFile "ml/start_stop_datafeed.yml", All

SkipFile "ml/post_data.yml", All

// These send empty strings for required parameters
// TODO i THINK this is now supported
SkipFile "ml/explain_data_frame_analytics.yml", Section "Test neither job id nor body"


// funny looking dispatch /_security/privilege/app?name
SkipFile "privileges/10_basic.yml", All

// 7.x only
// We skip the generation of this API till one of the later minors
SkipFile "indices.upgrade/10_basic.yml", All

// Sets a dictionary to null, we need to see if we can backport this from master
SkipFile "search.aggregation/240_max_buckets.yml", All
SkipFile "search.aggregation/180_percentiles_tdigest_metric.yml", Section "Invalid params test"
SkipFile "search.aggregation/190_percentiles_hdr_metric.yml", Section "Invalid params test"

// - Failed: Assert operation NumericAssert Length invalidated_api_keys "Long" Reason: Expected 2.000000 = 3.000000
SkipFile "api_key/11_invalidation.yml", Section "Test invalidate api key by realm name"

// Test looks for "testnode.crt", but "ca.crt" is returned first
SkipFile "ssl/10_basic.yml", Section "Test get SSL certificates"

Expand All @@ -58,7 +50,6 @@ let SkipList = dict<SkipFile,SkipSection> [

// Missing refreshes in the test
SkipFile "data_frame/transforms_start_stop.yml", All
SkipFile "ml/index_layout.yml", All

// Todo investigate
SkipFile "transform/transforms_start_stop.yml", Sections [
Expand All @@ -70,12 +61,14 @@ let SkipList = dict<SkipFile,SkipSection> [
"Test get multiple transform stats"
"Test get multiple transform stats where one does not have a task"
]

// More QA tests than API tests
SkipFile "data_frame/transforms_stats.yml", Sections [
"Test get multiple transform stats"
"Test get transform stats on missing transform"
"Test get multiple transform stats where one does not have a task"
]

// Invalid license makes subsequent tests fail
SkipFile "license/20_put_license.yml", All

Expand Down Expand Up @@ -103,18 +96,15 @@ let SkipList = dict<SkipFile,SkipSection> [
"Test open and close with non-existent job id"
]
// Failed: Actions custom Setup actions Reason: Setup
SkipFile "ml/stop_data_frame_analytics.yml", Section "Test stop given missing config and allow_no_match is true"
SkipFile "ml/start_stop_datafeed.yml", Section "Test stop given expression"
SkipFile "ml/stop_data_frame_analytics.yml", Sections [
"Test stop given missing config and allow_no_match is true"
"Test stop given missing config and allow_no_match is false"
]

SkipFile "transform/transforms_start_stop.yml", Sections [
"Test start transform"
"Verify start transform reuses destination index"
]
// Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
// resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
SkipFile "ml/start_stop_datafeed.yml", Sections [
"Test start datafeed when persistent task allocation disabled"
"Test start given field without mappings"
]
// Indexing step doesn't appear to work (getting total.hits=0)
SkipFile "monitoring/bulk/10_basic.yml",
Section "Bulk indexing of monitoring data on closed indices should throw an export exception"
Expand All @@ -130,8 +120,7 @@ let SkipList = dict<SkipFile,SkipSection> [
]
// TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
SkipFile "snapshot/10_basic.yml", Section "Create a source only snapshot and then restore it"
// illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
SkipFile "users/10_basic.yml", Section "Test put user with password hash"

// Slash in index name is not escaped (BUG)
SkipFile "security/authz/13_index_datemath.yml", Section "Test indexing documents with datemath, when permitted"
// Possibly a cluster health color mismatch...
Expand All @@ -149,9 +138,6 @@ let SkipList = dict<SkipFile,SkipSection> [
]
// Cannot connect to Docker IP
SkipFile "watcher/execute_watch/60_http_input.yml", All
// Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
// Also test does too much within a single test, so has to be disabled as whole, unfortunately.
SkipFile "xpack/15_basic.yml", All

// Snapshot testing requires local filesystem access
SkipFile "snapshot.create/10_basic.yml", All
Expand All @@ -161,24 +147,16 @@ let SkipList = dict<SkipFile,SkipSection> [
SkipFile "snapshot.status/10_basic.yml", All

// Datastreams are currently experimental
SkipFile "indices.data_stream/10_basic.yml", All
//SkipFile "indices.data_stream/10_basic.yml", All

// uses $stashed id in match with object
SkipFile "cluster.reroute/11_explain.yml", Sections [
"Explain API for non-existent node & shard"
]

//These are ignored because they were flagged on a big PR.

//additional enters in regex
SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ]

//Replace stashed value in body that is passed as string json
SkipFile "api_key/10_basic.yml", Section "Test get api key"

//new API TODO remove when we regenerate
SkipFile "cluster.voting_config_exclusions/10_basic.yml", All
// Additional entries in regex: Failed cat.templates 10_basic.yml: Assert operation Match $body RegexAssertion
SkipFile "cat.templates/10_basic.yml", Sections [ "Multiple template"; "Sort templates"; "No templates" ]

//TODO has dates without strings which trips up our yaml parser
SkipFile "runtime_fields/40_date.yml", All
// double / int in object comparison
Expand All @@ -194,7 +172,7 @@ let SkipList = dict<SkipFile,SkipSection> [

SkipFile "ml/inference_processor.yml", Section "Test simulate"

// TODO investigate post 7.11.0
// TODO: Review again soon once zip includes updated test file from https://github.com/elastic/elasticsearch/pull/71084
SkipFile "nodes.info/10_basic.yml", Section "node_info role test"

// TODO investigate
Expand Down
15 changes: 12 additions & 3 deletions tests/Tests.YamlRunner/TestSuiteBootstrap.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ open Elasticsearch.Net.Specification.CatApi
open Elasticsearch.Net.Specification.IndicesApi
open Tests.YamlRunner.Models
open System.Collections.Generic
open Elasticsearch.Net.Specification.ClusterApi

let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->
let firstFailure (responses:DynamicResponse seq) =
Expand Down Expand Up @@ -135,8 +136,17 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->
let wipeTemplateForXPack () = deleteTemplates() @ deleteComponentTemplates()

let wipeClusterSettings () =
let settings = client.Cluster.GetSettings<DynamicResponse>()
settings
let settings = client.Cluster.GetSettings<DynamicResponse>(ClusterGetSettingsRequestParameters(FlatSettings=true))
let payload =
[
"transient", dict [ for v in settings.Get<DynamicDictionary>("transient").Keys -> v, null ];
"persistent", dict [ for v in settings.Get<DynamicDictionary>("persistent").Keys -> v, null ];
]
|> dict
if payload.["transient"].Values.Count > 0 || payload.["transient"].Values.Count > 0 then
client.Cluster.PutSettings<DynamicResponse>(PostData.Serializable(payload))
else
settings

let deleteAllILMPolicies () =
let preserved = [
Expand Down Expand Up @@ -248,7 +258,6 @@ let DefaultSetup : Operation list = [Actions("Setup", fun (client, suite) ->

yield! stopTransforms()


if suite = Platinum then
let data = PostData.String @"{""password"":""x-pack-test-password"", ""roles"":[""superuser""]}"
yield client.Security.PutUser<DynamicResponse>("x_pack_rest_user", data)
Expand Down