diff --git a/src/Nest/XPack/Transform/TransformSettings.cs b/src/Nest/XPack/Transform/TransformSettings.cs index e9c09dcf18d..fcde6501e60 100644 --- a/src/Nest/XPack/Transform/TransformSettings.cs +++ b/src/Nest/XPack/Transform/TransformSettings.cs @@ -21,6 +21,9 @@ public interface ITransformSettings [DataMember(Name = "max_page_search_size")] public int? MaxPageSearchSize { get; set; } + + [DataMember(Name = "dates_as_epoch_millis")] + public bool? DatesAsEpochMilliseconds { get; set; } } /// @@ -31,12 +34,16 @@ public class TransformSettings : ITransformSettings /// public int? MaxPageSearchSize { get; set; } + + /// + public bool? DatesAsEpochMilliseconds { get; set; } } public class TransformSettingsDescriptor : DescriptorBase, ITransformSettings { float? ITransformSettings.DocsPerSecond { get; set; } int? ITransformSettings.MaxPageSearchSize { get; set; } + bool? ITransformSettings.DatesAsEpochMilliseconds { get; set; } /// public TransformSettingsDescriptor DocsPerSecond(float? docsPerSecond) => @@ -45,5 +52,9 @@ public TransformSettingsDescriptor DocsPerSecond(float? docsPerSecond) => /// public TransformSettingsDescriptor MaxPageSearchSize(int? maxPageSearchSize) => Assign(maxPageSearchSize, (a, v) => a.MaxPageSearchSize = v); + + /// + public TransformSettingsDescriptor DatesAsEpochMilliseconds(bool? datesAsEpochMillis = true) => + Assign(datesAsEpochMillis, (a, v) => a.DatesAsEpochMilliseconds = v); } } diff --git a/tests/Tests.Domain/Project.cs b/tests/Tests.Domain/Project.cs index f007221fabe..646c80964cc 100644 --- a/tests/Tests.Domain/Project.cs +++ b/tests/Tests.Domain/Project.cs @@ -196,8 +196,10 @@ public class Metadata public class ProjectTransform { public double? AverageCommits { get; set; } + + public long WeekStartedOnMillis { get; set; } - public long WeekStartedOn { get; set; } + public DateTime WeekStartedOnDate { get; set; } public long SumIntoMaster { get; set; } } diff --git a/tests/Tests/XPack/Transform/TransformApiTests.cs b/tests/Tests/XPack/Transform/TransformApiTests.cs index 0d3f9ef89bb..5a43e7a3636 100644 --- a/tests/Tests/XPack/Transform/TransformApiTests.cs +++ b/tests/Tests/XPack/Transform/TransformApiTests.cs @@ -1,9 +1,11 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Elastic.Elasticsearch.Xunit.XunitPlumbing; using FluentAssertions; using Nest; +using Tests.Core.Client; using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Domain; @@ -171,7 +173,7 @@ public TransformApiTests(WritableCluster cluster, EndpointUsage usage) : base(ne }, GroupBy = new Dictionary { - { "weekStartedOn", new DateHistogramGroupSource() + { TestClient.Configuration.InRange("<7.11.0") ? "weekStartedOnMillis" : "weekStartedOnDate", new DateHistogramGroupSource { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week @@ -207,7 +209,7 @@ public TransformApiTests(WritableCluster cluster, EndpointUsage usage) : base(ne ) ) .GroupBy(g => g - .DateHistogram("weekStartedOn", dh => dh + .DateHistogram(TestClient.Configuration.InRange("<7.11.0") ? "weekStartedOnMillis" : "weekStartedOnDate", dh => dh .Field(f => f.StartedOn) .CalendarInterval(DateInterval.Week) ) @@ -317,6 +319,11 @@ [I] public async Task PreviewTransformResponse() => await Assert await Assert(UpdateTransformStep, (v, r) => diff --git a/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs index a00171daa28..63735845d8b 100644 --- a/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs +++ b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs @@ -7,6 +7,8 @@ using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Elastic.Transport; using Nest; +using Tests.Core.Client; +using Tests.Core.Extensions; using Tests.Core.ManagedElasticsearch.Clusters; using Tests.Domain; using Tests.Framework.EndpointTests; @@ -16,87 +18,232 @@ namespace Tests.XPack.Transform { [SkipVersion("<7.8.0", "Settings introduced in 7.8.0")] - public class TransformApiWithSettingsTests : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest> + public class TransformApiWithSettingsTests + : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest> { public TransformApiWithSettingsTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } protected override LazyResponses ClientUsage() => Calls( - (client, f) => client.Transform.Preview(f), - (client, f) => client.Transform.PreviewAsync(f), - (client, r) => client.Transform.Preview(r), - (client, r) => client.Transform.PreviewAsync(r) - ); + (client, f) => client.Transform.Preview(f), + (client, f) => client.Transform.PreviewAsync(f), + (client, r) => client.Transform.Preview(r), + (client, r) => client.Transform.PreviewAsync(r)); protected override HttpMethod HttpMethod => HttpMethod.POST; - protected override string UrlPath => $"_transform/_preview"; + protected override string UrlPath => "_transform/_preview"; protected override bool ExpectIsValid => true; protected override int ExpectStatusCode => 200; protected override bool SupportsDeserialization => false; - protected override object ExpectJson => new + + protected override object ExpectJson { - description = CallIsolatedValue, - frequency = "1s", - source = new { index = new[] { "project" }, query = new { match_all = new { } } }, - dest = new { index = $"transform-{CallIsolatedValue}" }, - pivot = new + get { - aggregations = new + var settings = TestClient.Configuration.InRange("<7.11.0") + ? (dynamic)new { docs_per_second = 200.0, max_page_search_size = 200 } + : new { docs_per_second = 200.0, max_page_search_size = 200, dates_as_epoch_millis = true }; + + return new { - averageCommits = new + description = CallIsolatedValue, + frequency = "1s", + source = new { index = new[] { "project" }, query = new { match_all = new { } } }, + dest = new { index = $"transform-{CallIsolatedValue}" }, + pivot = new { - avg = new + aggregations = new { - field = "numberOfCommits" + averageCommits = new + { + avg = new + { + field = "numberOfCommits" + } + }, + sumIntoMaster = new + { + scripted_metric = new + { + combine_script = new + { + source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum" + }, + init_script = new + { + source = "state.masterCommits = []" + }, + map_script = new + { + source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)" + }, + reduce_script = new + { + source = "long sum = 0; for (s in states) { sum += s } return sum" + } + } + } + }, + group_by = new + { + weekStartedOnMillis = new + { + date_histogram = new + { + calendar_interval = "week", + field = "startedOn" + } + } } }, - sumIntoMaster = new + sync = new { - scripted_metric = new + time = new { - combine_script = new - { - source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum" - }, - init_script = new - { - source = "state.masterCommits = []" - }, - map_script = new + field = "lastActivity" + } + }, + settings + }; + } + } + + protected override PreviewTransformRequest Initializer + { + get + { + var settings = new TransformSettings { MaxPageSearchSize = 200, DocsPerSecond = 200 }; + + if (TestClient.Configuration.InRange(">=7.11.0")) + settings.DatesAsEpochMilliseconds = true; + + return new PreviewTransformRequest + { + Description = CallIsolatedValue, + Frequency = "1s", + Source = new TransformSource { Index = Index(), Query = new MatchAllQuery() }, + Destination = new TransformDestination { Index = $"transform-{CallIsolatedValue}" }, + Pivot = new TransformPivot + { + Aggregations = + new AverageAggregation("averageCommits", Field(f => f.NumberOfCommits)) && + new ScriptedMetricAggregation("sumIntoMaster") { - source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)" + InitScript = new InlineScript("state.masterCommits = []"), + MapScript = new InlineScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)"), + CombineScript = new InlineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum"), + ReduceScript = new InlineScript("long sum = 0; for (s in states) { sum += s } return sum") }, - reduce_script = new + GroupBy = new Dictionary + { { - source = "long sum = 0; for (s in states) { sum += s } return sum" + "weekStartedOnMillis", + new DateHistogramGroupSource { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week } } } - } - }, - group_by = new + }, + Sync = new TransformSyncContainer(new TransformTimeSync { Field = Field(f => f.LastActivity) }), + Settings = settings + }; + } + } + + protected override Func, IPreviewTransformRequest> Fluent => f => f + .Description(CallIsolatedValue) + .Frequency(new Time(1, TimeUnit.Second)) + .Source(s => s + .Index() + .Query(q => q.MatchAll()) + ) + .Destination(de => de + .Index($"transform-{CallIsolatedValue}") + ) + .Pivot(p => p + .Aggregations(a => a + .Average("averageCommits", avg => avg + .Field(fld => fld.NumberOfCommits) + ) + .ScriptedMetric("sumIntoMaster", sm => sm + .InitScript("state.masterCommits = []") + .MapScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)") + .CombineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum") + .ReduceScript("long sum = 0; for (s in states) { sum += s } return sum") + ) + ) + .GroupBy(g => g + .DateHistogram("weekStartedOnMillis", dh => dh + .Field(fld => fld.StartedOn) + .CalendarInterval(DateInterval.Week) + ) + ) + ) + .Sync(sy => sy + .Time(t => t + .Field(fld => fld.LastActivity) + ) + ).Settings(s => + { + var descriptor = s + .MaxPageSearchSize(200) + .DocsPerSecond(200); + + if (TestClient.Configuration.InRange(">=7.11.0")) + descriptor.DatesAsEpochMilliseconds(true); + + return descriptor; + }); + } + + [SkipVersion("<7.11.0", "Settings introduced in 7.11.0")] + public class TransformApiWithSettingsAndNoDatesAsMillisTests + : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest> + { + // From 7.11, by default, ISO dates are used for transforms, rather than epoch millis. + // This test verifies the default behaviour, without configuring the setting, results in a date + + public TransformApiWithSettingsAndNoDatesAsMillisTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override LazyResponses ClientUsage() => + Calls( + (client, f) => client.Transform.Preview(f), + (client, f) => client.Transform.PreviewAsync(f), + (client, r) => client.Transform.Preview(r), + (client, r) => client.Transform.PreviewAsync(r)); + + protected override HttpMethod HttpMethod => HttpMethod.POST; + protected override string UrlPath => "_transform/_preview"; + protected override bool ExpectIsValid => true; + protected override int ExpectStatusCode => 200; + protected override bool SupportsDeserialization => false; + + protected override object ExpectJson => + new + { + description = CallIsolatedValue, + frequency = "1s", + source = new { index = new[] { "project" }, query = new { match_all = new { } } }, + dest = new { index = $"transform-{CallIsolatedValue}" }, + pivot = new { - weekStartedOn = new + aggregations = new { - date_histogram = new + averageCommits = new { avg = new { field = "numberOfCommits" } }, + sumIntoMaster = new { - calendar_interval = "week", - field = "startedOn" + scripted_metric = new + { + combine_script = + new { source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum" }, + init_script = new { source = "state.masterCommits = []" }, + map_script = new { source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)" }, + reduce_script = new { source = "long sum = 0; for (s in states) { sum += s } return sum" } + } } - } - } - }, - sync = new - { - time = new - { - field = "lastActivity" - } - }, - settings = new - { - docs_per_second = 200.0, - max_page_search_size = 200 - } - }; + }, + group_by = new { weekStartedOnDate = new { date_histogram = new { calendar_interval = "week", field = "startedOn" } } } + }, + sync = new { time = new { field = "lastActivity" } }, + settings = new { docs_per_second = 200.0, max_page_search_size = 200 } + }; protected override PreviewTransformRequest Initializer => new PreviewTransformRequest { @@ -118,8 +265,8 @@ protected override LazyResponses ClientUsage() => Calls( GroupBy = new Dictionary { { - "weekStartedOn", - new DateHistogramGroupSource() { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week } + "weekStartedOnDate", + new DateHistogramGroupSource { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week } } } }, @@ -140,7 +287,7 @@ protected override LazyResponses ClientUsage() => Calls( .Pivot(p => p .Aggregations(a => a .Average("averageCommits", avg => avg - .Field(af => af.NumberOfCommits) + .Field(fld => fld.NumberOfCommits) ) .ScriptedMetric("sumIntoMaster", sm => sm .InitScript("state.masterCommits = []") @@ -150,20 +297,18 @@ protected override LazyResponses ClientUsage() => Calls( ) ) .GroupBy(g => g - .DateHistogram("weekStartedOn", dh => dh - .Field(df => df.StartedOn) + .DateHistogram("weekStartedOnDate", dh => dh + .Field(fld => fld.StartedOn) .CalendarInterval(DateInterval.Week) ) ) ) .Sync(sy => sy .Time(t => t - .Field(tf => tf.LastActivity) + .Field(fld => fld.LastActivity) ) - ) - .Settings(s => s + ).Settings(s => s .MaxPageSearchSize(200) - .DocsPerSecond(200) - ); + .DocsPerSecond(200)); } }