diff --git a/src/Nest/XPack/Transform/TransformSettings.cs b/src/Nest/XPack/Transform/TransformSettings.cs
index e9c09dcf18d..fcde6501e60 100644
--- a/src/Nest/XPack/Transform/TransformSettings.cs
+++ b/src/Nest/XPack/Transform/TransformSettings.cs
@@ -21,6 +21,9 @@ public interface ITransformSettings
[DataMember(Name = "max_page_search_size")]
public int? MaxPageSearchSize { get; set; }
+
+ [DataMember(Name = "dates_as_epoch_millis")]
+ public bool? DatesAsEpochMilliseconds { get; set; }
}
///
@@ -31,12 +34,16 @@ public class TransformSettings : ITransformSettings
///
public int? MaxPageSearchSize { get; set; }
+
+ ///
+ public bool? DatesAsEpochMilliseconds { get; set; }
}
public class TransformSettingsDescriptor : DescriptorBase, ITransformSettings
{
float? ITransformSettings.DocsPerSecond { get; set; }
int? ITransformSettings.MaxPageSearchSize { get; set; }
+ bool? ITransformSettings.DatesAsEpochMilliseconds { get; set; }
///
public TransformSettingsDescriptor DocsPerSecond(float? docsPerSecond) =>
@@ -45,5 +52,9 @@ public TransformSettingsDescriptor DocsPerSecond(float? docsPerSecond) =>
///
public TransformSettingsDescriptor MaxPageSearchSize(int? maxPageSearchSize) =>
Assign(maxPageSearchSize, (a, v) => a.MaxPageSearchSize = v);
+
+ ///
+ public TransformSettingsDescriptor DatesAsEpochMilliseconds(bool? datesAsEpochMillis = true) =>
+ Assign(datesAsEpochMillis, (a, v) => a.DatesAsEpochMilliseconds = v);
}
}
diff --git a/tests/Tests.Domain/Project.cs b/tests/Tests.Domain/Project.cs
index f007221fabe..646c80964cc 100644
--- a/tests/Tests.Domain/Project.cs
+++ b/tests/Tests.Domain/Project.cs
@@ -196,8 +196,10 @@ public class Metadata
public class ProjectTransform
{
public double? AverageCommits { get; set; }
+
+ public long WeekStartedOnMillis { get; set; }
- public long WeekStartedOn { get; set; }
+ public DateTime WeekStartedOnDate { get; set; }
public long SumIntoMaster { get; set; }
}
diff --git a/tests/Tests/XPack/Transform/TransformApiTests.cs b/tests/Tests/XPack/Transform/TransformApiTests.cs
index 0d3f9ef89bb..5a43e7a3636 100644
--- a/tests/Tests/XPack/Transform/TransformApiTests.cs
+++ b/tests/Tests/XPack/Transform/TransformApiTests.cs
@@ -1,9 +1,11 @@
+using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Elastic.Elasticsearch.Xunit.XunitPlumbing;
using FluentAssertions;
using Nest;
+using Tests.Core.Client;
using Tests.Core.Extensions;
using Tests.Core.ManagedElasticsearch.Clusters;
using Tests.Domain;
@@ -171,7 +173,7 @@ public TransformApiTests(WritableCluster cluster, EndpointUsage usage) : base(ne
},
GroupBy = new Dictionary
{
- { "weekStartedOn", new DateHistogramGroupSource()
+ { TestClient.Configuration.InRange("<7.11.0") ? "weekStartedOnMillis" : "weekStartedOnDate", new DateHistogramGroupSource
{
Field = Field(f => f.StartedOn),
CalendarInterval = DateInterval.Week
@@ -207,7 +209,7 @@ public TransformApiTests(WritableCluster cluster, EndpointUsage usage) : base(ne
)
)
.GroupBy(g => g
- .DateHistogram("weekStartedOn", dh => dh
+ .DateHistogram(TestClient.Configuration.InRange("<7.11.0") ? "weekStartedOnMillis" : "weekStartedOnDate", dh => dh
.Field(f => f.StartedOn)
.CalendarInterval(DateInterval.Week)
)
@@ -317,6 +319,11 @@ [I] public async Task PreviewTransformResponse() => await Assert await Assert(UpdateTransformStep, (v, r) =>
diff --git a/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs
index a00171daa28..63735845d8b 100644
--- a/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs
+++ b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs
@@ -7,6 +7,8 @@
using Elastic.Elasticsearch.Xunit.XunitPlumbing;
using Elastic.Transport;
using Nest;
+using Tests.Core.Client;
+using Tests.Core.Extensions;
using Tests.Core.ManagedElasticsearch.Clusters;
using Tests.Domain;
using Tests.Framework.EndpointTests;
@@ -16,87 +18,232 @@
namespace Tests.XPack.Transform
{
[SkipVersion("<7.8.0", "Settings introduced in 7.8.0")]
- public class TransformApiWithSettingsTests : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest>
+ public class TransformApiWithSettingsTests
+ : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest>
{
public TransformApiWithSettingsTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
protected override LazyResponses ClientUsage() => Calls(
- (client, f) => client.Transform.Preview(f),
- (client, f) => client.Transform.PreviewAsync(f),
- (client, r) => client.Transform.Preview(r),
- (client, r) => client.Transform.PreviewAsync(r)
- );
+ (client, f) => client.Transform.Preview(f),
+ (client, f) => client.Transform.PreviewAsync(f),
+ (client, r) => client.Transform.Preview(r),
+ (client, r) => client.Transform.PreviewAsync(r));
protected override HttpMethod HttpMethod => HttpMethod.POST;
- protected override string UrlPath => $"_transform/_preview";
+ protected override string UrlPath => "_transform/_preview";
protected override bool ExpectIsValid => true;
protected override int ExpectStatusCode => 200;
protected override bool SupportsDeserialization => false;
- protected override object ExpectJson => new
+
+ protected override object ExpectJson
{
- description = CallIsolatedValue,
- frequency = "1s",
- source = new { index = new[] { "project" }, query = new { match_all = new { } } },
- dest = new { index = $"transform-{CallIsolatedValue}" },
- pivot = new
+ get
{
- aggregations = new
+ var settings = TestClient.Configuration.InRange("<7.11.0")
+ ? (dynamic)new { docs_per_second = 200.0, max_page_search_size = 200 }
+ : new { docs_per_second = 200.0, max_page_search_size = 200, dates_as_epoch_millis = true };
+
+ return new
{
- averageCommits = new
+ description = CallIsolatedValue,
+ frequency = "1s",
+ source = new { index = new[] { "project" }, query = new { match_all = new { } } },
+ dest = new { index = $"transform-{CallIsolatedValue}" },
+ pivot = new
{
- avg = new
+ aggregations = new
{
- field = "numberOfCommits"
+ averageCommits = new
+ {
+ avg = new
+ {
+ field = "numberOfCommits"
+ }
+ },
+ sumIntoMaster = new
+ {
+ scripted_metric = new
+ {
+ combine_script = new
+ {
+ source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum"
+ },
+ init_script = new
+ {
+ source = "state.masterCommits = []"
+ },
+ map_script = new
+ {
+ source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)"
+ },
+ reduce_script = new
+ {
+ source = "long sum = 0; for (s in states) { sum += s } return sum"
+ }
+ }
+ }
+ },
+ group_by = new
+ {
+ weekStartedOnMillis = new
+ {
+ date_histogram = new
+ {
+ calendar_interval = "week",
+ field = "startedOn"
+ }
+ }
}
},
- sumIntoMaster = new
+ sync = new
{
- scripted_metric = new
+ time = new
{
- combine_script = new
- {
- source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum"
- },
- init_script = new
- {
- source = "state.masterCommits = []"
- },
- map_script = new
+ field = "lastActivity"
+ }
+ },
+ settings
+ };
+ }
+ }
+
+ protected override PreviewTransformRequest Initializer
+ {
+ get
+ {
+ var settings = new TransformSettings { MaxPageSearchSize = 200, DocsPerSecond = 200 };
+
+ if (TestClient.Configuration.InRange(">=7.11.0"))
+ settings.DatesAsEpochMilliseconds = true;
+
+ return new PreviewTransformRequest
+ {
+ Description = CallIsolatedValue,
+ Frequency = "1s",
+ Source = new TransformSource { Index = Index(), Query = new MatchAllQuery() },
+ Destination = new TransformDestination { Index = $"transform-{CallIsolatedValue}" },
+ Pivot = new TransformPivot
+ {
+ Aggregations =
+ new AverageAggregation("averageCommits", Field(f => f.NumberOfCommits)) &&
+ new ScriptedMetricAggregation("sumIntoMaster")
{
- source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)"
+ InitScript = new InlineScript("state.masterCommits = []"),
+ MapScript = new InlineScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)"),
+ CombineScript = new InlineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum"),
+ ReduceScript = new InlineScript("long sum = 0; for (s in states) { sum += s } return sum")
},
- reduce_script = new
+ GroupBy = new Dictionary
+ {
{
- source = "long sum = 0; for (s in states) { sum += s } return sum"
+ "weekStartedOnMillis",
+ new DateHistogramGroupSource { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week }
}
}
- }
- },
- group_by = new
+ },
+ Sync = new TransformSyncContainer(new TransformTimeSync { Field = Field(f => f.LastActivity) }),
+ Settings = settings
+ };
+ }
+ }
+
+ protected override Func, IPreviewTransformRequest> Fluent => f => f
+ .Description(CallIsolatedValue)
+ .Frequency(new Time(1, TimeUnit.Second))
+ .Source(s => s
+ .Index()
+ .Query(q => q.MatchAll())
+ )
+ .Destination(de => de
+ .Index($"transform-{CallIsolatedValue}")
+ )
+ .Pivot(p => p
+ .Aggregations(a => a
+ .Average("averageCommits", avg => avg
+ .Field(fld => fld.NumberOfCommits)
+ )
+ .ScriptedMetric("sumIntoMaster", sm => sm
+ .InitScript("state.masterCommits = []")
+ .MapScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)")
+ .CombineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum")
+ .ReduceScript("long sum = 0; for (s in states) { sum += s } return sum")
+ )
+ )
+ .GroupBy(g => g
+ .DateHistogram("weekStartedOnMillis", dh => dh
+ .Field(fld => fld.StartedOn)
+ .CalendarInterval(DateInterval.Week)
+ )
+ )
+ )
+ .Sync(sy => sy
+ .Time(t => t
+ .Field(fld => fld.LastActivity)
+ )
+ ).Settings(s =>
+ {
+ var descriptor = s
+ .MaxPageSearchSize(200)
+ .DocsPerSecond(200);
+
+ if (TestClient.Configuration.InRange(">=7.11.0"))
+ descriptor.DatesAsEpochMilliseconds(true);
+
+ return descriptor;
+ });
+ }
+
+ [SkipVersion("<7.11.0", "Settings introduced in 7.11.0")]
+ public class TransformApiWithSettingsAndNoDatesAsMillisTests
+ : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest>
+ {
+ // From 7.11, by default, ISO dates are used for transforms, rather than epoch millis.
+ // This test verifies the default behaviour, without configuring the setting, results in a date
+
+ public TransformApiWithSettingsAndNoDatesAsMillisTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+ protected override LazyResponses ClientUsage() =>
+ Calls(
+ (client, f) => client.Transform.Preview(f),
+ (client, f) => client.Transform.PreviewAsync(f),
+ (client, r) => client.Transform.Preview(r),
+ (client, r) => client.Transform.PreviewAsync(r));
+
+ protected override HttpMethod HttpMethod => HttpMethod.POST;
+ protected override string UrlPath => "_transform/_preview";
+ protected override bool ExpectIsValid => true;
+ protected override int ExpectStatusCode => 200;
+ protected override bool SupportsDeserialization => false;
+
+ protected override object ExpectJson =>
+ new
+ {
+ description = CallIsolatedValue,
+ frequency = "1s",
+ source = new { index = new[] { "project" }, query = new { match_all = new { } } },
+ dest = new { index = $"transform-{CallIsolatedValue}" },
+ pivot = new
{
- weekStartedOn = new
+ aggregations = new
{
- date_histogram = new
+ averageCommits = new { avg = new { field = "numberOfCommits" } },
+ sumIntoMaster = new
{
- calendar_interval = "week",
- field = "startedOn"
+ scripted_metric = new
+ {
+ combine_script =
+ new { source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum" },
+ init_script = new { source = "state.masterCommits = []" },
+ map_script = new { source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)" },
+ reduce_script = new { source = "long sum = 0; for (s in states) { sum += s } return sum" }
+ }
}
- }
- }
- },
- sync = new
- {
- time = new
- {
- field = "lastActivity"
- }
- },
- settings = new
- {
- docs_per_second = 200.0,
- max_page_search_size = 200
- }
- };
+ },
+ group_by = new { weekStartedOnDate = new { date_histogram = new { calendar_interval = "week", field = "startedOn" } } }
+ },
+ sync = new { time = new { field = "lastActivity" } },
+ settings = new { docs_per_second = 200.0, max_page_search_size = 200 }
+ };
protected override PreviewTransformRequest Initializer => new PreviewTransformRequest
{
@@ -118,8 +265,8 @@ protected override LazyResponses ClientUsage() => Calls(
GroupBy = new Dictionary
{
{
- "weekStartedOn",
- new DateHistogramGroupSource() { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week }
+ "weekStartedOnDate",
+ new DateHistogramGroupSource { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week }
}
}
},
@@ -140,7 +287,7 @@ protected override LazyResponses ClientUsage() => Calls(
.Pivot(p => p
.Aggregations(a => a
.Average("averageCommits", avg => avg
- .Field(af => af.NumberOfCommits)
+ .Field(fld => fld.NumberOfCommits)
)
.ScriptedMetric("sumIntoMaster", sm => sm
.InitScript("state.masterCommits = []")
@@ -150,20 +297,18 @@ protected override LazyResponses ClientUsage() => Calls(
)
)
.GroupBy(g => g
- .DateHistogram("weekStartedOn", dh => dh
- .Field(df => df.StartedOn)
+ .DateHistogram("weekStartedOnDate", dh => dh
+ .Field(fld => fld.StartedOn)
.CalendarInterval(DateInterval.Week)
)
)
)
.Sync(sy => sy
.Time(t => t
- .Field(tf => tf.LastActivity)
+ .Field(fld => fld.LastActivity)
)
- )
- .Settings(s => s
+ ).Settings(s => s
.MaxPageSearchSize(200)
- .DocsPerSecond(200)
- );
+ .DocsPerSecond(200));
}
}