The task cancel API test was a bit flakey,

src/Tests/Cat/CatRepositories/CatRepositoriesApiTests.cs

@@ -17,10 +17,10 @@ public class CatRepositoriesApiTests : ApiIntegrationTestBase<ICatResponse<CatRe
 
 		public CatRepositoriesApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			if (!TestClient.Configuration.RunIntegrationTests) return;
-			var repositoryLocation = Path.Combine(this._cluster.Node.RepositoryPath, RandomString());
+			var repositoryLocation = Path.Combine(this.Cluster.Node.RepositoryPath, RandomString());
 
 			var create = this.Client.CreateRepository(RepositoryName, cr => cr
 				.FileSystem(fs => fs

src/Tests/Cat/CatSnapshots/CatSnapshotsApiTests.cs

@@ -19,10 +19,10 @@ public class CatSnapshotsApiTests : ApiIntegrationTestBase<ICatResponse<CatSnaps
 
 		public CatSnapshotsApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			if (!TestClient.Configuration.RunIntegrationTests) return;
-			var repositoryLocation = Path.Combine(this._cluster.Node.RepositoryPath, RandomString());
+			var repositoryLocation = Path.Combine(this.Cluster.Node.RepositoryPath, RandomString());
 
 			var create = this.Client.CreateRepository(RepositoryName, cr => cr
 				.FileSystem(fs => fs

src/Tests/Cluster/TaskManagement/TasksCancel/TasksCancelApiTests.cs

@@ -10,29 +10,31 @@
 
 namespace Tests.Cluster.TaskManagement.TasksCancel
 {
-	// TODO Unit tests will fail in mixed mode because SetupTaskIds isn't setup
 	[Collection(IntegrationContext.OwnIndex)]
 	public class TasksCancelApiTests : ApiIntegrationTestBase<ITasksCancelResponse, ITasksCancelRequest, TasksCancelDescriptor, TasksCancelRequest>
 	{
-		public class Test
+		private TaskId TaskId => this.RanIntegrationSetup ? this.ExtendedValue<TaskId>("taskId") : "foo:1";
+
+		private class Test
 		{
 			public long Id { get; set; }
 			public string Flag { get; set; }
 		}
 
 		public TasksCancelApiTests(OwnIndexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected IDictionary<string, TaskId> SetupTaskIds { get; } = new Dictionary<string, TaskId>();
-
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var index in values.Values)
 			{
 				client.IndexMany(Enumerable.Range(0, 10000).Select(i => new Test { Id = i + 1, Flag = "bar" }), index);
 				client.Refresh(index);
 			}
-			foreach (var index in values.Values)
+			foreach (var view in values.Views)
 			{
+				values.CurrentView = view;
+				var index = values.Value;
+
 				var reindex = client.ReindexOnServer(r => r
 					.Source(s => s.Index(index))
 					.Destination(s => s.Index($"{index}-clone"))
@@ -42,7 +44,7 @@ protected override void BeforeAllCalls(IElasticClient client, IDictionary<Client
 				var taskId = reindex.Task;
 				var taskInfo = client.TasksList(new TasksListRequest(taskId));
 				taskInfo.IsValid.Should().BeTrue();
-				this.SetupTaskIds[index] = taskId;
+				values.ExtendedValue("taskId", taskId);
 			}
 		}
 		protected override LazyResponses ClientUsage() => Calls(
@@ -58,20 +60,18 @@ protected override LazyResponses ClientUsage() => Calls(
 		protected override string UrlPath => $"/_tasks/{Uri.EscapeDataString(this.TaskId.ToString())}/_cancel";
 		protected override bool SupportsDeserialization => false;
 
-		private TaskId TaskId => TestClient.Configuration.RunIntegrationTests ? this.SetupTaskIds[CallIsolatedValue] : "foo:1";
 
 		protected override Func<TasksCancelDescriptor, ITasksCancelRequest> Fluent => d => d
 			.TaskId(this.TaskId);
 
 		protected override TasksCancelRequest Initializer => new TasksCancelRequest(this.TaskId);
 
-		// TODO this test is flaky, sometimes SetupTaskIds is empty
 		protected override void ExpectResponse(ITasksCancelResponse response)
 		{
 			response.NodeFailures.Should().BeNullOrEmpty();
 			response.Nodes.Should().NotBeEmpty();
 			var tasks = response.Nodes.First().Value.Tasks;
-			tasks.Should().NotBeEmpty().And.ContainKey(this.SetupTaskIds[CallIsolatedValue]);
+			tasks.Should().NotBeEmpty().And.ContainKey(this.TaskId);
 		}
 	}
 }

src/Tests/Document/Multiple/DeleteByQuery/DeleteByQueryApiTests.cs

@@ -17,7 +17,7 @@ public class DeleteByQueryApiTests : ApiIntegrationTestBase<IDeleteByQueryRespon
 	{
 		public DeleteByQueryApiTests(OwnIndexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var index in values.Values)
 			{
@@ -44,7 +44,7 @@ protected override LazyResponses ClientUsage() => Calls(
 
 		protected override bool SupportsDeserialization => false;
 
-		protected override object ExpectJson { get; } = new 
+		protected override object ExpectJson { get; } = new
 		{
 			query = new
 			{
@@ -66,7 +66,7 @@ protected override LazyResponses ClientUsage() => Calls(
 					.Values(Project.Projects.First().Name, "x")
 				)
 			);
-			
+
 		protected override DeleteByQueryRequest Initializer => new DeleteByQueryRequest(this.Indices)
 		{
 			IgnoreUnavailable = true,

src/Tests/Document/Multiple/ReindexOnServer/ReindexOnServerApiTests.cs

@@ -23,7 +23,7 @@ public class Test
 
 		public ReindexOnServerApiTests(OwnIndexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var index in values.Values)
 			{

src/Tests/Document/Multiple/UpdateByQuery/UpdateByQueryApiTests.cs

@@ -23,7 +23,7 @@ public class Test
 
 		public UpdateByQueryApiTests(OwnIndexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var index in values.Values)
 			{
@@ -131,7 +131,7 @@ public class UpdateByQueryWithFailuresApiTests : UpdateByQueryApiTests
 	{
 		public UpdateByQueryWithFailuresApiTests(OwnIndexCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var index in values.Values)
 			{

src/Tests/Document/Single/Delete/DeleteApiTests.cs

@@ -14,7 +14,7 @@ public class DeleteApiTests : ApiIntegrationTestBase<IDeleteResponse, IDeleteReq
 	{
 		public DeleteApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var id in values.Values)
 				this.Client.Index(Project.Instance, i=>i.Id(id));

src/Tests/Document/Single/Exists/DocumentExistsApiTests.cs

@@ -14,7 +14,7 @@ public class DocumentExistsApiTests : ApiIntegrationTestBase<IExistsResponse, ID
 	{
 		public DocumentExistsApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var id in values.Values)
 				this.Client.Index(Project.Instance, i=>i.Id(id));

src/Tests/Document/Single/Update/UpdateApiTests.cs

@@ -14,7 +14,7 @@ public class UpdateApiTests : ApiIntegrationTestBase<IUpdateResponse<Project>, I
 	{
 		public UpdateApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var id in values.Values)
 				this.Client.Index(Project.Instance, i=>i.Id(id));

src/Tests/Document/Single/Update/UpdateWithSourceApiTests.cs

@@ -18,7 +18,7 @@ public class UpdateWithSourceApiTests : ApiIntegrationTestBase<IUpdateResponse<P
 	{
 		public UpdateWithSourceApiTests(IndexingCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
-		protected override void BeforeAllCalls(IElasticClient client, IDictionary<ClientMethod, string> values)
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
 			foreach (var id in values.Values)
 				this.Client.Index(Project.Instance, i=>i.Id(id));

src/Tests/Framework/ApiIntegrationTestBase.cs

@@ -8,7 +8,7 @@
 
 namespace Tests.Framework
 {
-	public abstract class ApiIntegrationTestBase<TResponse, TInterface, TDescriptor, TInitializer> 
+	public abstract class ApiIntegrationTestBase<TResponse, TInterface, TDescriptor, TInitializer>
 		: ApiTestBase<TResponse, TInterface, TDescriptor, TInitializer>
 		where TResponse : class, IResponse
 		where TDescriptor : class, TInterface
@@ -29,8 +29,7 @@ [I] protected async Task HandlesStatusCode() =>
 		[I] protected async Task ReturnsExpectedIsValid() =>
 			await this.AssertOnAllResponses(r=>r.IsValid.Should().Be(this.ExpectIsValid));
 
-		[I] protected async Task ReturnsExpectedResponse() =>
-			await this.AssertOnAllResponses(r => ExpectResponse(r));
+		[I] protected async Task ReturnsExpectedResponse() => await this.AssertOnAllResponses(ExpectResponse);
 
 		protected override Task AssertOnAllResponses(Action<TResponse> assert)
 		{

src/Tests/Framework/ApiTestBase.cs

@@ -16,87 +16,110 @@ public abstract class ApiTestBase<TResponse, TInterface, TDescriptor, TInitializ
 		where TInitializer : class, TInterface
 		where TInterface : class
 	{
-		protected readonly IIntegrationCluster _cluster;
+		private readonly EndpointUsage _usage;
 		private readonly LazyResponses _responses;
+		private readonly int _port;
+		private readonly CallUniqueValues _uniqueValues;
 
 		protected static string RandomString() => Guid.NewGuid().ToString("N").Substring(0, 8);
-		protected virtual ConnectionSettings GetConnectionSettings(ConnectionSettings settings) => settings;
-		protected virtual IElasticClient Client => this._cluster.Client(GetConnectionSettings);
+		protected bool RanIntegrationSetup => this._usage?.CalledSetup ?? false;
+
+		protected IIntegrationCluster Cluster { get; }
 
-		protected IDictionary<Integration.ClientMethod, string> UniqueValues { get; }
-		protected string CallIsolatedValue { get; private set; }
+		protected string CallIsolatedValue => _uniqueValues.Value;
+		protected T ExtendedValue<T>(string key) where T : class => this._uniqueValues.ExtendedValue<T>(key);
 
-		protected virtual void BeforeAllCalls(IElasticClient client, IDictionary<Integration.ClientMethod, string> values) { }
+		protected virtual void IntegrationSetup(IElasticClient client, CallUniqueValues values) { }
 		protected virtual void OnBeforeCall(IElasticClient client) { }
 		protected virtual void OnAfterCall(IElasticClient client) { }
 
-		protected virtual int Port { get; set; } = 9200;
+		protected IElasticClient Client => this.Cluster.Client(GetConnectionSettings);
+		protected virtual ConnectionSettings GetConnectionSettings(ConnectionSettings settings) => settings;
 
-		protected virtual TInitializer Initializer { get; }
+		protected virtual TDescriptor NewDescriptor() => Activator.CreateInstance<TDescriptor>();
 		protected virtual Func<TDescriptor, TInterface> Fluent { get; }
+		protected virtual TInitializer Initializer { get; }
 
-		protected virtual TDescriptor ClientDoesThisInternally(TDescriptor d) => d;
 		protected abstract LazyResponses ClientUsage();
 
-		protected virtual TDescriptor NewDescriptor() => Activator.CreateInstance<TDescriptor>();
-
 		protected abstract string UrlPath { get; }
 		protected abstract HttpMethod HttpMethod { get; }
 
 		protected ApiTestBase(IIntegrationCluster cluster, EndpointUsage usage)
 		{
-			this._cluster = cluster;
+			this._usage = usage;
+			this.Cluster = cluster;
+
 			this._responses = usage.CallOnce(this.ClientUsage);
-			this.Port = cluster.Node.Port;
-			this.UniqueValues = usage.CallUniqueValues;
-			this.CallIsolatedValue = UniqueValues[Integration.ClientMethod.Fluent];
+			this._port = cluster.Node.Port;
+			this._uniqueValues = usage.CallUniqueValues;
 			this.SetupSerialization();
 		}
 
-		protected virtual LazyResponses Calls(
+		[U] protected async Task HitsTheCorrectUrl() =>
+			await this.AssertOnAllResponses(r => this.AssertUrl(r.ApiCall.Uri));
+
+		[U] protected async Task UsesCorrectHttpMethod() =>
+			await this.AssertOnAllResponses(r => r.CallDetails.HttpMethod.Should().Be(this.HttpMethod));
+
+		[U] protected void SerializesInitializer() =>
+			this.AssertSerializesAndRoundTrips<TInterface>(this.Initializer);
+
+		[U] protected void SerializesFluent() =>
+			this.AssertSerializesAndRoundTrips(this.Fluent?.Invoke(NewDescriptor()));
+
+		protected LazyResponses Calls(
 			Func<IElasticClient, Func<TDescriptor, TInterface>, TResponse> fluent,
 			Func<IElasticClient, Func<TDescriptor, TInterface>, Task<TResponse>> fluentAsync,
 			Func<IElasticClient, TInitializer, TResponse> request,
 			Func<IElasticClient, TInitializer, Task<TResponse>> requestAsync
 		)
 		{
+			//this client is outside the lambda so that the callstack is one where we can get the method name
+			//of the current running test and send that as a header, great for e.g fiddler to relate requests with the test that sent it
 			var client = this.Client;
 			return new LazyResponses(async () =>
 			{
-				this.BeforeAllCalls(client, UniqueValues);
+				if (TestClient.Configuration.RunIntegrationTests)
+				{
+					this.IntegrationSetup(client, _uniqueValues);
+				}
+
+				this._usage.CalledSetup = true;
 
 				var dict = new Dictionary<ClientMethod, IResponse>();
-				this.CallIsolatedValue = UniqueValues[ClientMethod.Fluent];
+				_uniqueValues.CurrentView = ClientMethod.Fluent;
+
 				OnBeforeCall(client);
 				dict.Add(ClientMethod.Fluent, fluent(client, this.Fluent));
 				OnAfterCall(client);
 
-				this.CallIsolatedValue = UniqueValues[ClientMethod.FluentAsync];
+				_uniqueValues.CurrentView = ClientMethod.FluentAsync;
 				OnBeforeCall(client);
 				dict.Add(ClientMethod.FluentAsync, await fluentAsync(client, this.Fluent));
 				OnAfterCall(client);
 
-				this.CallIsolatedValue = UniqueValues[ClientMethod.Initializer];
+				_uniqueValues.CurrentView = ClientMethod.Initializer;
 				OnBeforeCall(client);
 				dict.Add(ClientMethod.Initializer, request(client, this.Initializer));
 				OnAfterCall(client);
 
-				this.CallIsolatedValue = UniqueValues[ClientMethod.InitializerAsync];
+				_uniqueValues.CurrentView = ClientMethod.InitializerAsync;
 				OnBeforeCall(client);
 				dict.Add(ClientMethod.InitializerAsync, await requestAsync(client, this.Initializer));
 				OnAfterCall(client);
 				return dict;
 			});
 		}
 
-		protected virtual void AssertUrl(Uri u)
+		private void AssertUrl(Uri u)
 		{
 			var paths = (this.UrlPath ?? "").Split(new[] { '?' }, 2);
 			string path = paths.First(), query = string.Empty;
 			if (paths.Length > 1)
 				query = paths.Last();
 
-			var expectedUri = new UriBuilder("http", "localhost", Port, path, "?" + query).Uri;
+			var expectedUri = new UriBuilder("http", "localhost", this._port, path, "?" + query).Uri;
 
 			u.AbsolutePath.Should().Be(expectedUri.AbsolutePath);
 			u = new UriBuilder(u.Scheme, u.Host, u.Port, u.AbsolutePath, u.Query.Replace("pretty=true&", "").Replace("pretty=true", "")).Uri;
@@ -131,7 +154,7 @@ protected virtual async Task AssertOnAllResponses(Action<TResponse> assert)
 				var response = kv.Value as TResponse;
 				try
 				{
-					this.CallIsolatedValue = UniqueValues[kv.Key];
+					this._uniqueValues.CurrentView = kv.Key;
 					assert(response);
 				}
 #pragma warning disable 7095 //enable this if you expect a single overload to act up
@@ -143,16 +166,5 @@ protected virtual async Task AssertOnAllResponses(Action<TResponse> assert)
 			}
 		}
 
-		[U] protected async Task HitsTheCorrectUrl() =>
-			await this.AssertOnAllResponses(r => this.AssertUrl(r.ApiCall.Uri));
-
-		[U] protected async Task UsesCorrectHttpMethod() =>
-			await this.AssertOnAllResponses(r => r.CallDetails.HttpMethod.Should().Be(this.HttpMethod));
-
-		[U] protected void SerializesInitializer() =>
-			this.AssertSerializesAndRoundTrips<TInterface>(this.Initializer);
-
-		[U] protected void SerializesFluent() =>
-			this.AssertSerializesAndRoundTrips(this.Fluent?.Invoke(this.ClientDoesThisInternally(NewDescriptor())));
 	}
 }

src/Tests/Framework/AsyncLazy.cs

@@ -61,4 +61,4 @@ public LazyResponses(Func<Dictionary<Integration.ClientMethod, IResponse>> facto
 
 		public LazyResponses(Func<Task<Dictionary<Integration.ClientMethod, IResponse>>> factory) : base(factory) { }
 	}
-}
+}