Skip to content

Fix dropped doc callback for retried docs #6815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions .ci/packages.lock.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
{
"version": 1,
"dependencies": {
".NETCoreApp,Version=v5.0": {
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
}
}
".NETCoreApp,Version=v5.0": {}
}
}
2 changes: 1 addition & 1 deletion Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
<ExposedPublicKey>002400000480000094000000060200000024000052534131000400000100010025d3a22bf3781ba85067374ad832dfcba3c4fa8dd89227e36121ba17b2c33ad6b6ce03e45e562050a031e2ff7fe12cff9060a50acbc6a0eef9ef32dc258d90f874b2e76b581938071ccc4b4d98204d1d6ca7a1988d7a211f9fc98efd808cf85f61675b11007d0eb0461dc86a968d6af8ebba7e6b540303b54f1c1f5325c252be</ExposedPublicKey>
</PropertyGroup>

<ItemGroup>
<ItemGroup Condition="'$(TargetFramework)'=='net461' or '$(TargetFramework)'=='net472'">
<!-- Include unconditionally due to difference in how dotnet cli and design time builds generate package.lock.json -->
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" PrivateAssets="All" Version="1.0.3"/>
</ItemGroup>
Expand Down
6 changes: 0 additions & 6 deletions build/scripts/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,6 @@
"resolved": "5.0.0",
"contentHash": "iHoYXA0VaSQUONGENB1aVafjDDZDZpwu39MtaRCTrmwFW/cTcK0b2yKNVYneFHJMc3ChtsSoM9lNtJ1dYXkHfA=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Newtonsoft.Json": {
"type": "Direct",
"requested": "[13.0.1, )",
Expand Down
6 changes: 0 additions & 6 deletions src/ApiGenerator/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,6 @@
"Microsoft.CodeAnalysis.Common": "[3.1.0-beta3-final]"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Newtonsoft.Json": {
"type": "Direct",
"requested": "[13.0.1, )",
Expand Down
6 changes: 0 additions & 6 deletions src/DocGenerator/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,6 @@
"NETStandard.Library": "1.6.1"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Newtonsoft.Json": {
"type": "Direct",
"requested": "[13.0.1, )",
Expand Down
6 changes: 0 additions & 6 deletions src/Elasticsearch.Net.VirtualizedCluster/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,6 @@
"resolved": "4.0.0",
"contentHash": "PCgUb5I1DIe0GsBZ2fPXr/tOgph9Gh6yoHcpsFxNmTvyOQFXxyOcjM7wPs8ylakPNdFYBUs9YeF2Wsx6K/nEDQ=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Microsoft.SourceLink.GitHub": {
"type": "Direct",
"requested": "[1.0.0, )",
Expand Down
12 changes: 0 additions & 12 deletions src/Elasticsearch.Net/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,6 @@
"resolved": "4.6.0",
"contentHash": "kxn3M2rnAGy5N5DgcIwcE8QTePWU/XiYcQVzn9HqTls2NKluVzVSmVWRjK7OUPWbljCXuZxHyhEz9kPRIQeXow=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Microsoft.SourceLink.GitHub": {
"type": "Direct",
"requested": "[1.0.0, )",
Expand Down Expand Up @@ -293,12 +287,6 @@
"resolved": "4.6.0",
"contentHash": "kxn3M2rnAGy5N5DgcIwcE8QTePWU/XiYcQVzn9HqTls2NKluVzVSmVWRjK7OUPWbljCXuZxHyhEz9kPRIQeXow=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Microsoft.SourceLink.GitHub": {
"type": "Direct",
"requested": "[1.0.0, )",
Expand Down
6 changes: 0 additions & 6 deletions src/Nest.JsonNetSerializer/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,6 @@
"resolved": "4.0.0",
"contentHash": "PCgUb5I1DIe0GsBZ2fPXr/tOgph9Gh6yoHcpsFxNmTvyOQFXxyOcjM7wPs8ylakPNdFYBUs9YeF2Wsx6K/nEDQ=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Microsoft.SourceLink.GitHub": {
"type": "Direct",
"requested": "[1.0.0, )",
Expand Down
31 changes: 25 additions & 6 deletions src/Nest/Document/Multiple/BulkAll/BulkAllObservable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -162,24 +162,43 @@ private async Task<BulkAllResponse> BulkAsync(IList<T> buffer, long page, int ba

var retryableDocuments = new List<T>();
var droppedDocuments = new List<Tuple<BulkResponseItemBase, T>>();
var retryableDocsRemainingAfterRetriesExceeded = false;

foreach (var documentWithResponse in response.Items.Zip(buffer, Tuple.Create))
{
if (documentWithResponse.Item1.IsValid) continue;
if (documentWithResponse.Item1.IsValid)
continue;

if (_retryPredicate(documentWithResponse.Item1, documentWithResponse.Item2))
retryableDocuments.Add(documentWithResponse.Item2);
{
if (backOffRetries < _backOffRetries)
{
retryableDocuments.Add(documentWithResponse.Item2);
}
else
{
// We still have retriable documents but have exceeded all retries, so we mark these as
// dropped so they get handled correctly.
retryableDocsRemainingAfterRetriesExceeded = true;
droppedDocuments.Add(documentWithResponse);
}
}
else
{
droppedDocuments.Add(documentWithResponse);
}
}

HandleDroppedDocuments(droppedDocuments, response);

if (retryableDocuments.Count > 0 && backOffRetries < _backOffRetries)
if (retryableDocsRemainingAfterRetriesExceeded)
{
throw ThrowOnBadBulk(response, $"Bulk indexing failed and after retrying {backOffRetries} times.");
}
else if (retryableDocuments.Count > 0)
{
return await RetryDocuments(page, ++backOffRetries, retryableDocuments).ConfigureAwait(false);

if (retryableDocuments.Count > 0)
throw ThrowOnBadBulk(response, $"Bulk indexing failed and after retrying {backOffRetries} times");
}

request.BackPressure?.Release();

Expand Down
6 changes: 0 additions & 6 deletions src/Nest/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,6 @@
"resolved": "4.0.0",
"contentHash": "PCgUb5I1DIe0GsBZ2fPXr/tOgph9Gh6yoHcpsFxNmTvyOQFXxyOcjM7wPs8ylakPNdFYBUs9YeF2Wsx6K/nEDQ=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Microsoft.SourceLink.GitHub": {
"type": "Direct",
"requested": "[1.0.0, )",
Expand Down
30 changes: 12 additions & 18 deletions tests/Tests.Benchmarking/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,6 @@
"LibGit2Sharp.NativeBinaries": "[1.0.245]"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"BenchmarkDotNet.Annotations": {
"type": "Transitive",
"resolved": "0.12.0",
Expand Down Expand Up @@ -194,8 +188,8 @@
},
"Microsoft.CodeCoverage": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "MsKhJmwIfHxNDbTIlgQy29UpWSWPpbZOQPhQ7xalRy+ABnl8/neFHZGzSP3XlpW2dKAXHTFrtIcKzW/kopY2Bg=="
"resolved": "17.3.2",
"contentHash": "+CeYNY9hYNRgv1wAID5koeDVob1ZOrOYfRRTLxU9Zm5ZMDMkMZ8wzXgakxVv+jtk8tPdE8Ze9vVE+czMKapv/Q=="
},
"Microsoft.CSharp": {
"type": "Transitive",
Expand All @@ -219,11 +213,11 @@
},
"Microsoft.NET.Test.Sdk": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "kYmkDYbcDd+jNvmMH4TMtgHjsUYbIsWENM2VcjB0X7TawXbehL5I8OIsu2TgFS/nQCgZE94InrqMxrm7WDy+Lw==",
"resolved": "17.3.2",
"contentHash": "apR0ha1T8FujBwq1P8i/DOZjbI5XhcP/i8As4NnVztVSpZG8GtWRPCstcmgkUkBpvEfcrrDPlJWbuZY+Hl1hSg==",
"dependencies": {
"Microsoft.CodeCoverage": "17.2.0",
"Microsoft.TestPlatform.TestHost": "17.2.0"
"Microsoft.CodeCoverage": "17.3.2",
"Microsoft.TestPlatform.TestHost": "17.3.2"
}
},
"Microsoft.NETCore.Platforms": {
Expand All @@ -238,19 +232,19 @@
},
"Microsoft.TestPlatform.ObjectModel": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "7j1KYDHLhU98XnCEbECMncXLydI9fNiFLcFsiBsP3lV6EkHOaj5kTPAWHYkKnPGRC9TbZUboSQq8rWI4dTQsxg==",
"resolved": "17.3.2",
"contentHash": "DJEIfSA2GDC+2m42vKGNR2hm+Uhta4SpCsLZVVvYIiYMjxtk7GzNnv82qvE4SCW3kIYllMg2D0rr8juuj/f7AA==",
"dependencies": {
"NuGet.Frameworks": "5.11.0",
"System.Reflection.Metadata": "1.6.0"
}
},
"Microsoft.TestPlatform.TestHost": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "bI67J+hers241h7eD2eecS02p9CbKcQDIeoRvO4FgMlTWg2ZTzc0D3uWLYr5U+K5x9O1pNmyMoMDbYIeWY/TWw==",
"resolved": "17.3.2",
"contentHash": "113J19v31pIx+PzmdEw67cWTZWh/YApnprbclFeat6szNbnpKOKG7Ap4PX5LT6E5Da+xONyilxvx2HZPpEaXPQ==",
"dependencies": {
"Microsoft.TestPlatform.ObjectModel": "17.2.0",
"Microsoft.TestPlatform.ObjectModel": "17.3.2",
"Newtonsoft.Json": "9.0.1"
}
},
Expand Down Expand Up @@ -1418,7 +1412,7 @@
"Elastic.Elasticsearch.Xunit": "0.2.6",
"FluentAssertions": "5.10.3",
"JunitXml.TestLogger": "2.1.78",
"Microsoft.NET.Test.Sdk": "17.2.0",
"Microsoft.NET.Test.Sdk": "17.3.2",
"NEST.JsonNetSerializer": "7.0.0",
"Nullean.VsTest.Pretty.TestLogger": "0.3.0",
"Proc": "0.6.1",
Expand Down
30 changes: 12 additions & 18 deletions tests/Tests.ClusterLauncher/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"dependencies": {
".NETCoreApp,Version=v5.0": {
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"Bogus": {
"type": "Transitive",
"resolved": "22.1.2",
Expand Down Expand Up @@ -73,8 +67,8 @@
},
"Microsoft.CodeCoverage": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "MsKhJmwIfHxNDbTIlgQy29UpWSWPpbZOQPhQ7xalRy+ABnl8/neFHZGzSP3XlpW2dKAXHTFrtIcKzW/kopY2Bg=="
"resolved": "17.3.2",
"contentHash": "+CeYNY9hYNRgv1wAID5koeDVob1ZOrOYfRRTLxU9Zm5ZMDMkMZ8wzXgakxVv+jtk8tPdE8Ze9vVE+czMKapv/Q=="
},
"Microsoft.CSharp": {
"type": "Transitive",
Expand All @@ -83,11 +77,11 @@
},
"Microsoft.NET.Test.Sdk": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "kYmkDYbcDd+jNvmMH4TMtgHjsUYbIsWENM2VcjB0X7TawXbehL5I8OIsu2TgFS/nQCgZE94InrqMxrm7WDy+Lw==",
"resolved": "17.3.2",
"contentHash": "apR0ha1T8FujBwq1P8i/DOZjbI5XhcP/i8As4NnVztVSpZG8GtWRPCstcmgkUkBpvEfcrrDPlJWbuZY+Hl1hSg==",
"dependencies": {
"Microsoft.CodeCoverage": "17.2.0",
"Microsoft.TestPlatform.TestHost": "17.2.0"
"Microsoft.CodeCoverage": "17.3.2",
"Microsoft.TestPlatform.TestHost": "17.3.2"
}
},
"Microsoft.NETCore.Platforms": {
Expand All @@ -102,19 +96,19 @@
},
"Microsoft.TestPlatform.ObjectModel": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "7j1KYDHLhU98XnCEbECMncXLydI9fNiFLcFsiBsP3lV6EkHOaj5kTPAWHYkKnPGRC9TbZUboSQq8rWI4dTQsxg==",
"resolved": "17.3.2",
"contentHash": "DJEIfSA2GDC+2m42vKGNR2hm+Uhta4SpCsLZVVvYIiYMjxtk7GzNnv82qvE4SCW3kIYllMg2D0rr8juuj/f7AA==",
"dependencies": {
"NuGet.Frameworks": "5.11.0",
"System.Reflection.Metadata": "1.6.0"
}
},
"Microsoft.TestPlatform.TestHost": {
"type": "Transitive",
"resolved": "17.2.0",
"contentHash": "bI67J+hers241h7eD2eecS02p9CbKcQDIeoRvO4FgMlTWg2ZTzc0D3uWLYr5U+K5x9O1pNmyMoMDbYIeWY/TWw==",
"resolved": "17.3.2",
"contentHash": "113J19v31pIx+PzmdEw67cWTZWh/YApnprbclFeat6szNbnpKOKG7Ap4PX5LT6E5Da+xONyilxvx2HZPpEaXPQ==",
"dependencies": {
"Microsoft.TestPlatform.ObjectModel": "17.2.0",
"Microsoft.TestPlatform.ObjectModel": "17.3.2",
"Newtonsoft.Json": "9.0.1"
}
},
Expand Down Expand Up @@ -897,7 +891,7 @@
"Elastic.Elasticsearch.Xunit": "0.2.6",
"FluentAssertions": "5.10.3",
"JunitXml.TestLogger": "2.1.78",
"Microsoft.NET.Test.Sdk": "17.2.0",
"Microsoft.NET.Test.Sdk": "17.3.2",
"NEST.JsonNetSerializer": "7.0.0",
"Nullean.VsTest.Pretty.TestLogger": "0.3.0",
"Proc": "0.6.1",
Expand Down
6 changes: 0 additions & 6 deletions tests/Tests.Configuration/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,6 @@
"System.Net.Http": "4.3.1"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"NETStandard.Library": {
"type": "Direct",
"requested": "[2.0.3, )",
Expand Down
2 changes: 1 addition & 1 deletion tests/Tests.Core/Tests.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<ProjectReference Include="$(SolutionRoot)\tests\Tests.Domain\Tests.Domain.csproj" />

<PackageReference Include="xunit" Version="2.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="Elastic.Elasticsearch.Xunit" Version="0.2.6" />
<PackageReference Include="JunitXml.TestLogger" Version="2.1.78" />

Expand Down
12 changes: 3 additions & 9 deletions tests/Tests.Core/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,9 @@
},
"Microsoft.NET.Test.Sdk": {
"type": "Direct",
"requested": "[17.2.0, )",
"resolved": "17.2.0",
"contentHash": "kYmkDYbcDd+jNvmMH4TMtgHjsUYbIsWENM2VcjB0X7TawXbehL5I8OIsu2TgFS/nQCgZE94InrqMxrm7WDy+Lw=="
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
"requested": "[17.3.2, )",
"resolved": "17.3.2",
"contentHash": "apR0ha1T8FujBwq1P8i/DOZjbI5XhcP/i8As4NnVztVSpZG8GtWRPCstcmgkUkBpvEfcrrDPlJWbuZY+Hl1hSg=="
},
"NETStandard.Library": {
"type": "Direct",
Expand Down
6 changes: 0 additions & 6 deletions tests/Tests.Domain/packages.lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,6 @@
"System.Net.Http": "4.3.1"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
"type": "Direct",
"requested": "[1.0.3, )",
"resolved": "1.0.3",
"contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg=="
},
"NETStandard.Library": {
"type": "Direct",
"requested": "[2.0.3, )",
Expand Down
Loading