Skip to content

Commit f0a4011

Browse files
dagnirdagnir
authored
Add retry tests for request bodies (#6056)
* Add retry tests for request bodies This commit adds tests for implementations of `RequestBody` and `AsyncRequestBody` where the requests are retried to ensure that they send the same content for every attempt. * Add 0 byte test * Try to reduce memory usage * Switch to plain Jetty server Unfortunately WireMock saves all the request bodies which increases memory usage. Use a plain Jetty server that doesn't save requests to avoid this. We don't rely on any WireWock request assertions so it works out. * Lower max size * Max 8MB * Force s3-tests to run at end * Increase memory
1 parent 0d84faf commit f0a4011

File tree

8 files changed

+982
-0
lines changed

8 files changed

+982
-0
lines changed

‎test/s3-tests/pom.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,22 @@
152152
<version>${awsjavasdk.version}</version>
153153
<scope>test</scope>
154154
</dependency>
155+
<dependency>
156+
<groupId>org.eclipse.jetty</groupId>
157+
<artifactId>jetty-servlet</artifactId>
158+
<scope>test</scope>
159+
</dependency>
160+
<dependency>
161+
<groupId>org.eclipse.jetty</groupId>
162+
<artifactId>jetty-server</artifactId>
163+
<scope>test</scope>
164+
</dependency>
165+
<dependency>
166+
<groupId>software.amazon.awssdk</groupId>
167+
<artifactId>bundle-sdk</artifactId>
168+
<version>${project.version}</version>
169+
<scope>test</scope>
170+
</dependency>
155171
</dependencies>
156172

157173
<build>

‎test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
20+
21+
import java.io.ByteArrayInputStream;
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.net.URI;
25+
import java.nio.ByteBuffer;
26+
import java.nio.charset.StandardCharsets;
27+
import java.util.ArrayList;
28+
import java.util.List;
29+
import java.util.concurrent.ExecutorService;
30+
import java.util.concurrent.Executors;
31+
import java.util.stream.IntStream;
32+
import org.junit.jupiter.api.AfterAll;
33+
import org.junit.jupiter.api.Assumptions;
34+
import org.junit.jupiter.api.BeforeAll;
35+
import org.junit.jupiter.api.BeforeEach;
36+
import org.junit.jupiter.params.ParameterizedTest;
37+
import org.junit.jupiter.params.provider.MethodSource;
38+
import software.amazon.awssdk.core.async.AsyncRequestBody;
39+
import software.amazon.awssdk.http.SdkHttpConfigurationOption;
40+
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
41+
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
42+
import software.amazon.awssdk.regions.Region;
43+
import software.amazon.awssdk.retries.StandardRetryStrategy;
44+
import software.amazon.awssdk.retries.api.BackoffStrategy;
45+
import software.amazon.awssdk.services.s3.model.S3Exception;
46+
import software.amazon.awssdk.utils.AttributeMap;
47+
48+
/**
49+
* Tests to ensure different {@link AsyncRequestBody} implementations return the same data for every retry.
50+
*/
51+
public class AsyncRequestBodyRetryTest extends BaseRequestBodyRetryTest {
52+
private static ExecutorService requestBodyExecutor;
53+
private static SdkAsyncHttpClient netty;
54+
private S3AsyncClient s3;
55+
56+
@BeforeAll
57+
public static void setup() throws Exception {
58+
BaseRequestBodyRetryTest.setup();
59+
requestBodyExecutor = Executors.newSingleThreadExecutor();
60+
netty = NettyNioAsyncHttpClient.builder()
61+
.buildWithDefaults(AttributeMap.builder()
62+
.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES,
63+
true)
64+
.build());
65+
}
66+
67+
@BeforeEach
68+
public void methodSetup() {
69+
s3 = S3AsyncClient.builder()
70+
.overrideConfiguration(o -> o.retryStrategy(StandardRetryStrategy.builder()
71+
.maxAttempts(3)
72+
.backoffStrategy(BackoffStrategy.retryImmediately())
73+
.build()))
74+
.region(Region.US_WEST_2)
75+
.endpointOverride(URI.create("https://localhost:" + serverHttpsPort()))
76+
.httpClient(netty)
77+
.forcePathStyle(true)
78+
.build();
79+
}
80+
81+
@AfterAll
82+
public static void teardown() throws Exception {
83+
BaseRequestBodyRetryTest.teardown();
84+
netty.close();
85+
requestBodyExecutor.shutdown();
86+
}
87+
88+
@ParameterizedTest
89+
@MethodSource("retryTestCases")
90+
void test_retries_allAttemptsSendSameBody(TestCase tc) throws IOException {
91+
Assumptions.assumeFalse(tc.type == BodyType.BLOCKING_INPUTSTREAM,
92+
"forBlockingInputStream does not support retrying");
93+
// all content is created the same way so this data should match what's in the RequestBody
94+
byte[] referenceData = makeArrayOfSize(tc.size.getNumBytes());
95+
String expectedCrc32 = calculateCrc32(new ByteArrayInputStream(referenceData));
96+
97+
AsyncRequestBody body = makeRequestBody(tc);
98+
99+
assertThatThrownBy(s3.putObject(r -> r.bucket("my-bucket").key("my-obj"), body)::join)
100+
.hasCauseInstanceOf(S3Exception.class)
101+
.matches(e -> {
102+
S3Exception s3e = (S3Exception) e.getCause();
103+
return s3e.numAttempts() == 3 && s3e.statusCode() == 500;
104+
}, "Should attempt total of 3 times");
105+
106+
List<String> requestBodyChecksums = getRequestChecksums();
107+
assertThat(requestBodyChecksums.size()).isEqualTo(3);
108+
109+
requestBodyChecksums.forEach(bodyChecksum -> assertThat(bodyChecksum).isEqualTo(expectedCrc32));
110+
}
111+
112+
private static List<TestCase> retryTestCases() {
113+
List<TestCase> testCases = new ArrayList<>();
114+
115+
for (BodyType type : BodyType.values()) {
116+
for (BodySize size : BodySize.values()) {
117+
testCases.add(new TestCase().type(type).size(size));
118+
}
119+
}
120+
121+
return testCases;
122+
}
123+
124+
private AsyncRequestBody makeRequestBody(TestCase tc) throws IOException {
125+
int nBytes = tc.size.getNumBytes();
126+
switch (tc.type) {
127+
case STRING:
128+
return AsyncRequestBody.fromString(makeStringOfSize(nBytes), StandardCharsets.UTF_8);
129+
case BYTES:
130+
return AsyncRequestBody.fromBytes(makeArrayOfSize(nBytes));
131+
case BYTES_UNSAFE:
132+
return AsyncRequestBody.fromBytesUnsafe(makeArrayOfSize(nBytes));
133+
case FILE:
134+
return AsyncRequestBody.fromFile(testFiles.get(tc.size));
135+
case INPUTSTREAM: {
136+
InputStream is = getMarkSupportedStreamOfSize(tc.size);
137+
return AsyncRequestBody.fromInputStream(cfg -> cfg.inputStream(is)
138+
.contentLength((long) nBytes)
139+
// read limit has to be positive
140+
.maxReadLimit(nBytes == 0 ? 1 : nBytes)
141+
.executor(requestBodyExecutor));
142+
}
143+
case REMAINING_BYTE_BUFFER:
144+
// fall through
145+
case REMAINING_BYTE_BUFFER_UNSAFE:
146+
// fall through
147+
case BYTE_BUFFER_UNSAFE:
148+
// fall through
149+
case BYTE_BUFFER: {
150+
ByteBuffer byteBuffer = ByteBuffer.wrap(makeArrayOfSize(nBytes));
151+
switch (tc.type) {
152+
case REMAINING_BYTE_BUFFER:
153+
return AsyncRequestBody.fromRemainingByteBuffer(byteBuffer);
154+
case REMAINING_BYTE_BUFFER_UNSAFE:
155+
return AsyncRequestBody.fromRemainingByteBufferUnsafe(byteBuffer);
156+
case BYTE_BUFFER_UNSAFE:
157+
return AsyncRequestBody.fromByteBufferUnsafe(byteBuffer);
158+
case BYTE_BUFFER:
159+
return AsyncRequestBody.fromByteBuffer(byteBuffer);
160+
default:
161+
throw new RuntimeException("Unexpected type: " + tc.type);
162+
}
163+
}
164+
case REMAINING_BYTE_BUFFERS:
165+
// fall through
166+
case REMAINING_BYTE_BUFFERS_UNSAFE:
167+
// fall through
168+
case BYTE_BUFFERS_UNSAFE:
169+
// fall through
170+
case BYTE_BUFFERS: {
171+
ByteBuffer[] buffers;
172+
if (tc.size.getNumBytes() > 0) {
173+
byte[] bbContent = getDataSegment();
174+
int nSegments = nBytes / bbContent.length;
175+
buffers = IntStream.range(0, nSegments)
176+
.mapToObj(i -> ByteBuffer.wrap(bbContent))
177+
.toArray(ByteBuffer[]::new);
178+
} else {
179+
// TODO: This is a workaround because you can't do AsyncRequestBody.fromByteBuffers(new ByteBuffer[0]); the
180+
// subscriber is never signaled onComplete. See issue JAVA-8215.
181+
buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) };
182+
}
183+
184+
switch (tc.type) {
185+
case REMAINING_BYTE_BUFFERS:
186+
return AsyncRequestBody.fromRemainingByteBuffers(buffers);
187+
case REMAINING_BYTE_BUFFERS_UNSAFE:
188+
return AsyncRequestBody.fromRemainingByteBuffersUnsafe(buffers);
189+
case BYTE_BUFFERS_UNSAFE:
190+
return AsyncRequestBody.fromByteBuffersUnsafe(buffers);
191+
case BYTE_BUFFERS:
192+
return AsyncRequestBody.fromByteBuffers(buffers);
193+
default:
194+
throw new RuntimeException("Unexpected type: " + tc.type);
195+
}
196+
}
197+
default:
198+
throw new RuntimeException("Unsupported body type: " + tc.type);
199+
}
200+
}
201+
202+
private enum BodyType {
203+
STRING,
204+
205+
BYTES,
206+
BYTES_UNSAFE,
207+
208+
INPUTSTREAM,
209+
BLOCKING_INPUTSTREAM, // Note: doesn't support retries, left out for testing
210+
211+
BYTE_BUFFER,
212+
BYTE_BUFFER_UNSAFE,
213+
REMAINING_BYTE_BUFFER,
214+
REMAINING_BYTE_BUFFER_UNSAFE,
215+
216+
217+
BYTE_BUFFERS,
218+
BYTE_BUFFERS_UNSAFE,
219+
REMAINING_BYTE_BUFFERS,
220+
REMAINING_BYTE_BUFFERS_UNSAFE,
221+
222+
FILE;
223+
}
224+
225+
226+
private static class TestCase {
227+
private BodyType type;
228+
private BodySize size;
229+
230+
public TestCase type(BodyType type) {
231+
this.type = type;
232+
return this;
233+
}
234+
235+
public TestCase size(BodySize size) {
236+
this.size = size;
237+
return this;
238+
}
239+
240+
@Override
241+
public String toString() {
242+
return "TestCase{" +
243+
"type=" + type +
244+
", size=" + size +
245+
'}';
246+
}
247+
}
248+
}

0 commit comments

Comments
 (0)