Skip to content

Commit 750f164

Browse files
zoewanggzoewangg
authored
Implement download directory in the S3TransferManager (#3027)
* [TM DownloadDirectory Part1] Create POJO classes for download directory (#2993) * Create POJO classes for download directory * Address feedback * Address comments * A couple of minor refactoring on the S3TransferManager (#2997) * [TM DownloadDirectory Part2] Implement download directory in transfer manager (#3010) * Implement download directory in transfer manager * Add more tests and address comments * Remove create parent directories logic and add changelog entry * [TM DownloadDirectory Part3] Various updates on downloadDirectory (#3020) * Implement download directory in transfer manager * Add more tests and address comments * Remove create parent directories logic and add changelog entry * Various updates on downloadDirectory 1. limit the number of concurrent download file 2. create nonexistent parent directories 3. normalize key if prefix is not empty * By default, set delimiter to null to avoid potentially excessive listObjectsV2 calls * Move async buffering logic to SdkPublisher * Move AsyncBufferingSubscriber back to s3-tranfer-manager module and address feedback * remove unnecessary isDelivering.set(false)
1 parent e0328b7 commit 750f164

File tree

38 files changed

+2578
-59
lines changed

38 files changed

+2578
-59
lines changed

‎.changes/next-release/bugfix-S3TransferManager-2f34394.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"category": "S3 Transfer Manager",
3+
"contributor": "",
4+
"type": "bugfix",
5+
"description": "A couple of minor refactoring on the S3TransferManager. 1. `CompletedDirectoryUpload#failedTransfers` now returns `List<FailedFileUpload>` instead of `Collection<FailedFileUpload>`. 2. `UploadDirectoryOverrideConfiguration#uploadFileRequestTransformer` now returns `Consumer<UploadFileRequest.Builder>` instead of `Optional<Consumer<UploadFileRequest.Builder>>` it will be no-op if no uploadFileRequestTransformer is provided"
6+
}

‎.changes/next-release/feature-S3TransferManager-4aba25e.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"category": "S3 Transfer Manager",
3+
"contributor": "",
4+
"type": "feature",
5+
"description": "Implement downloadDirectory API in the S3TransferManager"
6+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/async/SdkPublisher.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,5 +161,4 @@ default CompletableFuture<Void> subscribe(Consumer<T> consumer) {
161161
subscribe(new SequentialSubscriber<>(consumer, future));
162162
return future;
163163
}
164-
165164
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
21+
22+
import java.io.IOException;
23+
import java.io.UncheckedIOException;
24+
import java.nio.charset.StandardCharsets;
25+
import java.nio.file.FileVisitResult;
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.nio.file.Paths;
29+
import java.nio.file.SimpleFileVisitor;
30+
import java.nio.file.attribute.BasicFileAttributes;
31+
import java.util.concurrent.TimeUnit;
32+
import org.apache.commons.lang3.RandomStringUtils;
33+
import org.junit.After;
34+
import org.junit.AfterClass;
35+
import org.junit.Before;
36+
import org.junit.BeforeClass;
37+
import org.junit.Test;
38+
import software.amazon.awssdk.testutils.FileUtils;
39+
import software.amazon.awssdk.utils.Logger;
40+
41+
public class S3TransferManagerDownloadDirectoryIntegrationTest extends S3IntegrationTestBase {
42+
private static final Logger log = Logger.loggerFor(S3TransferManagerDownloadDirectoryIntegrationTest.class);
43+
private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
44+
private static final String TEST_BUCKET_CUSTOM_DELIMITER = temporaryBucketName("S3TransferManagerUploadIntegrationTest"
45+
+ "-delimiter");
46+
private static final String CUSTOM_DELIMITER = "-";
47+
48+
private static S3TransferManager tm;
49+
private static Path sourceDirectory;
50+
private Path destinationDirectory;
51+
52+
@BeforeClass
53+
public static void setUp() throws Exception {
54+
S3IntegrationTestBase.setUp();
55+
createBucket(TEST_BUCKET);
56+
createBucket(TEST_BUCKET_CUSTOM_DELIMITER);
57+
sourceDirectory = createLocalTestDirectory();
58+
59+
tm = S3TransferManager.builder()
60+
.s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
61+
.region(DEFAULT_REGION)
62+
.maxConcurrency(100))
63+
.build();
64+
65+
tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory).bucket(TEST_BUCKET)).completionFuture().join();
66+
67+
tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory)
68+
.delimiter(CUSTOM_DELIMITER)
69+
.bucket(TEST_BUCKET_CUSTOM_DELIMITER))
70+
.completionFuture().join();
71+
}
72+
73+
@Before
74+
public void setUpPerTest() throws IOException {
75+
destinationDirectory = Files.createTempDirectory("destination");
76+
}
77+
78+
@After
79+
public void cleanup() {
80+
FileUtils.cleanUpTestDirectory(destinationDirectory);
81+
}
82+
83+
@AfterClass
84+
public static void teardown() {
85+
try {
86+
FileUtils.cleanUpTestDirectory(sourceDirectory);
87+
} catch (Exception exception) {
88+
log.warn(() -> "Failed to clean up test directory " + sourceDirectory, exception);
89+
}
90+
91+
try {
92+
deleteBucketAndAllContents(TEST_BUCKET);
93+
} catch (Exception exception) {
94+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET, exception);
95+
}
96+
97+
try {
98+
deleteBucketAndAllContents(TEST_BUCKET_CUSTOM_DELIMITER);
99+
} catch (Exception exception) {
100+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET_CUSTOM_DELIMITER, exception);
101+
}
102+
103+
closeQuietly(tm, log.logger());
104+
S3IntegrationTestBase.cleanUp();
105+
}
106+
107+
/**
108+
* The destination directory structure should match with the directory uploaded
109+
* <pre>
110+
* {@code
111+
* - destination
112+
* - 2021
113+
* - 1.txt
114+
* - 2.txt
115+
* - 2022
116+
* - 1.txt
117+
* - important.txt
118+
* }
119+
* </pre>
120+
*/
121+
@Test
122+
public void downloadDirectory() throws Exception {
123+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
124+
.bucket(TEST_BUCKET));
125+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
126+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
127+
assertTwoDirectoriesHaveSameStructure(sourceDirectory, destinationDirectory);
128+
}
129+
130+
/**
131+
* The destination directory structure should be the following with prefix "notes"
132+
* <pre>
133+
* {@code
134+
* - source
135+
* - README.md
136+
* - CHANGELOG.md
137+
* - notes
138+
* - 2021
139+
* - 1.txt
140+
* - 2.txt
141+
* - 2022
142+
* - 1.txt
143+
* - important.txt
144+
* }
145+
* </pre>
146+
*/
147+
@Test
148+
public void downloadDirectory_withPrefix() throws Exception {
149+
String prefix = "notes";
150+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
151+
.prefix(prefix)
152+
.bucket(TEST_BUCKET));
153+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
154+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
155+
156+
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve(prefix), destinationDirectory);
157+
}
158+
159+
/**
160+
* The destination directory structure should be the following with prefix "notes"
161+
* <pre>
162+
* {@code
163+
* - destination
164+
* - 1.txt
165+
* - 2.txt
166+
* }
167+
* </pre>
168+
*/
169+
@Test
170+
public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
171+
String prefix = "notes-2021";
172+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
173+
.delimiter(CUSTOM_DELIMITER)
174+
.prefix(prefix)
175+
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
176+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
177+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
178+
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve("notes").resolve("2021"), destinationDirectory);
179+
}
180+
181+
private static void assertTwoDirectoriesHaveSameStructure(Path path, Path otherPath) {
182+
try {
183+
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
184+
@Override
185+
public FileVisitResult visitFile(Path file,
186+
BasicFileAttributes attrs)
187+
throws IOException {
188+
FileVisitResult result = super.visitFile(file, attrs);
189+
190+
Path relativePath = path.relativize(file);
191+
Path otherFile = otherPath.resolve(relativePath);
192+
log.debug(() -> String.format("Comparing %s with %s", file, otherFile));
193+
assertThat(file).hasSameBinaryContentAs(otherFile);
194+
return result;
195+
}
196+
});
197+
} catch (IOException e) {
198+
throw new UncheckedIOException(String.format("Failed to compare %s with %s", path, otherPath), e);
199+
}
200+
}
201+
202+
/**
203+
* Create a test directory with the following structure
204+
* <pre>
205+
* {@code
206+
* - source
207+
* - README.md
208+
* - CHANGELOG.md
209+
* - notes
210+
* - 2021
211+
* - 1.txt
212+
* - 2.txt
213+
* - 2022
214+
* - 1.txt
215+
* - important.txt
216+
* }
217+
* </pre>
218+
*/
219+
private static Path createLocalTestDirectory() throws IOException {
220+
Path directory = Files.createTempDirectory("source");
221+
222+
String directoryName = directory.toString();
223+
224+
Files.createDirectory(Paths.get(directoryName, "notes"));
225+
Files.createDirectory(Paths.get(directoryName, "notes", "2021"));
226+
Files.createDirectory(Paths.get(directoryName, "notes", "2022"));
227+
Files.write(Paths.get(directoryName, "README.md"), RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
228+
Files.write(Paths.get(directoryName, "CHANGELOG.md"), RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
229+
Files.write(Paths.get(directoryName, "notes", "2021", "1.txt"),
230+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
231+
Files.write(Paths.get(directoryName, "notes", "2021", "2.txt"),
232+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
233+
Files.write(Paths.get(directoryName, "notes", "2022", "1.txt"),
234+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
235+
Files.write(Paths.get(directoryName, "notes", "important.txt"),
236+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
237+
return directory;
238+
}
239+
}

0 commit comments

Comments
 (0)