Skip to content
This repository was archived by the owner on Jan 23, 2025. It is now read-only.

Commit 2699887

Browse files
coderReviewcoderReview
authored
Merge pull request #478 from appirio-tech/download-all-fix-2
Download all fix
2 parents 712a277 + 367003a commit 2699887

File tree

9 files changed

+132
-27
lines changed

9 files changed

+132
-27
lines changed

‎build-dependencies.xml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,9 @@
350350
<property name="asm-5.2.jar" value="${ext_libdir}/asm/asm-5.2.jar"/>
351351
<property name="asm-commons-5.2.jar" value="${ext_libdir}/asm/asm-commons-5.2.jar"/>
352352
<property name="asm-tree-5.2.jar" value="${ext_libdir}/asm/asm-tree-5.2.jar"/>
353-
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.0.004.jar"/>
353+
<property name="aws-java-sdk.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-1.11.490.jar"/>
354+
<property name="aws-java-sdk-core.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-core-1.11.490.jar"/>
355+
<property name="aws-java-sdk-s3.jar" value="${ext_libdir}/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar"/>
354356
<property name="jackson-core.jar" value="${ext_libdir}/jackson/2.8.1/jackson-core-asl.jar"/>
355357
<property name="jackson-mapper.jar" value="${ext_libdir}/jackson/2.8.1/jackson-mapper-asl.jar"/>
356358
<property name="jackson-annotations-2.8.1.jar" value="${ext_libdir}/jackson/2.8.1/jackson-annotations-2.8.1.jar"/>
@@ -635,6 +637,8 @@
635637
<pathelement location="${encoder.jar}"/>
636638
<pathelement location="${javaee.jar}"/>
637639
<pathelement location="${aws-java-sdk.jar}"/>
640+
<pathelement location="${aws-java-sdk-core.jar}"/>
641+
<pathelement location="${aws-java-sdk-s3.jar}"/>
638642
<pathelement location="${jrss.jar}"/>
639643
<pathelement location="${joda-time.jar}"/>
640644
<pathelement location="${mime-util.jar}"/>

‎build.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@
233233
<copy file="${asm-commons-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
234234
<copy file="${asm-tree-5.2.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
235235
<copy file="${aws-java-sdk.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
236+
<copy file="${aws-java-sdk-core.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
237+
<copy file="${aws-java-sdk-s3.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
236238
<copy file="${axis.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
237239
<copy file="${commons-dbcp.jar}" todir="${ear_shared_libdir}" overwrite="true"/>
238240
<copy file="${commons-discovery.jar}" todir="${ear_shared_libdir}" overwrite="true"/>

‎lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar

3.52 KB
Binary file not shown.

‎lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar

915 KB
Binary file not shown.

‎lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar

957 KB
Binary file not shown.

‎src/java/main/com/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.amazonaws.services.s3.model.GetObjectRequest;
77
import com.amazonaws.services.s3.model.S3Object;
8+
import com.amazonaws.services.s3.AmazonS3URI;
89
import com.topcoder.direct.services.view.action.contest.launch.ContestAction;
910
import com.topcoder.direct.services.view.dto.contest.ContestRoundType;
1011
import com.topcoder.direct.services.view.dto.contest.ContestType;
@@ -15,18 +16,19 @@
1516
import com.topcoder.service.project.SoftwareCompetition;
1617
import com.topcoder.servlet.request.FileUpload;
1718
import com.topcoder.servlet.request.UploadedFile;
19+
import com.topcoder.shared.util.logging.Logger;
1820
import org.apache.commons.io.FilenameUtils;
19-
20-
import java.io.BufferedInputStream;
21-
import java.io.ByteArrayInputStream;
22-
import java.io.ByteArrayOutputStream;
23-
import java.io.InputStream;
24-
import java.io.PipedInputStream;
25-
import java.io.PipedOutputStream;
21+
import org.apache.http.HttpEntity;
22+
import org.apache.http.HttpResponse;
23+
import org.apache.http.HttpStatus;
24+
import org.apache.http.client.HttpResponseException;
25+
import org.apache.http.client.methods.HttpGet;
26+
import org.apache.http.impl.client.DefaultHttpClient;
27+
28+
import java.io.*;
2629
import java.util.ArrayList;
2730
import java.util.List;
2831
import java.util.zip.ZipEntry;
29-
import java.util.zip.ZipInputStream;
3032
import java.util.zip.ZipOutputStream;
3133

3234
/**
@@ -65,6 +67,11 @@
6567
*/
6668
public class DownloadAllSoftwareSubmissionsAction extends ContestAction {
6769

70+
/**
71+
* Logging instance
72+
*/
73+
private static final Logger logger = Logger.getLogger(DownloadAllSoftwareSubmissionsAction.class);
74+
6875
/**
6976
* The id of the final submission type.
7077
*
@@ -287,15 +294,42 @@ public void run() {
287294
byte[] buffer = new byte[8192];
288295
int read;
289296
InputStream is = null;
297+
DefaultHttpClient httpClient = new DefaultHttpClient();
290298
try {
291299
for (Submission sub : submissionsToDownload) {
292300
String submissionFileZipName;
293-
// url != null is s3
301+
// url != null is s3/external url
294302
if (sub.getUpload().getUrl() != null) {
295-
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
296-
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
297-
is = s3Object.getObjectContent();
298-
submissionFileZipName = DirectUtils.getS3FileKey(sub.getUpload().getUrl());
303+
try {
304+
AmazonS3URI s3Uri = DirectUtils.getS3Uri(sub.getUpload().getUrl());
305+
if (s3Uri != null) {
306+
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
307+
DirectUtils.getS3FileKey(sub.getUpload().getUrl())));
308+
is = s3Object.getObjectContent();
309+
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getS3FileKey(sub.getUpload().getUrl());
310+
} else {
311+
// external url other than s3
312+
HttpGet request = new HttpGet(sub.getUpload().getUrl());
313+
HttpResponse response = httpClient.execute(request);
314+
// skip status code >=400
315+
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
316+
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
317+
}
318+
319+
HttpEntity entity = response.getEntity();
320+
if (entity != null) {
321+
is = entity.getContent();
322+
} else {
323+
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
324+
}
325+
submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getFileNameFromUrl(sub.getUpload().getUrl());
326+
}
327+
} catch (Exception e) {
328+
logger.error("Fail to get submission " + sub.getId() + " url: " + sub.getUpload().getUrl() +
329+
" message: " + e.getMessage());
330+
logger.info("Skipping submission " + sub.getId() + " url: " + sub.getUpload().getUrl());
331+
continue;
332+
}
299333
} else {
300334
UploadedFile file;
301335
if (DirectUtils.isStudio(contest)) {
@@ -350,6 +384,10 @@ public void run() {
350384
// ignore
351385
}
352386
}
387+
} finally {
388+
if (httpClient != null) {
389+
httpClient.getConnectionManager().shutdown();
390+
}
353391
}
354392
try {
355393
zos.close();

‎src/java/main/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
import com.amazonaws.services.s3.model.GetObjectRequest;
77
import com.amazonaws.services.s3.model.S3Object;
8+
import com.amazonaws.services.s3.model.S3Object;
9+
import com.amazonaws.services.s3.AmazonS3URI;
810
import com.topcoder.direct.services.view.action.BaseDirectStrutsAction;
911
import com.topcoder.direct.services.view.dto.contest.ContestType;
1012
import com.topcoder.direct.services.view.util.DirectUtils;
@@ -13,6 +15,12 @@
1315
import com.topcoder.service.project.SoftwareCompetition;
1416
import com.topcoder.servlet.request.FileUpload;
1517
import com.topcoder.servlet.request.UploadedFile;
18+
import org.apache.http.HttpEntity;
19+
import org.apache.http.HttpResponse;
20+
import org.apache.http.HttpStatus;
21+
import org.apache.http.client.HttpResponseException;
22+
import org.apache.http.client.methods.HttpGet;
23+
import org.apache.http.impl.client.DefaultHttpClient;
1624

1725
import java.io.InputStream;
1826

@@ -91,9 +99,9 @@ public class DownloadSoftwareSubmissionAction extends BaseDirectStrutsAction {
9199
private SoftwareCompetition contest;
92100

93101
/**
94-
* S3 url of uploaded file. Null if it use local file
102+
* External url of uploaded file. Null if it use local file
95103
*/
96-
private String s3Url;
104+
private String externalUrl;
97105

98106
/**
99107
* S3 bucket
@@ -144,7 +152,7 @@ protected void executeAction() throws Exception {
144152
uploadedFile = fileUpload.getUploadedFile(submission.getUpload().getParameter());
145153
}
146154
} else {
147-
s3Url = submission.getUpload().getUrl();
155+
externalUrl = submission.getUpload().getUrl();
148156
}
149157

150158
}
@@ -157,10 +165,27 @@ protected void executeAction() throws Exception {
157165
* if any error occurs when getting the input stream of the uploaded file.
158166
*/
159167
public InputStream getInputStream() throws Exception {
160-
if (s3Url != null) {
161-
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
162-
DirectUtils.getS3FileKey(s3Url)));
163-
return s3Object.getObjectContent();
168+
if (externalUrl != null) {
169+
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
170+
if (s3Uri != null) {
171+
S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket,
172+
DirectUtils.getS3FileKey(externalUrl)));
173+
return s3Object.getObjectContent();
174+
} else {
175+
DefaultHttpClient httpClient = new DefaultHttpClient();
176+
HttpGet request = new HttpGet(externalUrl);
177+
HttpResponse response = httpClient.execute(request);
178+
// skip status code >=400
179+
if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) {
180+
throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external");
181+
}
182+
183+
HttpEntity entity = response.getEntity();
184+
if (entity == null) {
185+
throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external");
186+
}
187+
return entity.getContent();
188+
}
164189
}
165190

166191
if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {
@@ -188,8 +213,12 @@ public InputStream getInputStream() throws Exception {
188213
* if any error occurs when getting the file name of the uploaded file.
189214
*/
190215
public String getContentDisposition() throws Exception {
191-
if (s3Url != null) {
192-
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(s3Url) + "\"";
216+
if (externalUrl != null) {
217+
AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl);
218+
if (s3Uri != null) {
219+
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(externalUrl) + "\"";
220+
}
221+
return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getFileNameFromUrl(externalUrl) + "\"";
193222
}
194223

195224
if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) {

‎src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,11 +214,11 @@ private Object getMapResult(SoftwareCompetition bean) {
214214

215215
// retrieve review scorecard id.
216216
for(com.topcoder.project.phases.Phase phase : bean.getProjectPhases().getAllPhases()){
217-
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName())){
217+
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
218218
result.put("reviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
219219
}
220220

221-
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName())){
221+
if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){
222222
result.put("iterativeReviewScorecardId", phase.getAttributes().get("Scorecard ID").toString());
223223
}
224224
}

‎src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.amazonaws.auth.PropertiesCredentials;
77
import com.amazonaws.services.s3.AmazonS3Client;
8+
import com.amazonaws.services.s3.AmazonS3URI;
89
import com.opensymphony.xwork2.ActionContext;
910
import com.topcoder.clients.dao.ProjectContestFeePercentageService;
1011
import com.topcoder.clients.dao.ProjectContestFeeService;
@@ -4075,9 +4076,40 @@ public static AmazonS3Client getS3Client() {
40754076
* @throws Exception if any exceptions occurs
40764077
*/
40774078
public static String getS3FileKey(String url) throws Exception {
4079+
AmazonS3URI s3Uri = getS3Uri(url);
4080+
if (s3Uri == null) {
4081+
return null;
4082+
}
4083+
return s3Uri.getKey();
4084+
}
4085+
4086+
/**
4087+
* Get upload uri from url.
4088+
*
4089+
* @param url upload url
4090+
* @return s3 uri
4091+
*/
4092+
public static AmazonS3URI getS3Uri(String url) {
4093+
try {
4094+
AmazonS3URI s3Uri = new AmazonS3URI(url);
4095+
return s3Uri;
4096+
} catch (IllegalArgumentException ex) {
4097+
// url doesn't seem to be a valid
4098+
return null;
4099+
}
4100+
}
4101+
4102+
/**
4103+
* Get filename from URL
4104+
*
4105+
* @param url
4106+
* @return filename
4107+
* @throws Exception
4108+
*/
4109+
public static String getFileNameFromUrl(String url) throws Exception {
40784110
String path = new URL(url).getPath();
4079-
int sep = path.lastIndexOf( '/' );
4080-
return ( sep < 0 ) ? path : path.substring( sep + 1 );
4111+
int sep = path.lastIndexOf('/');
4112+
return (sep < 0) ? path : path.substring(sep + 1);
40814113
}
40824114

40834115
/**

0 commit comments

Comments
 (0)