diff --git a/build-dependencies.xml b/build-dependencies.xml index 86d759cd3..f3cde9a22 100644 --- a/build-dependencies.xml +++ b/build-dependencies.xml @@ -350,7 +350,9 @@ - + + + @@ -635,6 +637,8 @@ + + diff --git a/build.xml b/build.xml index e6a622978..5e20f58ee 100644 --- a/build.xml +++ b/build.xml @@ -233,6 +233,8 @@ + + diff --git a/lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar b/lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar new file mode 100644 index 000000000..548cdc4b4 Binary files /dev/null and b/lib/third_party/aws-java-sdk/aws-java-sdk-1.11.490.jar differ diff --git a/lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar b/lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar new file mode 100644 index 000000000..ab8458186 Binary files /dev/null and b/lib/third_party/aws-java-sdk/aws-java-sdk-core-1.11.490.jar differ diff --git a/lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar b/lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar new file mode 100644 index 000000000..8a0e96502 Binary files /dev/null and b/lib/third_party/aws-java-sdk/aws-java-sdk-s3-1.11.490.jar differ diff --git a/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java b/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java index 0fecd4697..7d8300409 100644 --- a/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java +++ b/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadAllSoftwareSubmissionsAction.java @@ -5,6 +5,7 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.AmazonS3URI; import com.topcoder.direct.services.view.action.contest.launch.ContestAction; import com.topcoder.direct.services.view.dto.contest.ContestRoundType; import com.topcoder.direct.services.view.dto.contest.ContestType; @@ -15,18 +16,19 @@ import com.topcoder.service.project.SoftwareCompetition; import com.topcoder.servlet.request.FileUpload; import com.topcoder.servlet.request.UploadedFile; +import com.topcoder.shared.util.logging.Logger; import org.apache.commons.io.FilenameUtils; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; + +import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; /** @@ -65,6 +67,11 @@ */ public class DownloadAllSoftwareSubmissionsAction extends ContestAction { + /** + * Logging instance + */ + private static final Logger logger = Logger.getLogger(DownloadAllSoftwareSubmissionsAction.class); + /** * The id of the final submission type. * @@ -287,15 +294,42 @@ public void run() { byte[] buffer = new byte[8192]; int read; InputStream is = null; + DefaultHttpClient httpClient = new DefaultHttpClient(); try { for (Submission sub : submissionsToDownload) { String submissionFileZipName; - // url != null is s3 + // url != null is s3/external url if (sub.getUpload().getUrl() != null) { - S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket, - DirectUtils.getS3FileKey(sub.getUpload().getUrl()))); - is = s3Object.getObjectContent(); - submissionFileZipName = DirectUtils.getS3FileKey(sub.getUpload().getUrl()); + try { + AmazonS3URI s3Uri = DirectUtils.getS3Uri(sub.getUpload().getUrl()); + if (s3Uri != null) { + S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket, + DirectUtils.getS3FileKey(sub.getUpload().getUrl()))); + is = s3Object.getObjectContent(); + submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getS3FileKey(sub.getUpload().getUrl()); + } else { + // external url other than s3 + HttpGet request = new HttpGet(sub.getUpload().getUrl()); + HttpResponse response = httpClient.execute(request); + // skip status code >=400 + if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) { + throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external"); + } + + HttpEntity entity = response.getEntity(); + if (entity != null) { + is = entity.getContent(); + } else { + throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external"); + } + submissionFileZipName = "Submission-" + sub.getId() + "-" + DirectUtils.getFileNameFromUrl(sub.getUpload().getUrl()); + } + } catch (Exception e) { + logger.error("Fail to get submission " + sub.getId() + " url: " + sub.getUpload().getUrl() + + " message: " + e.getMessage()); + logger.info("Skipping submission " + sub.getId() + " url: " + sub.getUpload().getUrl()); + continue; + } } else { UploadedFile file; if (DirectUtils.isStudio(contest)) { @@ -350,6 +384,10 @@ public void run() { // ignore } } + } finally { + if (httpClient != null) { + httpClient.getConnectionManager().shutdown(); + } } try { zos.close(); diff --git a/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java b/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java index e63f3d242..f96af4385 100644 --- a/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java +++ b/src/java/main/com/topcoder/direct/services/view/action/contest/DownloadSoftwareSubmissionAction.java @@ -5,6 +5,8 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.AmazonS3URI; import com.topcoder.direct.services.view.action.BaseDirectStrutsAction; import com.topcoder.direct.services.view.dto.contest.ContestType; import com.topcoder.direct.services.view.util.DirectUtils; @@ -13,6 +15,12 @@ import com.topcoder.service.project.SoftwareCompetition; import com.topcoder.servlet.request.FileUpload; import com.topcoder.servlet.request.UploadedFile; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; import java.io.InputStream; @@ -91,9 +99,9 @@ public class DownloadSoftwareSubmissionAction extends BaseDirectStrutsAction { private SoftwareCompetition contest; /** - * S3 url of uploaded file. Null if it use local file + * External url of uploaded file. Null if it use local file */ - private String s3Url; + private String externalUrl; /** * S3 bucket @@ -144,7 +152,7 @@ protected void executeAction() throws Exception { uploadedFile = fileUpload.getUploadedFile(submission.getUpload().getParameter()); } } else { - s3Url = submission.getUpload().getUrl(); + externalUrl = submission.getUpload().getUrl(); } } @@ -157,10 +165,27 @@ protected void executeAction() throws Exception { * if any error occurs when getting the input stream of the uploaded file. */ public InputStream getInputStream() throws Exception { - if (s3Url != null) { - S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket, - DirectUtils.getS3FileKey(s3Url))); - return s3Object.getObjectContent(); + if (externalUrl != null) { + AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl); + if (s3Uri != null) { + S3Object s3Object = DirectUtils.getS3Client().getObject(new GetObjectRequest(s3Bucket, + DirectUtils.getS3FileKey(externalUrl))); + return s3Object.getObjectContent(); + } else { + DefaultHttpClient httpClient = new DefaultHttpClient(); + HttpGet request = new HttpGet(externalUrl); + HttpResponse response = httpClient.execute(request); + // skip status code >=400 + if (response.getStatusLine().getStatusCode() >= HttpStatus.SC_BAD_REQUEST) { + throw new HttpResponseException(response.getStatusLine().getStatusCode(), "Invalid file from external"); + } + + HttpEntity entity = response.getEntity(); + if (entity == null) { + throw new HttpResponseException(HttpStatus.SC_BAD_REQUEST, "Invalid response from external"); + } + return entity.getContent(); + } } if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) { @@ -188,8 +213,12 @@ public InputStream getInputStream() throws Exception { * if any error occurs when getting the file name of the uploaded file. */ public String getContentDisposition() throws Exception { - if (s3Url != null) { - return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(s3Url) + "\""; + if (externalUrl != null) { + AmazonS3URI s3Uri = DirectUtils.getS3Uri(externalUrl); + if (s3Uri != null) { + return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getS3FileKey(externalUrl) + "\""; + } + return "attachment; filename=\"submission-" + submission.getId() + "-" + DirectUtils.getFileNameFromUrl(externalUrl) + "\""; } if (contest.getProjectHeader().getProjectCategory().getId() == ContestType.COPILOT_POSTING.getId()) { diff --git a/src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java b/src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java index 5b9fba888..5a543a28f 100644 --- a/src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java +++ b/src/java/main/com/topcoder/direct/services/view/ajax/SoftwareCompetitionBeanProcessor.java @@ -214,11 +214,11 @@ private Object getMapResult(SoftwareCompetition bean) { // retrieve review scorecard id. for(com.topcoder.project.phases.Phase phase : bean.getProjectPhases().getAllPhases()){ - if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName())){ + if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){ result.put("reviewScorecardId", phase.getAttributes().get("Scorecard ID").toString()); } - if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName())){ + if(phase.getPhaseType().getName().equals(com.topcoder.project.phases.PhaseType.ITERATIVE_REVIEW_PHASE.getName()) && phase.getAttributes().get("Scorecard ID") != null){ result.put("iterativeReviewScorecardId", phase.getAttributes().get("Scorecard ID").toString()); } } diff --git a/src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java b/src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java index e69dc2153..447b9ef21 100644 --- a/src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java +++ b/src/java/main/com/topcoder/direct/services/view/util/DirectUtils.java @@ -5,6 +5,7 @@ import com.amazonaws.auth.PropertiesCredentials; import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3URI; import com.opensymphony.xwork2.ActionContext; import com.topcoder.clients.dao.ProjectContestFeePercentageService; import com.topcoder.clients.dao.ProjectContestFeeService; @@ -4075,9 +4076,40 @@ public static AmazonS3Client getS3Client() { * @throws Exception if any exceptions occurs */ public static String getS3FileKey(String url) throws Exception { + AmazonS3URI s3Uri = getS3Uri(url); + if (s3Uri == null) { + return null; + } + return s3Uri.getKey(); + } + + /** + * Get upload uri from url. + * + * @param url upload url + * @return s3 uri + */ + public static AmazonS3URI getS3Uri(String url) { + try { + AmazonS3URI s3Uri = new AmazonS3URI(url); + return s3Uri; + } catch (IllegalArgumentException ex) { + // url doesn't seem to be a valid + return null; + } + } + + /** + * Get filename from URL + * + * @param url + * @return filename + * @throws Exception + */ + public static String getFileNameFromUrl(String url) throws Exception { String path = new URL(url).getPath(); - int sep = path.lastIndexOf( '/' ); - return ( sep < 0 ) ? path : path.substring( sep + 1 ); + int sep = path.lastIndexOf('/'); + return (sep < 0) ? path : path.substring(sep + 1); } /**