From 969692eb0182f0d401e809614d3aa3e6235c7d49 Mon Sep 17 00:00:00 2001 From: toranger Date: Mon, 1 Apr 2019 20:57:07 +0800 Subject: [PATCH 01/16] Break point and transfer handler, first version --- include/cos_api.h | 46 ++- include/cos_defines.h | 16 +- include/cos_sys_config.h | 2 +- include/op/file_copy_task.h | 2 - include/op/file_download_task.h | 2 - include/op/file_upload_task.h | 25 +- include/op/object_op.h | 26 +- include/request/object_req.h | 11 + include/trsf/transfer_handler.h | 147 +++++++++ include/util/http_sender.h | 30 ++ include/util/simple_mutex.h | 2 +- src/CMakeLists.txt | 4 +- src/cos_api.cpp | 39 ++- src/op/file_upload_task.cpp | 20 +- src/op/object_op.cpp | 549 ++++++++++++++++++++++++++++++-- src/response/object_resp.cpp | 4 +- src/util/http_sender.cpp | 190 +++++++++++ unittest/object_op_test.cpp | 2 + 18 files changed, 1051 insertions(+), 66 deletions(-) create mode 100644 include/trsf/transfer_handler.h diff --git a/include/cos_api.h b/include/cos_api.h index cabab9c..7ceada3 100644 --- a/include/cos_api.h +++ b/include/cos_api.h @@ -6,7 +6,9 @@ #include "op/object_op.h" #include "op/service_op.h" #include "util/simple_mutex.h" + #include "Poco/SharedPtr.h" +#include "trsf/transfer_handler.h" namespace qcloud_cos { @@ -65,6 +67,7 @@ class CosAPI { /// \return 本次请求的调用情况(如状态码等) CosResult PutBucket(const PutBucketReq& request, PutBucketResp* response); + /// \brief 确认Bucket是否存在 /// (详见:https://cloud.tencent.com/document/product/436/7735) /// @@ -331,7 +334,13 @@ class CosAPI { /// /// \return 返回HTTP请求的状态码及错误信息 CosResult MultiUploadObject(const MultiUploadObjectReq& request, - MultiUploadObjectResp* response); + MultiUploadObjectResp* response) ; + + + Poco::SharedPtr TransferUploadObject(const MultiUploadObjectReq& request, + MultiUploadObjectResp* response) ; + + Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) ; /// \brief 舍弃一个分块上传并删除已上传的块 /// 详见: https://www.qcloud.com/document/product/436/7740 @@ -422,5 +431,40 @@ class CosAPI { static int s_cos_obj_num; }; +// Use for trsf the param into boost bind function +class AsynArgs { +public: + AsynArgs(ObjectOp* op) : m_op(op){} + AsynArgs(const AsynArgs& arg){ + this->m_op = arg.m_op; + } + virtual ~AsynArgs(){}; + ObjectOp* m_op; +}; + + +class TransferAsynArgs : public AsynArgs { +public: +TransferAsynArgs(ObjectOp* pObj, + const MultiUploadObjectReq& req, + MultiUploadObjectResp *resp, + Poco::SharedPtr& handler) : AsynArgs(pObj) , m_req(req), m_resp(resp){ + m_handler = handler; + + } + +TransferAsynArgs(const TransferAsynArgs& arg) + : AsynArgs(arg), + m_req(arg.m_req), + m_handler(arg.m_handler), + m_resp(arg.m_resp) { + } + virtual ~TransferAsynArgs() {} +public: + MultiUploadObjectReq m_req; + Poco::SharedPtr m_handler; + MultiUploadObjectResp* m_resp; +}; + } // namespace qcloud_cos #endif diff --git a/include/cos_defines.h b/include/cos_defines.h index 1f907ba..0755556 100644 --- a/include/cos_defines.h +++ b/include/cos_defines.h @@ -27,6 +27,8 @@ const int kMaxThreadPoolSizeUploadPart = 100; /// 分块上传的线程池最小数目 const int kMinThreadPoolSizeUploadPart = 1; +const int kMaxPartNumbers = 10000; + /// 分块大小1M const uint64_t kPartSize1M = 1 * 1024 * 1024; /// 分块大小5G @@ -63,20 +65,20 @@ typedef enum cos_log_level { #define COS_LOW_LOGPRN(level, fmt, ...) \ if (level <= CosSysConfig::GetLogLevel()) { \ if (CosSysConfig::GetLogOutType()== COS_LOG_STDOUT) { \ - fprintf(stdout,"%s:%s(%d) " fmt "%s\n", LOG_LEVEL_STRING(level),__func__,__LINE__, __VA_ARGS__); \ + fprintf(stdout,"%s:%s(%d) " fmt "\n", LOG_LEVEL_STRING(level),__func__,__LINE__, ##__VA_ARGS__); \ }else if (CosSysConfig::GetLogOutType() == COS_LOG_SYSLOG){ \ - syslog(LOG_INFO,"%s:%s(%d) " fmt "%s\n", LOG_LEVEL_STRING(level),__func__,__LINE__, __VA_ARGS__); \ + syslog(LOG_INFO,"%s:%s(%d) " fmt "\n", LOG_LEVEL_STRING(level),__func__,__LINE__, ##__VA_ARGS__); \ } else { \ } \ } else { \ } \ -#define SDK_LOG_DBG(fmt, ...) COS_LOW_LOGPRN(COS_LOG_DBG, fmt, ##__VA_ARGS__, "") -#define SDK_LOG_INFO(fmt, ...) COS_LOW_LOGPRN(COS_LOG_INFO, fmt, ##__VA_ARGS__, "") -#define SDK_LOG_WARN(fmt, ...) COS_LOW_LOGPRN(COS_LOG_WARN, fmt, ##__VA_ARGS__, "") -#define SDK_LOG_ERR(fmt, ...) COS_LOW_LOGPRN(COS_LOG_ERR, fmt, ##__VA_ARGS__, "") -#define SDK_LOG_COS(level, fmt, ...) COS_LOW_LOGPRN(level, fmt, ##__VA_ARGS__, "") +#define SDK_LOG_DBG(fmt, ...) COS_LOW_LOGPRN(COS_LOG_DBG, fmt, ##__VA_ARGS__) +#define SDK_LOG_INFO(fmt, ...) COS_LOW_LOGPRN(COS_LOG_INFO, fmt, ##__VA_ARGS__) +#define SDK_LOG_WARN(fmt, ...) COS_LOW_LOGPRN(COS_LOG_WARN, fmt, ##__VA_ARGS__) +#define SDK_LOG_ERR(fmt, ...) COS_LOW_LOGPRN(COS_LOG_ERR, fmt, ##__VA_ARGS__) +#define SDK_LOG_COS(level, fmt, ...) COS_LOW_LOGPRN(level, fmt, ##__VA_ARGS__) #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) diff --git a/include/cos_sys_config.h b/include/cos_sys_config.h index 4c12a60..db8cf3c 100644 --- a/include/cos_sys_config.h +++ b/include/cos_sys_config.h @@ -1,6 +1,5 @@ #ifndef COS_SYS_CONF_H #define COS_SYS_CONF_H -#include #include #include "cos_defines.h" @@ -139,6 +138,7 @@ class CosSysConfig { static bool m_is_check_md5; static std::string m_dest_domain; + }; } // namespace qcloud_cos diff --git a/include/op/file_copy_task.h b/include/op/file_copy_task.h index 2286ec4..34d9682 100644 --- a/include/op/file_copy_task.h +++ b/include/op/file_copy_task.h @@ -2,8 +2,6 @@ #define FILE_COPY_TASK_H #pragma once -#include - #include #include "cos_config.h" diff --git a/include/op/file_download_task.h b/include/op/file_download_task.h index 9516f83..799e580 100644 --- a/include/op/file_download_task.h +++ b/include/op/file_download_task.h @@ -9,8 +9,6 @@ #define FILE_DOWN_TASK_H #pragma once -#include - #include #include "cos_config.h" diff --git a/include/op/file_upload_task.h b/include/op/file_upload_task.h index 1775599..1e1a9db 100644 --- a/include/op/file_upload_task.h +++ b/include/op/file_upload_task.h @@ -2,8 +2,6 @@ #define FILE_UPLOAD_TASK_H #pragma once -#include - #include #include "cos_config.h" @@ -16,6 +14,9 @@ #include "util/http_sender.h" #include "util/string_util.h" +#include "Poco/SharedPtr.h" +#include "trsf/transfer_handler.h" + namespace qcloud_cos{ class FileUploadTask { @@ -46,6 +47,8 @@ class FileUploadTask { bool IsTaskSuccess() const; + void SetTaskSuccess() { m_is_task_success = true; } + int GetHttpStatus() const ; std::map GetRespHeaders() const; @@ -56,6 +59,21 @@ class FileUploadTask { std::string GetErrMsg() const { return m_err_msg; } + void SetResume(const bool is_resume) { m_is_resume = is_resume; } + + bool IsResume() const { return m_is_resume; } + + void SetHandler(const bool is_handler) { m_is_handler = is_handler; } + + bool IsHandler() const { return m_is_handler; } + + void SetResumeEtag(const std::string& etag) { m_resume_etag = etag; } + + std::string GetResumeEtag() const { return m_resume_etag; } + +public: + Poco::SharedPtr m_handler; + private: std::string m_full_url; std::map m_headers; @@ -69,6 +87,9 @@ class FileUploadTask { int m_http_status; std::map m_resp_headers; std::string m_err_msg; + bool m_is_resume; + std::string m_resume_etag; + bool m_is_handler; }; } diff --git a/include/op/object_op.h b/include/op/object_op.h index 13d6e07..dac9953 100644 --- a/include/op/object_op.h +++ b/include/op/object_op.h @@ -15,6 +15,9 @@ #include "request/object_req.h" #include "response/object_resp.h" +#include "Poco/SharedPtr.h" +#include "trsf/transfer_handler.h" + namespace qcloud_cos { class FileUploadTask; @@ -35,6 +38,14 @@ class ObjectOp : public BaseOp { /// \brief 判断object是否存在 bool IsObjectExist(const std::string& bucket_name, const std::string& object_name); + std::string GetResumableUploadID(const std::string& bucket_name, const std::string& object_name) ; + + bool CheckUploadPart(const MultiUploadObjectReq& req, const std::string& bucket_name, + const std::string& object_name, const std::string& uploadid, + const std::string& localpath, std::vector& already_exist); + + bool check_single_part(const std::string& local_file_path, uint64_t offset, uint64_t local_part_size, + uint64_t size, std::string& etag); /// \brief 获取对应Object的meta信息数据 /// /// \param request HeadObject请求 @@ -140,8 +151,11 @@ class ObjectOp : public BaseOp { /// \param response MultiUploadObject返回 /// /// \return 返回HTTP请求的状态码及错误信息 + CosResult MultiUploadObject(const MultiUploadObjectReq& req, MultiUploadObjectResp* resp, Poco::SharedPtr& handler); + CosResult MultiUploadObject(const MultiUploadObjectReq& req, MultiUploadObjectResp* resp); + Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path); /// \brief 舍弃一个分块上传并删除已上传的块 /// /// \param req AbortMultiUpload请求 @@ -217,7 +231,17 @@ class ObjectOp : public BaseOp { CosResult MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, std::vector* etags_ptr, - std::vector* part_numbers_ptr); + std::vector* part_numbers_ptr, + bool resume_flag, + const std::vector& already_exist_parts, + Poco::SharedPtr& handler); + + CosResult MultiThreadUpload(const MultiUploadObjectReq& req, + const std::string& upload_id, + std::vector* etags_ptr, + std::vector* part_numbers_ptr, + bool resume_flag, + const std::vector& already_exist_parts); // 读取文件内容, 并返回读取的长度 uint64_t GetContent(const std::string& src, std::string* file_content) const; diff --git a/include/request/object_req.h b/include/request/object_req.h index 60d26d4..143119c 100644 --- a/include/request/object_req.h +++ b/include/request/object_req.h @@ -723,12 +723,23 @@ class MultiUploadObjectReq : public ObjectReq { return m_xcos_meta; } + void SetUploadID(const std::string& uploadid) { + if (!uploadid.empty()) { + m_uploadid = uploadid; + } + } + + std::string GetUploadID() const { + return m_uploadid; + } + private: std::string m_local_file_path; uint64_t m_part_size; int m_thread_pool_size; std::map m_xcos_meta; bool mb_set_meta; + std::string m_uploadid; }; class AbortMultiUploadReq : public ObjectReq { diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h new file mode 100644 index 0000000..033ede8 --- /dev/null +++ b/include/trsf/transfer_handler.h @@ -0,0 +1,147 @@ +#ifndef __TRSF_HANDLER_H__ +#define __TRSF_HANDLER_H__ + +#include +#include +#include + +#include "boost/thread/mutex.hpp" +#include "boost/thread/condition_variable.hpp" +#include "Poco/SharedPtr.h" +#include "util/simple_mutex.h" + +#include "op/cos_result.h" + +namespace qcloud_cos{ + class PartState { + public: + PartState(); + PartState(int part_num, std::string& etag, size_t size, bool last_part = false); + + void SetPartNum(int number) { m_part_num = number; } + int GetPartNum() const { return m_part_num; } + + void SetEtag(const std::string& etag) { m_etag = etag; } + std::string GetEtag() const { return m_etag; } + + void SetSize(size_t size) { m_size_inbytes = size; } + size_t GetSize() const { return m_size_inbytes; } + + void SetLastPart(bool lastpart) { m_lastpart = lastpart; } + bool IsLastPart() { return m_lastpart; } + + private: + int m_part_num; + // current use the md5 + std::string m_etag; + + size_t m_size_inbytes; + + // TODO for now just care about the whole progress + /* size_t m_current_progress_inbytes; */ + /* size_t m_range_begin; */ + + bool m_lastpart; + }; + + typedef Poco::SharedPtr PartPointer; + // Key is partnumber + typedef std::map PartStateMap; + + enum class TransferStatus + { + NOT_START, + //Operation is now running + IN_PROGRESS, + //Operation was canceled. + CANCELED, + //Operation failed + FAILED, + //Operation was successful + COMPLETED, + //Operation either failed or was canceled and a user deleted the multi-part upload . + ABORTED + }; + + // For now support the multiupload + class TransferHandler { + public: + // Upload + TransferHandler(const std::string& bucket_name, const std::string& object_name, + uint64_t total_size, const std::string& file_path=""); + + void SetBucketName(const std::string& bucket_name) { m_bucket_name = bucket_name; } + std::string GetBucketName() const { return m_bucket_name; } + + void SetObjectName(const std::string& object_name) { m_object_name = object_name; } + std::string GetObjectName() const { return m_object_name; } + + void SetLocalFilePath(const std::string& local_file_path) { m_local_file_path = local_file_path; } + std::string GetLocalFilePath() const { return m_local_file_path; } + + void SetTotalSize(uint64_t total_size) { m_total_size = total_size; } + uint64_t GetTotalSize() const { return m_total_size; } + + // Notice there can not backwards + void UpdateProgress(uint64_t update_prog); + uint64_t GetProgress() const; + + void UpdateStatus(TransferStatus status); + TransferStatus GetStatus() const; + + void SetUploadID(const std::string& uploadid) { m_uploadid = uploadid; } + std::string GetUploadID() const { return m_uploadid; } + + void Cancel(); + bool ShouldContinue() const; + + bool IsFinishStatus(TransferStatus status) const; + bool IsAllowTransition(TransferStatus org, TransferStatus dst) const; + void WaitUntilFinish(); + + public: + //origin result + CosResult m_result; + + private: + std::string m_bucket_name; + std::string m_object_name; + std::string m_local_file_path; + uint64_t m_total_size; + // The m_current_progress best to use the atomic. but can not support c11 for now, so use the mutex. + uint64_t m_current_progress; + TransferStatus m_status; + std::string m_uploadid; + // Is cancel + bool m_cancel; + + PartStateMap m_part_map; + + // Mutex lock for the progress + /* mutable SimpleMutex m_lock_prog; */ + mutable boost::mutex m_lock_prog; + // Mutex lock for the status + /* mutable SimpleMutex m_lock_stat; */ + mutable boost::mutex m_lock_stat; + + // Condition + mutable boost::condition_variable m_cond; + // Mutex lock for the part map + mutable SimpleMutex m_lock_part; + + + + }; + + class HandleStreamCopier { + public: + static std::streamsize handleCopyStream(std::istream& istr, std::ostream& ostr, + Poco::SharedPtr& handler, std::size_t bufferSize = 8192); + + }; + + + +} + +#endif diff --git a/include/util/http_sender.h b/include/util/http_sender.h index 2129e0f..d03bf1b 100644 --- a/include/util/http_sender.h +++ b/include/util/http_sender.h @@ -17,10 +17,40 @@ #include "request/base_req.h" #include "response/base_resp.h" +#include "Poco/SharedPtr.h" +#include "trsf/transfer_handler.h" + namespace qcloud_cos { class HttpSender { public: + // trsf handler + static int SendRequest(const std::string& http_method, + const std::string& url_str, + const std::map& req_params, + const std::map& req_headers, + const std::string& req_body, + uint64_t conn_timeout_in_ms, + uint64_t recv_timeout_in_ms, + std::map* resp_headers, + std::string* resp_body, + std::string* err_msg, + Poco::SharedPtr& handler, + bool is_check_md5 = false); + // real trsf handler process + static int SendRequest(const std::string& http_method, + const std::string& url_str, + const std::map& req_params, + const std::map& req_headers, + std::istream& is, + uint64_t conn_timeout_in_ms, + uint64_t recv_timeout_in_ms, + std::map* resp_headers, + std::ostream& resp_stream, + std::string* err_msg, + Poco::SharedPtr& handler, + bool is_check_md5 = false); + static int SendRequest(const std::string& http_method, const std::string& url_str, const std::map& req_params, diff --git a/include/util/simple_mutex.h b/include/util/simple_mutex.h index 9a817d8..c35e7a4 100644 --- a/include/util/simple_mutex.h +++ b/include/util/simple_mutex.h @@ -27,7 +27,7 @@ class SimpleMutex { // mutex holder class SimpleMutexLocker { public: - SimpleMutexLocker(SimpleMutex* mutex) : m_mutex(mutex) { + SimpleMutexLocker(SimpleMutex& mutex) : m_mutex(&mutex) { m_mutex->Lock(); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d8298cd..ca172a3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,7 @@ if (OPENSSL_VERSION VERSION_LESS 1.1.0) op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp util/codec_util.cpp util/file_util.cpp util/http_sender.cpp - util/sha1.cpp util/string_util.cpp) + util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) ELSE() message("new version upper than 1.1.0") set(COSSDK_SOURCE_FILES cos_api.cpp cos_config.cpp cos_sys_config.cpp @@ -21,7 +21,7 @@ ELSE() op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp util/codec_util_high_openssl.cpp util/file_util.cpp util/http_sender.cpp - util/sha1.cpp util/string_util.cpp) + util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) ENDIF() add_library(cossdk STATIC ${COSSDK_SOURCE_FILES}) diff --git a/src/cos_api.cpp b/src/cos_api.cpp index 2223e21..1ae5ca1 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -1,7 +1,5 @@ #include "cos_api.h" -#include - #include "threadpool/boost/threadpool.hpp" #include "Poco/Net/HTTPStreamFactory.h" #include "Poco/Net/HTTPSStreamFactory.h" @@ -28,7 +26,7 @@ CosAPI::~CosAPI() { } int CosAPI::CosInit() { - SimpleMutexLocker locker(&s_init_mutex); + SimpleMutexLocker locker(s_init_mutex); ++s_cos_obj_num; if (!s_init) { if (!s_poco_init) { @@ -46,7 +44,7 @@ int CosAPI::CosInit() { } void CosAPI::CosUInit() { - SimpleMutexLocker locker(&s_init_mutex); + SimpleMutexLocker locker(s_init_mutex); --s_cos_obj_num; if (s_init && s_cos_obj_num == 0) { if (g_threadpool){ @@ -252,11 +250,44 @@ CosResult CosAPI::CompleteMultiUpload(const CompleteMultiUploadReq& request, return m_object_op.CompleteMultiUpload(request, response); } +void TransferSubmit(TransferAsynArgs args) { + ObjectOp *op = args.m_op; + MultiUploadObjectReq req = args.m_req; + MultiUploadObjectResp* resp = args.m_resp; + Poco::SharedPtr handler = args.m_handler; + + // task is m_object_op.MultiUploadObject(request, response, handler); + op->MultiUploadObject(req, resp, handler); +} + +// Async to transfer +Poco::SharedPtr CosAPI::TransferUploadObject(const MultiUploadObjectReq& request, + MultiUploadObjectResp* response) { + // create the handler + Poco::SharedPtr handler = CreateUploadHandler(request.GetBucketName(), request.GetObjectName(), + request.GetLocalFilePath()); + TransferAsynArgs args(&m_object_op, request, response, handler); + + // use the cos's boost thread pool to submit the task + if(g_threadpool) { + g_threadpool->schedule(boost::bind(&TransferSubmit, args)); + }else { + handler->UpdateStatus(TransferStatus::FAILED); + } + // return the handler outside. + return handler; +} + CosResult CosAPI::MultiUploadObject(const MultiUploadObjectReq& request, MultiUploadObjectResp* response) { + return m_object_op.MultiUploadObject(request, response); } +Poco::SharedPtr CosAPI::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) { + return m_object_op.CreateUploadHandler(bucket_name, object_name, local_path); +} + CosResult CosAPI::AbortMultiUpload(const AbortMultiUploadReq& request, AbortMultiUploadResp* response) { return m_object_op.AbortMultiUpload(request, response); diff --git a/src/op/file_upload_task.cpp b/src/op/file_upload_task.cpp index 5055b4a..df5013a 100644 --- a/src/op/file_upload_task.cpp +++ b/src/op/file_upload_task.cpp @@ -21,7 +21,7 @@ FileUploadTask::FileUploadTask(const std::string& full_url, const size_t data_len) : m_full_url(full_url), m_data_buf_ptr(pbuf), m_data_len(data_len), m_conn_timeout_in_ms(conn_timeout_in_ms), m_recv_timeout_in_ms(recv_timeout_in_ms), - m_resp(""), m_is_task_success(false) { + m_resp(""), m_is_task_success(false), m_is_resume(false), m_is_handler(false) { } FileUploadTask::FileUploadTask(const std::string& full_url, @@ -33,7 +33,8 @@ FileUploadTask::FileUploadTask(const std::string& full_url, const size_t data_len) : m_full_url(full_url), m_headers(headers), m_params(params), m_conn_timeout_in_ms(conn_timeout_in_ms), m_recv_timeout_in_ms(recv_timeout_in_ms), - m_data_buf_ptr(pbuf), m_data_len(data_len), m_resp(""), m_is_task_success(false) { + m_data_buf_ptr(pbuf), m_data_len(data_len), m_resp(""), m_is_task_success(false), + m_is_resume(false), m_is_handler(false) { } void FileUploadTask::Run() { @@ -89,9 +90,16 @@ void FileUploadTask::UploadTask() { loop++; m_resp_headers.clear(); m_resp = ""; - m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, - body, m_conn_timeout_in_ms, m_recv_timeout_in_ms, - &m_resp_headers, &m_resp, &m_err_msg); + + if(IsHandler()) { + m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, + body, m_conn_timeout_in_ms, m_recv_timeout_in_ms, + &m_resp_headers, &m_resp, &m_err_msg, m_handler); + }else { + m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, + body, m_conn_timeout_in_ms, m_recv_timeout_in_ms, + &m_resp_headers, &m_resp, &m_err_msg); + } if (m_http_status != 200) { SDK_LOG_ERR("FileUpload: url(%s) fail, httpcode:%d, resp: %s", @@ -109,7 +117,7 @@ void FileUploadTask::UploadTask() { } m_is_task_success = true; - } while (!m_is_task_success && loop <= kMaxRetryTimes); + } while (!m_is_task_success && loop <= kMaxRetryTimes && m_handler->ShouldContinue()); return; } diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 7a9bd3f..efa96df 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -6,6 +6,7 @@ // Description: #include "op/object_op.h" +#include "op/bucket_op.h" #include #include @@ -25,6 +26,9 @@ #include "util/http_sender.h" #include "util/string_util.h" +#include "request/bucket_req.h" +#include "response/bucket_resp.h" + #include "Poco/MD5Engine.h" #include "Poco/DigestStream.h" #include "Poco/StreamCopier.h" @@ -42,6 +46,145 @@ bool ObjectOp::IsObjectExist(const std::string& bucket_name, const std::string& return false; } +std::string ObjectOp::GetResumableUploadID(const std::string& bucket_name, const std::string& object_name) { + ListMultipartUploadReq req(bucket_name); + req.SetPrefix(object_name); + ListMultipartUploadResp resp; + + std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), + req.GetBucketName()); + std::string path = req.GetPath(); + CosResult result = NormalAction(host, path, req, "", false, &resp); + + std::vector rst = resp.GetUpload(); + int index = rst.size() - 1; + while (index >= 0) { + if (rst[index].m_key == object_name) { + return rst[index].m_uploadid; + } + index--; + } + return ""; +} + +bool ObjectOp::check_single_part(const std::string& local_file_path, uint64_t offset, uint64_t local_part_size, + uint64_t size, std::string& etag) { + if (local_part_size != size) { + return false; + } + + std::ifstream fin(local_file_path.c_str(), std::ios::in | std::ios::binary); + if (!fin.is_open()) { + SDK_LOG_ERR("CheckUploadPart: file open fail, %s", local_file_path.c_str()); + return false; + } + + fin.seekg (offset); + + // allocate memory: + char *data = new char[local_part_size]; + + // read data as a block: + fin.read (data,local_part_size); + std::cout << "gount is :" << fin.gcount() << std::endl; + + fin.seekg (0, fin.beg); + fin.close(); + + // print content: + std::istringstream stringStream(std::string(data, local_part_size)); + // std::istringstream stringStream(data); + + Poco::MD5Engine md5; + Poco::DigestOutputStream dos(md5); + Poco::StreamCopier::copyStream(stringStream, dos); + + // dos << stringStream.str(); + dos.flush(); + + std::string md5_str = Poco::DigestEngine::digestToHex(md5.digest()); + + delete []data; + dos.close(); + + if (md5_str != etag) { + return false; + } + return true; +} + // TODO tmd +bool ObjectOp::CheckUploadPart(const MultiUploadObjectReq& req, const std::string& bucket_name, + const std::string& object_name, const std::string& uploadid, + const std::string& localpath, std::vector& already_exist) { + // Count the size info + std::string local_file_path = req.GetLocalFilePath(); + std::ifstream fin(local_file_path.c_str(), std::ios::in | std::ios::binary); + if (!fin.is_open()){ + SDK_LOG_ERR("CheckUploadPart: file open fail, %s", local_file_path.c_str()); + return false; + } + uint64_t file_size = FileUtil::GetFileLen(local_file_path); + uint64_t part_size = req.GetPartSize(); + uint64_t part_num = file_size / part_size; + uint64_t last_part_size = file_size % part_size; + + if (0 != last_part_size) { + part_num += 1; + } else { + last_part_size = part_size; + } + if (part_num > kMaxPartNumbers) { + return false; + } + + ListPartsReq list_req(bucket_name, object_name, uploadid); + ListPartsResp resp; + int part_num_marker = 0; + bool list_over_flag = false; + + std::vector parts_info; + + while (!list_over_flag) { + std::string marker = StringUtil::IntToString(part_num_marker); + list_req.SetPartNumberMarker(marker); + CosResult result = ListParts(list_req, &resp); + // Add to the parts_info; + std::vector rst = resp.GetParts(); + for (std::vector::const_iterator itr = rst.begin(); itr != rst.end(); ++itr) { + parts_info.push_back(*itr); + } + + if (!resp.IsTruncated()) { + list_over_flag = true; + }else { + part_num_marker = int(resp.GetNextPartNumberMarker()); + } + } + + for (std::vector::const_iterator itr = parts_info.begin(); itr != parts_info.end(); ++itr) { + uint64_t sev_part_num = itr->m_part_num; + if (sev_part_num > part_num) { + return false; + } + uint64_t offset = (sev_part_num - 1) * part_size; + uint64_t local_part_size = part_size; + if (sev_part_num == part_num) { + local_part_size = last_part_size; + } + + // check single upload part each md5 + std::string etag = itr->m_etag; + if (!check_single_part(local_file_path, offset, local_part_size, itr->m_size, etag)) { + return false; + } + + // add the part_num with etags in already exist + already_exist[sev_part_num] = itr->m_etag; + } + + return true; +} + CosResult ObjectOp::HeadObject(const HeadObjectReq& req, HeadObjectResp* resp) { std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), req.GetBucketName()); @@ -208,6 +351,8 @@ CosResult ObjectOp::DeleteObjects(const DeleteObjectsReq& req, DeleteObjectsResp additional_params, req_body, false, resp); } + +// Origin call CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, MultiUploadObjectResp* resp) { CosResult result; @@ -216,66 +361,188 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, std::string object_name = req.GetObjectName(); std::string local_file_path = req.GetLocalFilePath(); - std::ifstream fin(local_file_path.c_str() , std::ios::in); - if (!fin) { - result.SetErrorInfo("Open local file fail, local file=" + local_file_path); - return result; + bool resume_flag = false; + // There is mem or cpu problem, if use the red-black tree might be slow + std::vector already_exist_parts(kMaxPartNumbers); + // check the breakpoint + std::string resume_uploadid = GetResumableUploadID(bucket_name, object_name); + if (!resume_uploadid.empty()) { + resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, local_file_path, already_exist_parts); } - // 1. Init - InitMultiUploadReq init_req(bucket_name, object_name); - const std::string& server_side_encryption = req.GetHeader("x-cos-server-side-encryption"); - if (!server_side_encryption.empty()) { - init_req.SetXCosServerSideEncryption(server_side_encryption); - } + if (!resume_flag) { + // 1. Init + InitMultiUploadReq init_req(bucket_name, object_name); + const std::string& server_side_encryption = req.GetHeader("x-cos-server-side-encryption"); + if (!server_side_encryption.empty()) { + init_req.SetXCosServerSideEncryption(server_side_encryption); + } + + if (req.IsSetXCosMeta()) { + const std::map xcos_meta = req.GetXCosMeta(); + std::map::const_iterator iter = xcos_meta.begin(); + for(; iter != xcos_meta.end(); iter++) { + init_req.SetXCosMeta(iter->first, iter->second); + } + } - if (req.IsSetXCosMeta()) { - const std::map xcos_meta = req.GetXCosMeta(); - std::map::const_iterator iter = xcos_meta.begin(); - for(; iter != xcos_meta.end(); iter++) { - init_req.SetXCosMeta(iter->first, iter->second); + InitMultiUploadResp init_resp; + init_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); + init_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms()); + result = InitMultiUpload(init_req, &init_resp); + if (!result.IsSucc()) { + SDK_LOG_ERR("Multi upload object fail, check init mutli result."); + resp->CopyFrom(init_resp); + return result; + } + resume_uploadid = init_resp.GetUploadId(); + if (resume_uploadid.empty()) { + SDK_LOG_ERR("Multi upload object fail, upload id is empty."); + resp->CopyFrom(init_resp); + return result; } } - InitMultiUploadResp init_resp; - init_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); - init_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms()); - result = InitMultiUpload(init_req, &init_resp); + // 2. Multi Upload + std::vector etags; + std::vector part_numbers; + // TODO(返回值判断), add the already exist parts + result = MultiThreadUpload(req, resume_uploadid, &etags, &part_numbers, resume_flag, already_exist_parts); if (!result.IsSucc()) { - SDK_LOG_ERR("Multi upload object fail, check init mutli result."); - resp->CopyFrom(init_resp); + SDK_LOG_ERR("Multi upload object fail, check upload mutli result."); + // Copy失败则需要Abort + AbortMultiUploadReq abort_req(req.GetBucketName(), + req.GetObjectName(), resume_uploadid); + AbortMultiUploadResp abort_resp; + + CosResult abort_result = AbortMultiUpload(abort_req, &abort_resp); + if (!abort_result.IsSucc()) { + SDK_LOG_ERR("Upload failed, and abort muliti upload also failed" + ", resume_uploadid=%s", resume_uploadid.c_str()); + return abort_result; + } return result; } - std::string upload_id = init_resp.GetUploadId(); - if (upload_id.empty()) { - SDK_LOG_ERR("Multi upload object fail, upload id is empty."); - resp->CopyFrom(init_resp); - return result; + + // 3. Complete + CompleteMultiUploadReq comp_req(bucket_name, object_name, resume_uploadid); + CompleteMultiUploadResp comp_resp; + comp_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); + comp_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms() * 2); // Complete的超时翻倍 + comp_req.SetEtags(etags); + comp_req.SetPartNumbers(part_numbers); + + result = CompleteMultiUpload(comp_req, &comp_resp); + resp->CopyFrom(comp_resp); + + return result; +} + +// Create the handler +Poco::SharedPtr ObjectOp::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) { + TransferHandler *p = new TransferHandler(bucket_name, object_name, 0, local_path); + Poco::SharedPtr handler(p); + + uint64_t file_size = FileUtil::GetFileLen(local_path); + + handler->SetTotalSize(file_size); + return handler; +} + +// Transfer call +CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, + MultiUploadObjectResp* resp, + Poco::SharedPtr& handler) { + CosResult result; + uint64_t app_id = GetAppId(); + std::string bucket_name = req.GetBucketName(); + std::string object_name = req.GetObjectName(); + std::string local_file_path = req.GetLocalFilePath(); + + bool resume_flag = false; + // There is mem or cpu problem, if use the red-black tree might be slow + std::vector already_exist_parts(kMaxPartNumbers); + // check the breakpoint + std::string resume_uploadid = GetResumableUploadID(bucket_name, object_name); + if (!resume_uploadid.empty()) { + resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, local_file_path, already_exist_parts); } + if (!resume_flag) { + // 1. Init + InitMultiUploadReq init_req(bucket_name, object_name); + const std::string& server_side_encryption = req.GetHeader("x-cos-server-side-encryption"); + if (!server_side_encryption.empty()) { + init_req.SetXCosServerSideEncryption(server_side_encryption); + } + + if (req.IsSetXCosMeta()) { + const std::map xcos_meta = req.GetXCosMeta(); + std::map::const_iterator iter = xcos_meta.begin(); + for(; iter != xcos_meta.end(); iter++) { + init_req.SetXCosMeta(iter->first, iter->second); + } + } + + InitMultiUploadResp init_resp; + init_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); + init_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms()); + result = InitMultiUpload(init_req, &init_resp); + if (!result.IsSucc()) { + SDK_LOG_ERR("Multi upload object fail, check init mutli result."); + resp->CopyFrom(init_resp); + handler->UpdateStatus(TransferStatus::FAILED); + handler->m_result = result; + return result; + } + resume_uploadid = init_resp.GetUploadId(); + if (resume_uploadid.empty()) { + SDK_LOG_ERR("Multi upload object fail, upload id is empty."); + resp->CopyFrom(init_resp); + handler->UpdateStatus(TransferStatus::FAILED); + handler->m_result = result; + return result; + } + } + SDK_LOG_INFO("Multi upload object handler way id:%s, resumed:%d, already exist number:%d", resume_uploadid.c_str(), resume_flag); + // 2. Multi Upload std::vector etags; std::vector part_numbers; - // TODO(返回值判断) - result = MultiThreadUpload(req, upload_id, &etags, &part_numbers); + // TODO(返回值判断), add the already exist parts + handler->SetUploadID(resume_uploadid); + handler->UpdateStatus(TransferStatus::IN_PROGRESS); + + result = MultiThreadUpload(req, resume_uploadid, &etags, &part_numbers, resume_flag, already_exist_parts, handler); + if (!handler->ShouldContinue()) { + SDK_LOG_INFO("Multi upload object, canceled"); + handler->UpdateStatus(TransferStatus::CANCELED); + return result; + } + + // Notice the cancel way not need to abort the uploadid if (!result.IsSucc()) { SDK_LOG_ERR("Multi upload object fail, check upload mutli result."); // Copy失败则需要Abort AbortMultiUploadReq abort_req(req.GetBucketName(), - req.GetObjectName(), upload_id); + req.GetObjectName(), resume_uploadid); AbortMultiUploadResp abort_resp; CosResult abort_result = AbortMultiUpload(abort_req, &abort_resp); if (!abort_result.IsSucc()) { SDK_LOG_ERR("Upload failed, and abort muliti upload also failed" - ", upload_id=%s", upload_id.c_str()); + ", resume_uploadid=%s", resume_uploadid.c_str()); + handler->UpdateStatus(TransferStatus::FAILED); + handler->m_result = abort_result; return abort_result; } + handler->UpdateStatus(TransferStatus::ABORTED); + handler->m_result = result; return result; } // 3. Complete - CompleteMultiUploadReq comp_req(bucket_name, object_name, upload_id); + CompleteMultiUploadReq comp_req(bucket_name, object_name, resume_uploadid); CompleteMultiUploadResp comp_resp; comp_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); comp_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms() * 2); // Complete的超时翻倍 @@ -283,7 +550,13 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, comp_req.SetPartNumbers(part_numbers); result = CompleteMultiUpload(comp_req, &comp_resp); + if (!result.IsSucc()) { + handler->UpdateStatus(TransferStatus::FAILED); + }else { + handler->UpdateStatus(TransferStatus::COMPLETED); + } resp->CopyFrom(comp_resp); + handler->m_result = result; return result; } @@ -853,10 +1126,174 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb } // TODO(sevenyou) 多线程上传, 返回的resp内容需要再斟酌下. + // origin way +CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, + const std::string& upload_id, + std::vector* etags_ptr, + std::vector* part_numbers_ptr, + bool resume_flag, + const std::vector& already_exist_parts) { + CosResult result; + std::string path = "/" + req.GetObjectName(); + std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), + req.GetBucketName()); + + // 1. 获取文件大小 + std::string local_file_path = req.GetLocalFilePath(); + std::ifstream fin(local_file_path.c_str(), std::ios::in | std::ios::binary); + if (!fin.is_open()){ + SDK_LOG_ERR("FileUploadSliceData: file open fail, %s", local_file_path.c_str()); + result.SetErrorInfo("local file not exist, local_file=" + local_file_path); + return result; + } + uint64_t file_size = FileUtil::GetFileLen(local_file_path); + + // 2. 初始化upload task + uint64_t offset = 0; + bool task_fail_flag = false; + + uint64_t part_size = req.GetPartSize(); + int pool_size = req.GetThreadPoolSize(); + + // Check the part number + uint64_t part_number = file_size / part_size; + uint64_t last_part_size = file_size % part_size; + if (0 != last_part_size) { + part_number += 1; + }else { + last_part_size = part_size; // for now not use this. + } + + if (part_number > kMaxPartNumbers) { + SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %d", part_number); + result.SetErrorInfo("part number bigger than 10000"); + return result; + } + + + unsigned char** file_content_buf = new unsigned char*[pool_size]; + for(int i = 0; i < pool_size; ++i) { + file_content_buf[i] = new unsigned char[part_size]; + } + + std::string dest_url = GetRealUrl(host, path, req.IsHttps()); + FileUploadTask** pptaskArr = new FileUploadTask*[pool_size]; + for (int i = 0; i < pool_size; ++i) { + pptaskArr[i] = new FileUploadTask(dest_url, req.GetConnTimeoutInms(), req.GetRecvTimeoutInms()); + } + + SDK_LOG_DBG("upload data,url=%s, poolsize=%u, part_size=%lu, file_size=%lu", + dest_url.c_str(), pool_size, part_size, file_size); + + boost::threadpool::pool tp(pool_size); + + // 3. 多线程upload + { + uint64_t part_number = 1; + while (offset < file_size) { + int task_index = 0; + for (; task_index < pool_size; ++task_index) { + fin.read((char *)file_content_buf[task_index], part_size); + size_t read_len = fin.gcount(); + if (read_len == 0 && fin.eof()) { + SDK_LOG_DBG("read over, task_index: %d", task_index); + break; + } + + SDK_LOG_DBG("upload data, task_index=%d, file_size=%lu, offset=%lu, len=%lu", + task_index, file_size, offset, read_len); + + // Check the resume + FileUploadTask* ptask = pptaskArr[task_index]; + if (resume_flag && !already_exist_parts[part_number].empty()) { + // Already has this part + ptask->SetResume(resume_flag); + ptask->SetResumeEtag(already_exist_parts[part_number]); + std::cout << "task etag is" << already_exist_parts[part_number] << std::endl; + ptask->SetTaskSuccess(); + SDK_LOG_INFO("upload data part:%d has resumed", part_number); + + }else { + FillUploadTask(upload_id, host, path, file_content_buf[task_index], read_len, + part_number, ptask); + tp.schedule(boost::bind(&FileUploadTask::Run, ptask)); + } + + offset += read_len; + part_numbers_ptr->push_back(part_number); + ++part_number; + } + + int max_task_num = task_index; + + tp.wait(); + for (task_index = 0; task_index < max_task_num; ++task_index) { + FileUploadTask* ptask = pptaskArr[task_index]; + if (!ptask->IsTaskSuccess()) { + const std::string& task_resp = ptask->GetTaskResp(); + const std::map& task_resp_headers = ptask->GetRespHeaders(); + SDK_LOG_ERR("upload data, upload task fail, rsp:%s", task_resp.c_str()); + result.SetHttpStatus(ptask->GetHttpStatus()); + if (ptask->GetHttpStatus() == -1) { + result.SetErrorInfo(ptask->GetErrMsg()); + } else if (!result.ParseFromHttpResponse(task_resp_headers, task_resp)) { + result.SetErrorInfo(task_resp); + } + + task_fail_flag = true; + break; + } + + // 找不到etag也算失败 + const std::map& resp_header = ptask->GetRespHeaders(); + std::map::const_iterator itr = resp_header.find("ETag"); + if (itr != resp_header.end()) { + etags_ptr->push_back(itr->second); + } else if (ptask->IsResume() && !ptask->GetResumeEtag().empty()) { + etags_ptr->push_back(ptask->GetResumeEtag()); + } else { + std::string err_info = "upload data, upload task succ, " + "but response header missing etag field."; + SDK_LOG_ERR("%s", err_info.c_str()); + result.SetHttpStatus(ptask->GetHttpStatus()); + task_fail_flag = true; + break; + } + } + + if (task_fail_flag) { + break; + } + } + } + + if (!task_fail_flag) { + result.SetSucc(); + } + + // 释放相关资源 + fin.close(); + for (int i = 0; i< pool_size; ++i) { + delete pptaskArr[i]; + } + delete [] pptaskArr; + + for (int i = 0; i < pool_size; ++i) { + delete [] file_content_buf[i]; + } + delete [] file_content_buf; + + return result; +} + +//Trsf way CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, std::vector* etags_ptr, - std::vector* part_numbers_ptr) { + std::vector* part_numbers_ptr, + bool resume_flag, + const std::vector& already_exist_parts, + Poco::SharedPtr& handler) { CosResult result; std::string path = "/" + req.GetObjectName(); std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), @@ -878,6 +1315,23 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, uint64_t part_size = req.GetPartSize(); int pool_size = req.GetThreadPoolSize(); + + // Check the part number + uint64_t part_number = file_size / part_size; + uint64_t last_part_size = file_size % part_size; + if (0 != last_part_size) { + part_number += 1; + }else { + last_part_size = part_size; // for now not use this. + } + + if (part_number > kMaxPartNumbers) { + SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %d", part_number); + result.SetErrorInfo("part number bigger than 10000"); + return result; + } + + unsigned char** file_content_buf = new unsigned char*[pool_size]; for(int i = 0; i < pool_size; ++i) { file_content_buf[i] = new unsigned char[part_size]; @@ -899,6 +1353,12 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, uint64_t part_number = 1; while (offset < file_size) { int task_index = 0; + if (!handler->ShouldContinue()) { + task_fail_flag = true; + result.SetErrorInfo("FileUpload handler canceled"); + break; + } + for (; task_index < pool_size; ++task_index) { fin.read((char *)file_content_buf[task_index], part_size); size_t read_len = fin.gcount(); @@ -910,10 +1370,27 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, SDK_LOG_DBG("upload data, task_index=%d, file_size=%lu, offset=%lu, len=%lu", task_index, file_size, offset, read_len); + // Check the resume FileUploadTask* ptask = pptaskArr[task_index]; - FillUploadTask(upload_id, host, path, file_content_buf[task_index], read_len, - part_number, ptask); - tp.schedule(boost::bind(&FileUploadTask::Run, ptask)); + + ptask->SetHandler(true); + + if (resume_flag && !already_exist_parts[part_number].empty()) { + // Already has this part + ptask->SetResume(resume_flag); + ptask->SetResumeEtag(already_exist_parts[part_number]); + std::cout << "task etag is" << already_exist_parts[part_number] << std::endl; + ptask->SetTaskSuccess(); + SDK_LOG_INFO("upload data part:%d has resumed", part_number); + + handler->UpdateProgress(read_len); + }else { + ptask->m_handler = handler; + FillUploadTask(upload_id, host, path, file_content_buf[task_index], read_len, + part_number, ptask); + tp.schedule(boost::bind(&FileUploadTask::Run, ptask)); + } + offset += read_len; part_numbers_ptr->push_back(part_number); ++part_number; @@ -944,6 +1421,8 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, std::map::const_iterator itr = resp_header.find("ETag"); if (itr != resp_header.end()) { etags_ptr->push_back(itr->second); + } else if (ptask->IsResume() && !ptask->GetResumeEtag().empty()) { + etags_ptr->push_back(ptask->GetResumeEtag()); } else { std::string err_info = "upload data, upload task succ, " "but response header missing etag field."; diff --git a/src/response/object_resp.cpp b/src/response/object_resp.cpp index 6044a8c..a67aa26 100644 --- a/src/response/object_resp.cpp +++ b/src/response/object_resp.cpp @@ -188,7 +188,7 @@ bool ListPartsResp::ParseFromXmlString(const std::string& body) { const std::string& init_node_name = init_node->name(); if ("ID" == init_node_name) { m_initiator.m_id = init_node->value(); - } else if ("DisplyName" == init_node_name) { + } else if ("DisplayName" == init_node_name) { m_initiator.m_display_name = init_node->value(); } else { SDK_LOG_WARN("Unknown field in Initiator node, field_name=%s", @@ -201,7 +201,7 @@ bool ListPartsResp::ParseFromXmlString(const std::string& body) { const std::string& owner_node_name = owner_node->name(); if ("ID" == owner_node_name) { m_owner.m_id = owner_node->value(); - } else if ("DisplyName" == owner_node_name) { + } else if ("DisplayName" == owner_node_name) { m_owner.m_display_name = owner_node->value(); } else { SDK_LOG_WARN("Unknown field in Owner node, field_name=%s", diff --git a/src/util/http_sender.cpp b/src/util/http_sender.cpp index f797678..a58bb10 100644 --- a/src/util/http_sender.cpp +++ b/src/util/http_sender.cpp @@ -31,6 +31,38 @@ namespace qcloud_cos { +// Trsf handler +int HttpSender::SendRequest(const std::string& http_method, + const std::string& url_str, + const std::map& req_params, + const std::map& req_headers, + const std::string& req_body, + uint64_t conn_timeout_in_ms, + uint64_t recv_timeout_in_ms, + std::map* resp_headers, + std::string* resp_body, + std::string* err_msg, + Poco::SharedPtr& handler, + bool is_check_md5) { + std::istringstream is(req_body); + std::ostringstream oss; + int ret = SendRequest(http_method, + url_str, + req_params, + req_headers, + is, + conn_timeout_in_ms, + recv_timeout_in_ms, + resp_headers, + oss, + err_msg, + handler, + is_check_md5); + *resp_body = oss.str(); + return ret; +} + + int HttpSender::SendRequest(const std::string& http_method, const std::string& url_str, const std::map& req_params, @@ -411,6 +443,164 @@ int HttpSender::SendRequest(const std::string& http_method, return res.getStatus(); } + +// Real trsf handler process +int HttpSender::SendRequest(const std::string& http_method, + const std::string& url_str, + const std::map& req_params, + const std::map& req_headers, + std::istream& is, + uint64_t conn_timeout_in_ms, + uint64_t recv_timeout_in_ms, + std::map* resp_headers, + std::ostream& resp_stream, + std::string* err_msg, + Poco::SharedPtr& handler, + bool is_check_md5) { + Poco::Net::HTTPResponse res; + try { + Poco::URI url(url_str); + boost::scoped_ptr session; + if (StringUtil::StringStartsWithIgnoreCase(url_str, "https")) { + Poco::Net::Context::Ptr context = new Poco::Net::Context(Poco::Net::Context::CLIENT_USE, + "", "", "", Poco::Net::Context::VERIFY_RELAXED, + 9, true, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"); + session.reset(new Poco::Net::HTTPSClientSession(url.getHost(), url.getPort(), context)); + } else { + session.reset(new Poco::Net::HTTPClientSession(url.getHost(), url.getPort())); + } + + session->setTimeout(Poco::Timespan(0, conn_timeout_in_ms * 1000)); + + // 1. 拼接path_query字符串 + std::string path = url.getPath(); + if (path.empty()) { + path += "/"; + } + + std::string query_str; + for (std::map::const_iterator c_itr = req_params.begin(); + c_itr != req_params.end(); ++c_itr) { + std::string part; + if (c_itr->second.empty()) { + part = CodecUtil::UrlEncode(c_itr->first) + "&"; + } else { + part = CodecUtil::UrlEncode(c_itr->first) + "=" + CodecUtil::UrlEncode(c_itr->second) + "&"; + } + query_str += part; + } + + if (!query_str.empty()) { + query_str = "?" + query_str.substr(0, query_str.size() - 1); + } + std::string path_and_query_str = CodecUtil::EncodeKey(path) + query_str; + + // 2. 创建http request, 并填充头部 + Poco::Net::HTTPRequest req(http_method, path_and_query_str, Poco::Net::HTTPMessage::HTTP_1_1); + for (std::map::const_iterator c_itr = req_headers.begin(); + c_itr != req_headers.end(); ++c_itr) { + req.add(c_itr->first, (c_itr->second).c_str()); + } + + // 3. 计算长度 + std::streampos pos = is.tellg(); + is.seekg(0, std::ios::end); + req.setContentLength(is.tellg()); + is.seekg(pos); + +#ifdef __COS_DEBUG__ + std::ostringstream debug_os; + req.write(debug_os); + SDK_LOG_DBG("request=[%s]", debug_os.str().c_str()); +#endif + + // 4. 发送请求 + std::ostream& os = session->sendRequest(req); + // According to the copyStream to handle the process + // TODO overwrite the copyStream insider to record + // Poco::StreamCopier::copyStream(is, os); + HandleStreamCopier::handleCopyStream(is, os, handler); + + // 5. 接收返回 + Poco::Net::StreamSocket& ss = session->socket(); + ss.setReceiveTimeout(Poco::Timespan(0, recv_timeout_in_ms * 1000)); + std::istream& recv_stream = session->receiveResponse(res); + + // 6. 处理返回 + int ret = res.getStatus(); + resp_headers->insert(res.begin(), res.end()); + + std::string etag = ""; + std::map::const_iterator etag_itr + = resp_headers->find("ETag"); + if (etag_itr != resp_headers->end()) { + etag = StringUtil::Trim(etag_itr->second, "\""); + } + + if (is_check_md5 && !StringUtil::IsV4ETag(etag) + && !StringUtil::IsMultipartUploadETag(etag)) { + SDK_LOG_DBG("Check Response Md5"); + Poco::MD5Engine md5; + Poco::DigestOutputStream dos(md5); + + // explicit iostream (streambuf* sb); + std::stringbuf ibuf; + std::iostream io_tmp(&ibuf); + + // The Poco session->receiveResponse return the streambuf which dose not overload the base_iostream seekpos which is the realization of the tellg and seekg. + // It casue the recv_stream can not relocation the begin postion, so can not reuse of the recv_stream. + // FIXME it might has property issue. + Poco::StreamCopier::copyStream(recv_stream, io_tmp); + + std::streampos pos = io_tmp.tellg(); + Poco::StreamCopier::copyStream(io_tmp, dos); + io_tmp.clear(); + io_tmp.seekg(pos); + dos.close(); + + std::string md5_str = Poco::DigestEngine::digestToHex(md5.digest()); + + if (etag != md5_str) { + *err_msg = "Md5 of response body is not equal to the etag in the header." + " Body Md5= " + md5_str + ", etag=" + etag; + SDK_LOG_ERR("Check Md5 fail, %s", err_msg->c_str()); + ret = -1; + } + Poco::StreamCopier::copyStream(io_tmp, resp_stream); + }else { + Poco::StreamCopier::copyStream(recv_stream, resp_stream); + } + +#ifdef __COS_DEBUG__ + SDK_LOG_DBG("response header :\n"); + for (std::map::const_iterator itr = resp_headers->begin(); + itr != resp_headers->end(); ++itr) { + SDK_LOG_DBG("key=[%s], value=[%s]\n", itr->first.c_str(), itr->second.c_str()); + } +#endif + SDK_LOG_INFO("Send request over, status=%d, reason=%s", + res.getStatus(), res.getReason().c_str()); + return ret; + } catch (Poco::Net::NetException& ex){ + SDK_LOG_ERR("Net Exception:%s", ex.displayText().c_str()); + *err_msg = "Net Exception:" + ex.displayText(); + return -1; + } catch (Poco::TimeoutException& ex) { + SDK_LOG_ERR("TimeoutException:%s", ex.displayText().c_str()); + *err_msg = "TimeoutException:" + ex.displayText(); + return -1; + } catch (Poco::AssertionViolationException& ex) { + // handle the cancel way or other violation exception + *err_msg = "AssertionViolationException:%s" + ex.displayText(); + return -1; + } catch (const std::exception &ex) { + SDK_LOG_ERR("Exception:%s, errno=%d", std::string(ex.what()).c_str(), errno); + *err_msg = "Exception:" + std::string(ex.what()); + return -1; + } + + return res.getStatus(); +} // TODO(sevenyou) 挪走 uint64_t HttpSender::GetTimeStampInUs() { // 构造时间 diff --git a/unittest/object_op_test.cpp b/unittest/object_op_test.cpp index 361e684..e0c4264 100644 --- a/unittest/object_op_test.cpp +++ b/unittest/object_op_test.cpp @@ -9,6 +9,7 @@ #include "common.h" #include "cos_api.h" +#include "trsf/transfer_handler.h" namespace qcloud_cos { @@ -464,6 +465,7 @@ TEST_F(ObjectOpTest, MultiUploadObjectTest_OneStep) { MultiUploadObjectReq req(m_bucket_name, object_name, filename); MultiUploadObjectResp resp; + CosResult result = m_client->MultiUploadObject(req, &resp); EXPECT_TRUE(result.IsSucc()); From 7d29f5caea65288878dbc69d66081ef981487b2a Mon Sep 17 00:00:00 2001 From: toranger Date: Mon, 1 Apr 2019 21:00:22 +0800 Subject: [PATCH 02/16] add lost file --- src/trsf/transfer_handler.cpp | 143 ++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 src/trsf/transfer_handler.cpp diff --git a/src/trsf/transfer_handler.cpp b/src/trsf/transfer_handler.cpp new file mode 100644 index 0000000..43b2743 --- /dev/null +++ b/src/trsf/transfer_handler.cpp @@ -0,0 +1,143 @@ +#include "trsf/transfer_handler.h" + +#include "Poco/Buffer.h" +#include "boost/thread/lock_guard.hpp" + +namespace qcloud_cos { + PartState::PartState() : + m_part_num(0), + m_etag(""), + m_size_inbytes(0), + m_lastpart(false){} + + PartState::PartState(int part_num, std::string& etag, size_t size, bool last_part) : + m_part_num(part_num), + m_etag(etag), + m_size_inbytes(size), + m_lastpart(last_part){} + + + TransferHandler::TransferHandler(const std::string& bucket_name, const std::string& object_name, + uint64_t total_size, const std::string& file_path) : + m_bucket_name(bucket_name), + m_object_name(object_name), + m_local_file_path(file_path), + m_total_size(total_size), + m_current_progress(0), + m_status(TransferStatus::NOT_START), + m_uploadid(""), + m_cancel(false) {} + + void TransferHandler::UpdateProgress(uint64_t update_prog) { + boost::lock_guard locker(m_lock_prog); + + m_current_progress += update_prog; + + // Notice the progress there can not backwards, but the each parts has retry counts, + // should limit the progress no bigger than the total size. + // s3 has two invariants:(1) Never lock; (2) Never go backwards. Complete me. + if (m_current_progress > m_total_size) { + m_current_progress = m_total_size; + } + + } + + uint64_t TransferHandler::GetProgress() const { + boost::lock_guard locker(m_lock_prog); + return m_current_progress; + } + + bool TransferHandler::IsFinishStatus(TransferStatus status) const { + switch(status) + { + case TransferStatus::ABORTED: + case TransferStatus::COMPLETED: + case TransferStatus::FAILED: + case TransferStatus::CANCELED: + return true; + default: + return false; + } + } + + bool TransferHandler::IsAllowTransition(TransferStatus org, TransferStatus dst) const { + if (org == dst) { + return true; + } + + if (IsFinishStatus(org) && IsFinishStatus(dst)) { + return org == TransferStatus::CANCELED && dst == TransferStatus::ABORTED; + } + + return true; + } + + void TransferHandler::UpdateStatus(TransferStatus status) { + // There can unlock the condition to release the waitUntilFinish + // But need use the boost *-* + boost::unique_lock locker(m_lock_stat); + if (IsAllowTransition(m_status, status)) { + m_status = status; + + if(IsFinishStatus(status)) { + if(status == TransferStatus::COMPLETED){ + // Some other logic + } + locker.unlock(); + m_cond.notify_all(); + + } + } + + } + + TransferStatus TransferHandler::GetStatus() const { + boost::lock_guard locker(m_lock_stat); + return m_status; + } + + void TransferHandler::Cancel() { + boost::lock_guard locker(m_lock_stat); + m_cancel = true; + } + + bool TransferHandler::ShouldContinue() const { + boost::lock_guard locker(m_lock_stat); + return !m_cancel; + } + + + void TransferHandler::WaitUntilFinish() { + boost::unique_lock locker(m_lock_stat); + while(!IsFinishStatus(m_status)) { + m_cond.wait(locker); + } + } + + + std::streamsize HandleStreamCopier::handleCopyStream(std::istream& istr, std::ostream& ostr, + Poco::SharedPtr& handler,std::size_t bufferSize) { + poco_assert (bufferSize > 0); + + Poco::Buffer buffer(bufferSize); + std::streamsize len = 0; + istr.read(buffer.begin(), bufferSize); + std::streamsize n = istr.gcount(); + while (n > 0) { + // Throw the AssertionViolationException if the conditon is not true + poco_assert (handler->ShouldContinue()); + + len += n; + ostr.write(buffer.begin(), n); + // update progress + handler->UpdateProgress(n); + if (istr && ostr) { + istr.read(buffer.begin(), bufferSize); + n = istr.gcount(); + }else { + n = 0; + } + } + return len; + } +} From 4ecc7e6748fb5512a1be3e8037840909679ec72e Mon Sep 17 00:00:00 2001 From: toranger Date: Tue, 2 Apr 2019 21:18:26 +0800 Subject: [PATCH 03/16] Windows compatible first version --- include/cos_api.h | 4 +- include/cos_config.h | 7 +- include/cos_defines.h | 20 +++- include/request/object_req.h | 2 +- include/trsf/transfer_handler.h | 4 +- include/util/auth_tool.h | 162 ++++++++++++++++---------------- include/util/simple_mutex.h | 85 ++++++++++++++--- src/CMakeLists.txt | 66 +++++++------ src/cos_api.cpp | 7 +- src/cos_config.cpp | 20 ++-- src/op/file_download_task.cpp | 2 +- src/op/object_op.cpp | 59 +++++++----- src/request/bucket_req.cpp | 2 +- src/util/auth_tool.cpp | 8 +- src/util/http_sender.cpp | 26 ++++- src/util/string_util.cpp | 6 ++ 16 files changed, 305 insertions(+), 175 deletions(-) diff --git a/include/cos_api.h b/include/cos_api.h index 7ceada3..29133bc 100644 --- a/include/cos_api.h +++ b/include/cos_api.h @@ -5,8 +5,8 @@ #include "op/cos_result.h" #include "op/object_op.h" #include "op/service_op.h" -#include "util/simple_mutex.h" +#include "boost/thread/mutex.hpp" #include "Poco/SharedPtr.h" #include "trsf/transfer_handler.h" @@ -425,7 +425,7 @@ class CosAPI { BucketOp m_bucket_op; // 内部封装bucket相关的操作 ServiceOp m_service_op; // 内部封装service相关的操作 - static SimpleMutex s_init_mutex; + mutable boost::mutex s_init_mutex; static bool s_init; static bool s_poco_init; static int s_cos_obj_num; diff --git a/include/cos_config.h b/include/cos_config.h index 8e1ea20..82a5836 100644 --- a/include/cos_config.h +++ b/include/cos_config.h @@ -1,10 +1,11 @@ #ifndef COS_CONFIG_H #define COS_CONFIG_H +#include #include -#include -#include "util/simple_mutex.h" +#include "boost/thread.hpp" +#include "boost/thread/shared_mutex.hpp" namespace qcloud_cos{ class CosConfig{ @@ -116,7 +117,7 @@ class CosConfig{ void SetConfigCredentail(const std::string& access_key, const std::string& secret_key, const std::string& tmp_token); private: - mutable SimpleRWLock m_lock; + mutable boost::shared_mutex m_lock; uint64_t m_app_id; std::string m_access_key; std::string m_secret_key; diff --git a/include/cos_defines.h b/include/cos_defines.h index 0755556..aa8e9f0 100644 --- a/include/cos_defines.h +++ b/include/cos_defines.h @@ -2,7 +2,10 @@ #define COS_DEFINE_H #include #include + +#if !defined(WIN32) #include +#endif #include #include @@ -62,6 +65,18 @@ typedef enum cos_log_level { (level == COS_LOG_WARN) ? "[WARN] " : \ (level == COS_LOG_ERR) ? "[ERR] " : "[CRIT]") + +#if defined(WIN32) +#define COS_LOW_LOGPRN(level, fmt, ...) \ + if (level <= CosSysConfig::GetLogLevel()) { \ + if (CosSysConfig::GetLogOutType()== COS_LOG_STDOUT) { \ + fprintf(stdout,"%s:%s(%d) " fmt "\n", LOG_LEVEL_STRING(level),__func__,__LINE__, ##__VA_ARGS__); \ + }else if (CosSysConfig::GetLogOutType() == COS_LOG_SYSLOG){ \ + } else { \ + } \ + } else { \ + } +#else #define COS_LOW_LOGPRN(level, fmt, ...) \ if (level <= CosSysConfig::GetLogLevel()) { \ if (CosSysConfig::GetLogOutType()== COS_LOG_STDOUT) { \ @@ -71,9 +86,12 @@ typedef enum cos_log_level { } else { \ } \ } else { \ - } \ + } +#endif + +// For now just support the std output log for windows #define SDK_LOG_DBG(fmt, ...) COS_LOW_LOGPRN(COS_LOG_DBG, fmt, ##__VA_ARGS__) #define SDK_LOG_INFO(fmt, ...) COS_LOW_LOGPRN(COS_LOG_INFO, fmt, ##__VA_ARGS__) #define SDK_LOG_WARN(fmt, ...) COS_LOW_LOGPRN(COS_LOG_WARN, fmt, ##__VA_ARGS__) diff --git a/include/request/object_req.h b/include/request/object_req.h index 143119c..13bfe3d 100644 --- a/include/request/object_req.h +++ b/include/request/object_req.h @@ -392,7 +392,7 @@ class DeleteObjectsReq : public BaseReq { return m_objvers; } - uint32_t GetObjectVerionsSize() const { + size_t GetObjectVerionsSize() const { return m_objvers.size(); } diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h index 033ede8..0db9b95 100644 --- a/include/trsf/transfer_handler.h +++ b/include/trsf/transfer_handler.h @@ -8,7 +8,7 @@ #include "boost/thread/mutex.hpp" #include "boost/thread/condition_variable.hpp" #include "Poco/SharedPtr.h" -#include "util/simple_mutex.h" +//#include "util/simple_mutex.h" #include "op/cos_result.h" @@ -127,7 +127,7 @@ namespace qcloud_cos{ // Condition mutable boost::condition_variable m_cond; // Mutex lock for the part map - mutable SimpleMutex m_lock_part; + //mutable SimpleMutex m_lock_part; diff --git a/include/util/auth_tool.h b/include/util/auth_tool.h index bb18208..2afc47d 100644 --- a/include/util/auth_tool.h +++ b/include/util/auth_tool.h @@ -1,79 +1,83 @@ -#ifndef UTIL_AUTHTOOl_H -#define UTIL_AUTHTOOl_H - -#include - -#include -#include -#include - -#include "request/base_req.h" -#include "util/noncopyable.h" - -namespace qcloud_cos { - -class AuthTool : private NonCopyable { -public: - /// \brief ǩָЧ(ͨCosSysConfig, Ĭ60s)ʹ - /// - /// \param secret_id ӵеĿʶ ID֤ - /// \param secret_key ӵеĿԿ - /// \param http_method http,POST/GET/HEAD/PUT, Сд - /// \param in_uri http uri - /// \param headers http headerļֵ - /// \param params http paramsļֵ - /// - /// \return ַʽǩؿմʧ - static std::string Sign(const std::string& secret_id, - const std::string& secret_key, - const std::string& http_method, - const std::string& in_uri, - const std::map& headers, - const std::map& params); - - /// \brief ǩָЧʹ - /// - /// \param secret_id ӵеĿʶ ID֤ - /// \param secret_key ӵеĿԿ - /// \param http_method http,POST/GET/HEAD/PUT, Сд - /// \param in_uri http uri - /// \param headers http headerļֵ - /// \param params http paramsļֵ - /// - /// \return ַʽǩؿմʧ - static std::string Sign(const std::string& secret_id, - const std::string& secret_key, - const std::string& http_method, - const std::string& in_uri, - const std::map& headers, - const std::map& params, - uint64_t start_time_in_s, - uint64_t end_time_in_s); - -private: - /// \brief paramsеݣתСд,keyparam_list key=valueparam_value_list - /// \param params - /// \param key_encode keyǷuri - /// \param value_encode valueǷuri - /// \param value_lower valueǷСд - /// \param param_list б;ָ - /// \param param_value_list ֵб,&ָ - /// \retval - static void FillMap(const std::map ¶ms, - bool key_encode, - bool value_encode, - bool value_lower, - std::string* param_list, - std::string* param_value_list); - - /// \brief ҳҪȨͷ,Ŀǰhost conent-type xͷĶҪȨ - /// \param hedaers ͷkv - /// \param filted_req_headers ҪȨͷ - /// \retval - static void FilterAndSetSignHeader(const std::map& headers, - std::map* filted_req_headers); -}; - -} // namespace qcloud_cos - -#endif // AUTHTOOL_H +#ifndef UTIL_AUTHTOOl_H +#define UTIL_AUTHTOOl_H + +#include + +#include +#include +#include + +#include "request/base_req.h" +#include "util/noncopyable.h" + +namespace qcloud_cos { + +class AuthTool : private NonCopyable { +public: + /// \brief 返回签名，可以在指定的有效期内(通过CosSysConfig设置, 默认60s)使用 + /// + /// \param secret_id 开发者拥有的项目身份识别 ID，用以身份认证 + /// \param secret_key 开发者拥有的项目身份密钥 + /// \param http_method http方法,如POST/GET/HEAD/PUT等, 传入大小写不敏感 + /// \param in_uri http uri + /// \param headers http header的键值对 + /// \param params http params的键值对 + /// + /// \return 字符串形式的签名，返回空串代表失败 + static std::string Sign(const std::string& secret_id, + const std::string& secret_key, + const std::string& http_method, + const std::string& in_uri, + const std::map& headers, + const std::map& params); + + /// \brief 返回签名，可以在指定的有效期内使用 + /// + /// \param secret_id 开发者拥有的项目身份识别 ID，用以身份认证 + /// \param secret_key 开发者拥有的项目身份密钥 + /// \param http_method http方法,如POST/GET/HEAD/PUT等, 传入大小写不敏感 + /// \param in_uri http uri + /// \param headers http header的键值对 + /// \param params http params的键值对 + /// + /// \return 字符串形式的签名，返回空串代表失败 + static std::string Sign(const std::string& secret_id, + const std::string& secret_key, + const std::string& http_method, + const std::string& in_uri, + const std::map& headers, + const std::map& params, + uint64_t start_time_in_s, + uint64_t end_time_in_s); + +private: + /// \brief 把params中的数据，转小写，正排,key放在param_list key=value放param_value_list + /// \param params 参数 + /// \param key_encode key是否进行uri编码 + /// \param value_encode value是否进行uri编码 + /// \param value_lower value是否小写 + /// \param param_list 参数名列表，以;分隔 + /// \param param_value_list 参数键值对列表,以&分隔 + /// \retval 无 + static void FillMap(const std::map ¶ms, + bool key_encode, + bool value_encode, + bool value_lower, + std::string* param_list, + std::string* param_value_list); + + /// \brief 找出需要鉴权的头部，并设置,目前host conent-type 还有x开头的都要鉴权 + /// \param hedaers 头部的kv对 + /// \param filted_req_headers 需要鉴权的头部 + /// \retval 无 + static void FilterAndSetSignHeader(const std::map& headers, + std::map* filted_req_headers); +}; + +} // namespace qcloud_cos + +#endif // AUTHTOOL_H + + + + diff --git a/include/util/simple_mutex.h b/include/util/simple_mutex.h index c35e7a4..428dd99 100644 --- a/include/util/simple_mutex.h +++ b/include/util/simple_mutex.h @@ -1,27 +1,57 @@ #ifndef SIMPLE_MUTEX_H #define SIMPLE_MUTEX_H + +#if defined(WIN32) +#include +#include +#else #include +#endif +/** + * Notice SimpleMutext and SimpleRWLock are not used in current project, + * now use the boost mutex and shared_mutex instead. + * when want to reuse this header need notice the windows header included order. + */ class SimpleMutex { public: SimpleMutex() { - pthread_mutex_init(&m_mutex, NULL); +#if defined(WIN32) + m_mutex = CreateMutexA(NULL, FALSE, NULL); +#else + pthread_mutex_init(&m_mutex, NULL); +#endif } ~SimpleMutex() { - pthread_mutex_destroy(&m_mutex); +#if defined(WIN32) + CloseHandle(m_mutex); +#else + pthread_mutex_destroy(&m_mutex); +#endif } void Lock() { - pthread_mutex_lock(&m_mutex); - } +#if defined(WIN32) + DWORD d = WaitForSingleObject(m_mutex, INFINITE); +#else + pthread_mutex_lock(&m_mutex); +#endif } void Unlock() { - pthread_mutex_unlock(&m_mutex); +#if defined(WIN32) + ReleaseMutex(m_mutex); +#else + pthread_mutex_unlock(&m_mutex); +#endif } private: - pthread_mutex_t m_mutex; +#if defined(WIN32) + HANDLE m_mutex; +#else + pthread_mutex_t m_mutex; +#endif }; // mutex holder @@ -40,31 +70,60 @@ class SimpleMutexLocker { }; -// scoped rwlock for cos_config +// Scoped rwlock for cos_config +// Window's rwlock some version can not support, so there now use the mutex lock, +// then think about changing into the boost::shared_lock and boost::shared_mutex. class SimpleRWLock { public: SimpleRWLock() { - pthread_rwlock_init(&m_lock,NULL); +#if defined(WIN32) + m_lock = CreateMutexA(NULL, FALSE, NULL); +#else + pthread_rwlock_init(&m_lock, NULL); +#endif } ~SimpleRWLock() { - pthread_rwlock_destroy(&m_lock); +#if defined(WIN32) + CloseHandle(m_lock); +#else + pthread_rwlock_destroy(&m_lock); +#endif } void WriteLock() { - pthread_rwlock_wrlock(&m_lock); +#if defined(WIN32) + DWORD d = WaitForSingleObject(m_lock, INFINITE); +#else + pthread_rwlock_wrlock(&m_lock); +#endif + } void ReadLock() { - pthread_rwlock_rdlock(&m_lock); +#if defined(WIN32) + DWORD d = WaitForSingleObject(m_lock, INFINITE); +#else + pthread_rwlock_rdlock(&m_lock); +#endif + } void Unlock() { - pthread_rwlock_unlock(&m_lock); +#if defined(WIN32) + ReleaseMutex(m_lock); +#else + pthread_rwlock_unlock(&m_lock); +#endif } private: - pthread_rwlock_t m_lock; +#if defined(WIN32) + HANDLE m_lock; +#else + pthread_rwlock_t m_lock; +#endif + }; class SimpleWLocker { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ca172a3..340df06 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,29 +1,37 @@ -cmake_minimum_required(VERSION 2.8) -CMAKE_policy(SET CMP0015 NEW) - -# CMakeLists for src directory -PROJECT(COS_CPP_SDK) - -if (OPENSSL_VERSION VERSION_LESS 1.1.0) - message("old openssl version less than 1.1.0") - set(COSSDK_SOURCE_FILES cos_api.cpp cos_config.cpp cos_sys_config.cpp - request/base_req.cpp request/bucket_req.cpp request/object_req.cpp response/base_resp.cpp - response/object_resp.cpp response/bucket_resp.cpp response/service_resp.cpp - op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp - op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp - util/codec_util.cpp util/file_util.cpp util/http_sender.cpp - util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) -ELSE() - message("new version upper than 1.1.0") - set(COSSDK_SOURCE_FILES cos_api.cpp cos_config.cpp cos_sys_config.cpp - request/base_req.cpp request/bucket_req.cpp request/object_req.cpp response/base_resp.cpp - response/object_resp.cpp response/bucket_resp.cpp response/service_resp.cpp - op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp - op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp - util/codec_util_high_openssl.cpp util/file_util.cpp util/http_sender.cpp - util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) -ENDIF() - -add_library(cossdk STATIC ${COSSDK_SOURCE_FILES}) -target_link_libraries(cossdk PocoNetSSL PocoNet PocoCrypto PocoUtil PocoJSON PocoXML PocoFoundation ssl crypto rt stdc++ pthread jsoncpp boost_thread boost_system) -set_target_properties(cossdk PROPERTIES OUTPUT_NAME "cossdk") +cmake_minimum_required(VERSION 2.8) +CMAKE_policy(SET CMP0015 NEW) + +# CMakeLists for src directory +PROJECT(COS_CPP_SDK) + +if (OPENSSL_VERSION VERSION_LESS 1.1.0) + message("old openssl version less than 1.1.0") + set(COSSDK_SOURCE_FILES cos_api.cpp cos_config.cpp cos_sys_config.cpp + request/base_req.cpp request/bucket_req.cpp request/object_req.cpp response/base_resp.cpp + response/object_resp.cpp response/bucket_resp.cpp response/service_resp.cpp + op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp + op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp + util/codec_util.cpp util/file_util.cpp util/http_sender.cpp + util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) +ELSE() + message("new version upper than 1.1.0") + set(COSSDK_SOURCE_FILES cos_api.cpp cos_config.cpp cos_sys_config.cpp + request/base_req.cpp request/bucket_req.cpp request/object_req.cpp response/base_resp.cpp + response/object_resp.cpp response/bucket_resp.cpp response/service_resp.cpp + op/file_copy_task.cpp op/file_download_task.cpp op/file_upload_task.cpp op/base_op.cpp op/object_op.cpp + op/bucket_op.cpp op/service_op.cpp op/cos_result.cpp util/auth_tool.cpp + util/codec_util_high_openssl.cpp util/file_util.cpp util/http_sender.cpp + util/sha1.cpp util/string_util.cpp trsf/transfer_handler.cpp) +ENDIF() + + +add_library(cossdk STATIC ${COSSDK_SOURCE_FILES}) + +# When use the on windows need change the boost library according to the local name +if(WIN32) + target_link_libraries(cossdk PocoFoundation PocoNet PocoNetSSL PocoCrypto PocoUtil PocoJSON PocoXML ssl crypto jsoncpp libboost_system-vc141-mt-x64-1_69 libboost_thread-vc141-mt-x64-1_69) +else() + target_link_libraries(cossdk PocoNetSSL PocoNet PocoCrypto PocoUtil PocoJSON PocoXML PocoFoundation ssl crypto rt stdc++ pthread jsoncpp boost_thread boost_system) +endif() + +set_target_properties(cossdk PROPERTIES OUTPUT_NAME "cossdk") diff --git a/src/cos_api.cpp b/src/cos_api.cpp index 1ae5ca1..f435e08 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -1,5 +1,6 @@ #include "cos_api.h" +#include "boost/thread/lock_guard.hpp" #include "threadpool/boost/threadpool.hpp" #include "Poco/Net/HTTPStreamFactory.h" #include "Poco/Net/HTTPSStreamFactory.h" @@ -13,7 +14,7 @@ namespace qcloud_cos { bool CosAPI::s_init = false; bool CosAPI::s_poco_init = false; int CosAPI::s_cos_obj_num = 0; -SimpleMutex CosAPI::s_init_mutex = SimpleMutex(); + boost::threadpool::pool* g_threadpool = NULL; CosAPI::CosAPI(CosConfig& config) @@ -26,7 +27,7 @@ CosAPI::~CosAPI() { } int CosAPI::CosInit() { - SimpleMutexLocker locker(s_init_mutex); + boost::lock_guard locker(s_init_mutex); ++s_cos_obj_num; if (!s_init) { if (!s_poco_init) { @@ -44,7 +45,7 @@ int CosAPI::CosInit() { } void CosAPI::CosUInit() { - SimpleMutexLocker locker(s_init_mutex); + boost::lock_guard locker(s_init_mutex); --s_cos_obj_num; if (s_init && s_cos_obj_num == 0) { if (g_threadpool){ diff --git a/src/cos_config.cpp b/src/cos_config.cpp index d5d84cf..065a0dc 100644 --- a/src/cos_config.cpp +++ b/src/cos_config.cpp @@ -130,15 +130,13 @@ uint64_t CosConfig::GetAppId() const { } std::string CosConfig::GetAccessKey() const { - SimpleRLocker lock(m_lock); - std::string ak = m_access_key; - return ak; + boost::shared_lock lock(m_lock); + return m_access_key; } std::string CosConfig::GetSecretKey() const { - SimpleRLocker lock(m_lock); - std::string sk = m_secret_key; - return sk; + boost::shared_lock lock(m_lock); + return m_secret_key; } std::string CosConfig::GetRegion() const { @@ -146,13 +144,15 @@ std::string CosConfig::GetRegion() const { } std::string CosConfig::GetTmpToken() const { - SimpleRLocker lock(m_lock); - std::string token = m_tmp_token; - return token; + boost::shared_lock lock(m_lock); + return m_tmp_token; } void CosConfig::SetConfigCredentail(const std::string& access_key, const std::string& secret_key, const std::string& tmp_token) { - SimpleWLocker lock(m_lock); + // get upgradable access + boost::upgrade_lock lock(m_lock); + // get exclusive access + boost::upgrade_to_unique_lock uniqueLock(lock); m_access_key = access_key; m_secret_key = secret_key; m_tmp_token = tmp_token; diff --git a/src/op/file_download_task.cpp b/src/op/file_download_task.cpp index d6b53f8..9e7b829 100644 --- a/src/op/file_download_task.cpp +++ b/src/op/file_download_task.cpp @@ -57,7 +57,7 @@ std::map FileDownTask::GetRespHeaders() { void FileDownTask::DownTask() { char range_head[128]; memset(range_head, 0, sizeof(range_head)); - snprintf(range_head, sizeof(range_head), "bytes=%lu-%lu", + snprintf(range_head, sizeof(range_head), "bytes=%llu-%llu", m_offset, (m_offset + m_data_len - 1)); // 增加Range头域，避免大文件时将整个文件下载 diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index efa96df..1e4d959 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -16,6 +16,9 @@ #include "threadpool/boost/threadpool.hpp" #include +#if defined(WIN32) +#include +#endif #include "cos_sys_config.h" #include "op/file_copy_task.h" @@ -57,6 +60,7 @@ std::string ObjectOp::GetResumableUploadID(const std::string& bucket_name, const CosResult result = NormalAction(host, path, req, "", false, &resp); std::vector rst = resp.GetUpload(); + // Notice the index type, if size_t might over int index = rst.size() - 1; while (index >= 0) { if (rst[index].m_key == object_name) { @@ -504,16 +508,17 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, return result; } } - SDK_LOG_INFO("Multi upload object handler way id:%s, resumed:%d, already exist number:%d", resume_uploadid.c_str(), resume_flag); - + SDK_LOG_INFO("Multi upload object handler way id:%s, resumed:%d", resume_uploadid.c_str(), resume_flag); + // 2. Multi Upload std::vector etags; std::vector part_numbers; - // TODO(返回值判断), add the already exist parts + // Add the already exist parts handler->SetUploadID(resume_uploadid); handler->UpdateStatus(TransferStatus::IN_PROGRESS); result = MultiThreadUpload(req, resume_uploadid, &etags, &part_numbers, resume_flag, already_exist_parts, handler); + // Cancel way if (!handler->ShouldContinue()) { SDK_LOG_INFO("Multi upload object, canceled"); handler->UpdateStatus(TransferStatus::CANCELED); @@ -523,7 +528,7 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, // Notice the cancel way not need to abort the uploadid if (!result.IsSucc()) { SDK_LOG_ERR("Multi upload object fail, check upload mutli result."); - // Copy失败则需要Abort + // When copy failed need abort. AbortMultiUploadReq abort_req(req.GetBucketName(), req.GetObjectName(), resume_uploadid); AbortMultiUploadResp abort_resp; @@ -545,7 +550,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, CompleteMultiUploadReq comp_req(bucket_name, object_name, resume_uploadid); CompleteMultiUploadResp comp_resp; comp_req.SetConnTimeoutInms(req.GetConnTimeoutInms()); - comp_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms() * 2); // Complete的超时翻倍 + // Double timeout time + comp_req.SetRecvTimeoutInms(req.GetRecvTimeoutInms() * 2); comp_req.SetEtags(etags); comp_req.SetPartNumbers(part_numbers); @@ -788,7 +794,7 @@ CosResult ObjectOp::Copy(const CopyReq& req, CopyResp* resp) { // 源文件小于5G则采用PutObjectCopy进行复制 if (file_size < kPartSize5G || src_region == m_config->GetRegion()) { - SDK_LOG_INFO("File Size=%ld less than 5G, use put object copy.", file_size); + SDK_LOG_INFO("File Size=%lld less than 5G, use put object copy.", file_size); PutObjectCopyReq put_copy_req(req.GetBucketName(), req.GetObjectName()); put_copy_req.AddHeaders(req.GetHeaders()); PutObjectCopyResp put_copy_resp; @@ -801,7 +807,7 @@ CosResult ObjectOp::Copy(const CopyReq& req, CopyResp* resp) { } return result; } else if (file_size < req.GetPartSize() * 10000) { - SDK_LOG_INFO("File Size=%ld bigger than 5G, use put object copy.", file_size); + SDK_LOG_INFO("File Size=%lld bigger than 5G, use put object copy.", file_size); // 1. InitMultiUploadReq InitMultiUploadReq init_req(req.GetBucketName(), req.GetObjectName()); InitMultiUploadResp init_resp; @@ -847,7 +853,7 @@ CosResult ObjectOp::Copy(const CopyReq& req, CopyResp* resp) { if (end >= file_size) { end = file_size - 1; } - SDK_LOG_DBG("copy data, task_index=%d, file_size=%lu, offset=%lu, end=%lu", + SDK_LOG_DBG("copy data, task_index=%d, file_size=%llu, offset=%llu, end=%llu", task_index, file_size, offset, end); std::string range = "bytes=" + StringUtil::Uint64ToString(offset) + "-" + StringUtil::Uint64ToString(end); @@ -923,7 +929,7 @@ CosResult ObjectOp::Copy(const CopyReq& req, CopyResp* resp) { return result; } else { SDK_LOG_ERR("Source Object is too large or your upload copy part size in config" - "is too small, src obj size=%ld, copy_part_size=%ld", + "is too small, src obj size=%lld, copy_part_size=%lld", file_size, CosSysConfig::GetUploadCopyPartSize()); result.SetErrorInfo("Could not copy object, because of object size is too large " "or part size is too small."); @@ -990,8 +996,13 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb // 3. 打开本地文件 std::string local_path = req.GetLocalFilePath(); - int fd = open(local_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, - S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); +#if defined(WIN32) + int fd = _open(local_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, + _S_IREAD | _S_IWRITE); +#else + int fd = open(local_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, + S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); +#endif if (-1 == fd) { std::string err_info = "open file(" + local_path + ") fail, errno=" + StringUtil::IntToString(errno); @@ -1020,7 +1031,7 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb req.GetConnTimeoutInms(), req.GetRecvTimeoutInms()); } - SDK_LOG_DBG("download data,url=%s, poolsize=%u,slice_size=%u,file_size=%lu", + SDK_LOG_DBG("download data,url=%s, poolsize=%u,slice_size=%u,file_size=%llu", dest_url.c_str(), pool_size, slice_size, file_size); std::vector vec_offset; @@ -1031,11 +1042,11 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb unsigned down_times = 0; bool is_header_set = false; while(offset < file_size) { - SDK_LOG_DBG("down data, offset=%lu, file_size=%lu", offset, file_size); + SDK_LOG_DBG("down data, offset=%llu, file_size=%llu", offset, file_size); unsigned task_index = 0; vec_offset.clear(); for (; task_index < pool_size && (offset < file_size); ++task_index) { - SDK_LOG_DBG("down data, task_index=%d, file_size=%lu, offset=%lu", + SDK_LOG_DBG("down data, task_index=%d, file_size=%llu, offset=%llu", task_index, file_size, offset); FileDownTask* ptask = pptaskArr[task_index]; @@ -1092,8 +1103,8 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb resp->ParseFromHeaders(ptask->GetRespHeaders()); is_header_set = true; } - SDK_LOG_DBG("down data, down_times=%u,task_index=%d, file_size=%lu, " - "offset=%lu, downlen:%lu ", + SDK_LOG_DBG("down data, down_times=%u,task_index=%d, file_size=%llu, " + "offset=%llu, downlen:%zu ", down_times,task_index, file_size, vec_offset[task_index], ptask->GetDownLoadLen()); } @@ -1165,7 +1176,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, } if (part_number > kMaxPartNumbers) { - SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %d", part_number); + SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %lld", part_number); result.SetErrorInfo("part number bigger than 10000"); return result; } @@ -1182,7 +1193,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, pptaskArr[i] = new FileUploadTask(dest_url, req.GetConnTimeoutInms(), req.GetRecvTimeoutInms()); } - SDK_LOG_DBG("upload data,url=%s, poolsize=%u, part_size=%lu, file_size=%lu", + SDK_LOG_DBG("upload data,url=%s, poolsize=%u, part_size=%llu, file_size=%llu", dest_url.c_str(), pool_size, part_size, file_size); boost::threadpool::pool tp(pool_size); @@ -1200,7 +1211,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, break; } - SDK_LOG_DBG("upload data, task_index=%d, file_size=%lu, offset=%lu, len=%lu", + SDK_LOG_DBG("upload data, task_index=%d, file_size=%llu, offset=%llu, len=%zu", task_index, file_size, offset, read_len); // Check the resume @@ -1211,7 +1222,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, ptask->SetResumeEtag(already_exist_parts[part_number]); std::cout << "task etag is" << already_exist_parts[part_number] << std::endl; ptask->SetTaskSuccess(); - SDK_LOG_INFO("upload data part:%d has resumed", part_number); + SDK_LOG_INFO("upload data part:%lld has resumed", part_number); }else { FillUploadTask(upload_id, host, path, file_content_buf[task_index], read_len, @@ -1326,7 +1337,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, } if (part_number > kMaxPartNumbers) { - SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %d", part_number); + SDK_LOG_ERR("FileUploadSliceData: part number bigger than 10000, %lld", part_number); result.SetErrorInfo("part number bigger than 10000"); return result; } @@ -1343,7 +1354,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, pptaskArr[i] = new FileUploadTask(dest_url, req.GetConnTimeoutInms(), req.GetRecvTimeoutInms()); } - SDK_LOG_DBG("upload data,url=%s, poolsize=%u, part_size=%lu, file_size=%lu", + SDK_LOG_DBG("upload data,url=%s, poolsize=%u, part_size=%llu, file_size=%llu", dest_url.c_str(), pool_size, part_size, file_size); boost::threadpool::pool tp(pool_size); @@ -1367,7 +1378,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, break; } - SDK_LOG_DBG("upload data, task_index=%d, file_size=%lu, offset=%lu, len=%lu", + SDK_LOG_DBG("upload data, task_index=%d, file_size=%llu, offset=%llu, len=%zu", task_index, file_size, offset, read_len); // Check the resume @@ -1381,7 +1392,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, ptask->SetResumeEtag(already_exist_parts[part_number]); std::cout << "task etag is" << already_exist_parts[part_number] << std::endl; ptask->SetTaskSuccess(); - SDK_LOG_INFO("upload data part:%d has resumed", part_number); + SDK_LOG_INFO("upload data part:%lld has resumed", part_number); handler->UpdateProgress(read_len); }else { diff --git a/src/request/bucket_req.cpp b/src/request/bucket_req.cpp index 12794a9..f0c3acf 100644 --- a/src/request/bucket_req.cpp +++ b/src/request/bucket_req.cpp @@ -104,7 +104,7 @@ bool PutBucketLifecycleReq::GenerateRequestBody(std::string* body) const { "Filter", NULL); LifecycleFilter filter = rule.GetFilter(); std::vector tags = filter.GetTags(); - int cnt = tags.size(); + size_t cnt = tags.size(); if (filter.HasPrefix()) { ++cnt; } diff --git a/src/util/auth_tool.cpp b/src/util/auth_tool.cpp index 008880e..5a6fdab 100644 --- a/src/util/auth_tool.cpp +++ b/src/util/auth_tool.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -13,6 +13,12 @@ #include "util/http_sender.h" #include "cos_sys_config.h" +#if defined(WIN32) +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#endif + + namespace qcloud_cos { void AuthTool::FilterAndSetSignHeader(const std::map &headers, diff --git a/src/util/http_sender.cpp b/src/util/http_sender.cpp index a58bb10..cf994aa 100644 --- a/src/util/http_sender.cpp +++ b/src/util/http_sender.cpp @@ -7,7 +7,11 @@ #include "util/http_sender.h" -#include +#if defined(WIN32) +#include +#else +#include +#endif #include #include @@ -601,12 +605,24 @@ int HttpSender::SendRequest(const std::string& http_method, return res.getStatus(); } -// TODO(sevenyou) 挪走 + +// Must notice for now the sign auth time is second. +// the time function in windows can only support to second, +// also can use the GetSystemTimeAsFileTime which the resolution is only 15625 microseconds +// (if you make successive calls to gettimeofday() until it changes value). +// It's not much better than calling GetLocalTime() which is accurate to between 15000 and 31000 microseconds. +// This differs significantly from the unix implementations which are accurate close to the microsecond. uint64_t HttpSender::GetTimeStampInUs() { // 构造时间 - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec * 1000000 + tv.tv_usec; +#if defined(WIN32) + time_t ltime; + time(<ime); + return (uint64_t)(ltime * 1000000); +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; +#endif } } // namespace qcloud_cos diff --git a/src/util/string_util.cpp b/src/util/string_util.cpp index f61a02f..986af5e 100644 --- a/src/util/string_util.cpp +++ b/src/util/string_util.cpp @@ -6,6 +6,12 @@ #include #include +#if defined(WIN32) +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#endif + + namespace qcloud_cos { std::string& StringUtil::Trim(std::string &s) { From a928121aa8b3b4963d19d125c897fbbc3663916d Mon Sep 17 00:00:00 2001 From: toranger Date: Wed, 3 Apr 2019 16:10:02 +0800 Subject: [PATCH 04/16] Fix the param order and other details --- include/cos_api.h | 19 ++++++------ include/op/object_op.h | 22 ++++++++------ include/trsf/transfer_handler.h | 16 +++------- include/util/http_sender.h | 3 +- src/cos_api.cpp | 13 ++++---- src/op/file_upload_task.cpp | 4 +-- src/op/object_op.cpp | 54 +++++++++++++++++---------------- src/trsf/transfer_handler.cpp | 15 +++++---- src/util/http_sender.cpp | 2 +- 9 files changed, 74 insertions(+), 74 deletions(-) diff --git a/include/cos_api.h b/include/cos_api.h index 29133bc..25ba783 100644 --- a/include/cos_api.h +++ b/include/cos_api.h @@ -338,9 +338,10 @@ class CosAPI { Poco::SharedPtr TransferUploadObject(const MultiUploadObjectReq& request, - MultiUploadObjectResp* response) ; + MultiUploadObjectResp* response) ; - Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) ; + Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, + const std::string& local_path) ; /// \brief 舍弃一个分块上传并删除已上传的块 /// 详见: https://www.qcloud.com/document/product/436/7740 @@ -434,11 +435,11 @@ class CosAPI { // Use for trsf the param into boost bind function class AsynArgs { public: - AsynArgs(ObjectOp* op) : m_op(op){} - AsynArgs(const AsynArgs& arg){ + AsynArgs(ObjectOp* op) : m_op(op) {} + AsynArgs(const AsynArgs& arg) { this->m_op = arg.m_op; } - virtual ~AsynArgs(){}; + virtual ~AsynArgs() {}; ObjectOp* m_op; }; @@ -448,16 +449,16 @@ class TransferAsynArgs : public AsynArgs { TransferAsynArgs(ObjectOp* pObj, const MultiUploadObjectReq& req, MultiUploadObjectResp *resp, - Poco::SharedPtr& handler) : AsynArgs(pObj) , m_req(req), m_resp(resp){ + Poco::SharedPtr& handler) : AsynArgs(pObj) , m_req(req), m_resp(resp) { m_handler = handler; } TransferAsynArgs(const TransferAsynArgs& arg) : AsynArgs(arg), - m_req(arg.m_req), - m_handler(arg.m_handler), - m_resp(arg.m_resp) { + m_req(arg.m_req), + m_handler(arg.m_handler), + m_resp(arg.m_resp) { } virtual ~TransferAsynArgs() {} public: diff --git a/include/op/object_op.h b/include/op/object_op.h index dac9953..5d4c262 100644 --- a/include/op/object_op.h +++ b/include/op/object_op.h @@ -45,7 +45,7 @@ class ObjectOp : public BaseOp { const std::string& localpath, std::vector& already_exist); bool check_single_part(const std::string& local_file_path, uint64_t offset, uint64_t local_part_size, - uint64_t size, std::string& etag); + uint64_t size, const std::string& etag); /// \brief 获取对应Object的meta信息数据 /// /// \param request HeadObject请求 @@ -151,11 +151,13 @@ class ObjectOp : public BaseOp { /// \param response MultiUploadObject返回 /// /// \return 返回HTTP请求的状态码及错误信息 - CosResult MultiUploadObject(const MultiUploadObjectReq& req, MultiUploadObjectResp* resp, Poco::SharedPtr& handler); + CosResult MultiUploadObject(const MultiUploadObjectReq& req, Poco::SharedPtr& handler, + MultiUploadObjectResp* resp); CosResult MultiUploadObject(const MultiUploadObjectReq& req, MultiUploadObjectResp* resp); - Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path); + Poco::SharedPtr CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, + const std::string& local_path); /// \brief 舍弃一个分块上传并删除已上传的块 /// /// \param req AbortMultiUpload请求 @@ -230,18 +232,18 @@ class ObjectOp : public BaseOp { // 上传文件, 内部使用多线程 CosResult MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, - std::vector* etags_ptr, - std::vector* part_numbers_ptr, - bool resume_flag, const std::vector& already_exist_parts, - Poco::SharedPtr& handler); + Poco::SharedPtr& handler, + bool resume_flag, + std::vector* etags_ptr, + std::vector* part_numbers_ptr); CosResult MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, - std::vector* etags_ptr, - std::vector* part_numbers_ptr, + const std::vector& already_exist_parts, bool resume_flag, - const std::vector& already_exist_parts); + std::vector* etags_ptr, + std::vector* part_numbers_ptr); // 读取文件内容, 并返回读取的长度 uint64_t GetContent(const std::string& src, std::string* file_content) const; diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h index 0db9b95..673bdd3 100644 --- a/include/trsf/transfer_handler.h +++ b/include/trsf/transfer_handler.h @@ -8,7 +8,6 @@ #include "boost/thread/mutex.hpp" #include "boost/thread/condition_variable.hpp" #include "Poco/SharedPtr.h" -//#include "util/simple_mutex.h" #include "op/cos_result.h" @@ -100,7 +99,7 @@ namespace qcloud_cos{ void WaitUntilFinish(); public: - //origin result + // Origin result CosResult m_result; private: @@ -118,25 +117,20 @@ namespace qcloud_cos{ PartStateMap m_part_map; // Mutex lock for the progress - /* mutable SimpleMutex m_lock_prog; */ mutable boost::mutex m_lock_prog; // Mutex lock for the status - /* mutable SimpleMutex m_lock_stat; */ mutable boost::mutex m_lock_stat; - // Condition mutable boost::condition_variable m_cond; - // Mutex lock for the part map - //mutable SimpleMutex m_lock_part; - - + // Mutex lock for the part map + // mutable boost::mutex m_lock_parts; }; class HandleStreamCopier { public: - static std::streamsize handleCopyStream(std::istream& istr, std::ostream& ostr, - Poco::SharedPtr& handler, std::size_t bufferSize = 8192); + static std::streamsize handleCopyStream(std::istream& istr, std::ostream& ostr, Poco::SharedPtr& handler, + std::size_t bufferSize = 8192); }; diff --git a/include/util/http_sender.h b/include/util/http_sender.h index d03bf1b..d8aff9d 100644 --- a/include/util/http_sender.h +++ b/include/util/http_sender.h @@ -30,13 +30,14 @@ class HttpSender { const std::map& req_params, const std::map& req_headers, const std::string& req_body, + Poco::SharedPtr& handler, uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms, std::map* resp_headers, std::string* resp_body, std::string* err_msg, - Poco::SharedPtr& handler, bool is_check_md5 = false); + // real trsf handler process static int SendRequest(const std::string& http_method, const std::string& url_str, diff --git a/src/cos_api.cpp b/src/cos_api.cpp index f435e08..b33624e 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -258,24 +258,24 @@ void TransferSubmit(TransferAsynArgs args) { Poco::SharedPtr handler = args.m_handler; // task is m_object_op.MultiUploadObject(request, response, handler); - op->MultiUploadObject(req, resp, handler); + op->MultiUploadObject(req, handler, resp); } // Async to transfer Poco::SharedPtr CosAPI::TransferUploadObject(const MultiUploadObjectReq& request, - MultiUploadObjectResp* response) { - // create the handler + MultiUploadObjectResp* response) { + // Create the handler Poco::SharedPtr handler = CreateUploadHandler(request.GetBucketName(), request.GetObjectName(), request.GetLocalFilePath()); TransferAsynArgs args(&m_object_op, request, response, handler); - // use the cos's boost thread pool to submit the task + // Use the cos's boost thread pool to submit the task if(g_threadpool) { g_threadpool->schedule(boost::bind(&TransferSubmit, args)); }else { handler->UpdateStatus(TransferStatus::FAILED); } - // return the handler outside. + // Return the handler outside. return handler; } @@ -285,7 +285,8 @@ CosResult CosAPI::MultiUploadObject(const MultiUploadObjectReq& request, return m_object_op.MultiUploadObject(request, response); } -Poco::SharedPtr CosAPI::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) { +Poco::SharedPtr CosAPI::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, + const std::string& local_path) { return m_object_op.CreateUploadHandler(bucket_name, object_name, local_path); } diff --git a/src/op/file_upload_task.cpp b/src/op/file_upload_task.cpp index df5013a..2f51885 100644 --- a/src/op/file_upload_task.cpp +++ b/src/op/file_upload_task.cpp @@ -93,8 +93,8 @@ void FileUploadTask::UploadTask() { if(IsHandler()) { m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, - body, m_conn_timeout_in_ms, m_recv_timeout_in_ms, - &m_resp_headers, &m_resp, &m_err_msg, m_handler); + body, m_handler, m_conn_timeout_in_ms, m_recv_timeout_in_ms, + &m_resp_headers, &m_resp, &m_err_msg); }else { m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, body, m_conn_timeout_in_ms, m_recv_timeout_in_ms, diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 1e4d959..3252c85 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -72,7 +72,7 @@ std::string ObjectOp::GetResumableUploadID(const std::string& bucket_name, const } bool ObjectOp::check_single_part(const std::string& local_file_path, uint64_t offset, uint64_t local_part_size, - uint64_t size, std::string& etag) { + uint64_t size, const std::string& etag) { if (local_part_size != size) { return false; } @@ -85,25 +85,23 @@ bool ObjectOp::check_single_part(const std::string& local_file_path, uint64_t of fin.seekg (offset); - // allocate memory: + // Allocate memory: char *data = new char[local_part_size]; - // read data as a block: + // Read data as a block: fin.read (data,local_part_size); std::cout << "gount is :" << fin.gcount() << std::endl; fin.seekg (0, fin.beg); fin.close(); - // print content: + // Print content: std::istringstream stringStream(std::string(data, local_part_size)); - // std::istringstream stringStream(data); Poco::MD5Engine md5; Poco::DigestOutputStream dos(md5); Poco::StreamCopier::copyStream(stringStream, dos); - // dos << stringStream.str(); dos.flush(); std::string md5_str = Poco::DigestEngine::digestToHex(md5.digest()); @@ -116,7 +114,7 @@ bool ObjectOp::check_single_part(const std::string& local_file_path, uint64_t of } return true; } - // TODO tmd + bool ObjectOp::CheckUploadPart(const MultiUploadObjectReq& req, const std::string& bucket_name, const std::string& object_name, const std::string& uploadid, const std::string& localpath, std::vector& already_exist) { @@ -176,13 +174,13 @@ bool ObjectOp::CheckUploadPart(const MultiUploadObjectReq& req, const std::strin local_part_size = last_part_size; } - // check single upload part each md5 + // Check single upload part each md5 std::string etag = itr->m_etag; if (!check_single_part(local_file_path, offset, local_part_size, itr->m_size, etag)) { return false; } - // add the part_num with etags in already exist + // Add the part_num with etags in already exist already_exist[sev_part_num] = itr->m_etag; } @@ -371,7 +369,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, // check the breakpoint std::string resume_uploadid = GetResumableUploadID(bucket_name, object_name); if (!resume_uploadid.empty()) { - resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, local_file_path, already_exist_parts); + resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, + local_file_path, already_exist_parts); } if (!resume_flag) { @@ -411,7 +410,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, std::vector etags; std::vector part_numbers; // TODO(返回值判断), add the already exist parts - result = MultiThreadUpload(req, resume_uploadid, &etags, &part_numbers, resume_flag, already_exist_parts); + result = MultiThreadUpload(req, resume_uploadid, already_exist_parts, + resume_flag, &etags, &part_numbers); if (!result.IsSucc()) { SDK_LOG_ERR("Multi upload object fail, check upload mutli result."); // Copy失败则需要Abort @@ -443,7 +443,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, } // Create the handler -Poco::SharedPtr ObjectOp::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, const std::string& local_path) { +Poco::SharedPtr ObjectOp::CreateUploadHandler(const std::string& bucket_name, const std::string& object_name, + const std::string& local_path) { TransferHandler *p = new TransferHandler(bucket_name, object_name, 0, local_path); Poco::SharedPtr handler(p); @@ -455,8 +456,8 @@ Poco::SharedPtr ObjectOp::CreateUploadHandler(const std::string // Transfer call CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, - MultiUploadObjectResp* resp, - Poco::SharedPtr& handler) { + Poco::SharedPtr& handler, + MultiUploadObjectResp* resp) { CosResult result; uint64_t app_id = GetAppId(); std::string bucket_name = req.GetBucketName(); @@ -469,7 +470,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, // check the breakpoint std::string resume_uploadid = GetResumableUploadID(bucket_name, object_name); if (!resume_uploadid.empty()) { - resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, local_file_path, already_exist_parts); + resume_flag = CheckUploadPart(req, bucket_name, object_name, resume_uploadid, + local_file_path, already_exist_parts); } if (!resume_flag) { @@ -517,7 +519,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, handler->SetUploadID(resume_uploadid); handler->UpdateStatus(TransferStatus::IN_PROGRESS); - result = MultiThreadUpload(req, resume_uploadid, &etags, &part_numbers, resume_flag, already_exist_parts, handler); + result = MultiThreadUpload(req, resume_uploadid, already_exist_parts, handler, + resume_flag, &etags, &part_numbers); // Cancel way if (!handler->ShouldContinue()) { SDK_LOG_INFO("Multi upload object, canceled"); @@ -1136,14 +1139,13 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb return result; } -// TODO(sevenyou) 多线程上传, 返回的resp内容需要再斟酌下. - // origin way +// Origin way CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, - std::vector* etags_ptr, - std::vector* part_numbers_ptr, + const std::vector& already_exist_parts, bool resume_flag, - const std::vector& already_exist_parts) { + std::vector* etags_ptr, + std::vector* part_numbers_ptr) { CosResult result; std::string path = "/" + req.GetObjectName(); std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), @@ -1297,14 +1299,14 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, return result; } -//Trsf way +// Trsf way CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, const std::string& upload_id, - std::vector* etags_ptr, - std::vector* part_numbers_ptr, - bool resume_flag, const std::vector& already_exist_parts, - Poco::SharedPtr& handler) { + Poco::SharedPtr& handler, + bool resume_flag, + std::vector* etags_ptr, + std::vector* part_numbers_ptr) { CosResult result; std::string path = "/" + req.GetObjectName(); std::string host = CosSysConfig::GetHost(GetAppId(), m_config->GetRegion(), diff --git a/src/trsf/transfer_handler.cpp b/src/trsf/transfer_handler.cpp index 43b2743..b6a0540 100644 --- a/src/trsf/transfer_handler.cpp +++ b/src/trsf/transfer_handler.cpp @@ -48,8 +48,7 @@ namespace qcloud_cos { } bool TransferHandler::IsFinishStatus(TransferStatus status) const { - switch(status) - { + switch (status) { case TransferStatus::ABORTED: case TransferStatus::COMPLETED: case TransferStatus::FAILED: @@ -79,8 +78,8 @@ namespace qcloud_cos { if (IsAllowTransition(m_status, status)) { m_status = status; - if(IsFinishStatus(status)) { - if(status == TransferStatus::COMPLETED){ + if (IsFinishStatus(status)) { + if (status == TransferStatus::COMPLETED) { // Some other logic } locker.unlock(); @@ -109,14 +108,14 @@ namespace qcloud_cos { void TransferHandler::WaitUntilFinish() { boost::unique_lock locker(m_lock_stat); - while(!IsFinishStatus(m_status)) { + while (!IsFinishStatus(m_status)) { m_cond.wait(locker); } } - std::streamsize HandleStreamCopier::handleCopyStream(std::istream& istr, std::ostream& ostr, - Poco::SharedPtr& handler,std::size_t bufferSize) { + std::streamsize HandleStreamCopier::handleCopyStream(std::istream& istr, std::ostream& ostr, Poco::SharedPtr& handler, + std::size_t bufferSize) { poco_assert (bufferSize > 0); Poco::Buffer buffer(bufferSize); @@ -134,7 +133,7 @@ namespace qcloud_cos { if (istr && ostr) { istr.read(buffer.begin(), bufferSize); n = istr.gcount(); - }else { + } else { n = 0; } } diff --git a/src/util/http_sender.cpp b/src/util/http_sender.cpp index cf994aa..43e5b93 100644 --- a/src/util/http_sender.cpp +++ b/src/util/http_sender.cpp @@ -41,12 +41,12 @@ int HttpSender::SendRequest(const std::string& http_method, const std::map& req_params, const std::map& req_headers, const std::string& req_body, + Poco::SharedPtr& handler, uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms, std::map* resp_headers, std::string* resp_body, std::string* err_msg, - Poco::SharedPtr& handler, bool is_check_md5) { std::istringstream is(req_body); std::ostringstream oss; From 883de1640d1b09919655e535059a00370ec7e93d Mon Sep 17 00:00:00 2001 From: toranger Date: Wed, 3 Apr 2019 18:25:28 +0800 Subject: [PATCH 05/16] Use the boost bind send params, replace the middle asyn class, add the TransferUploadObject demo --- demo/cos_demo.cpp | 49 ++++++++++++++++++++++++++++++++- include/trsf/transfer_handler.h | 8 ++++++ src/cos_api.cpp | 17 +++++------- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/demo/cos_demo.cpp b/demo/cos_demo.cpp index 998e5c4..b04129a 100644 --- a/demo/cos_demo.cpp +++ b/demo/cos_demo.cpp @@ -6,6 +6,7 @@ // Description: #include +#include #include #include #include @@ -15,6 +16,8 @@ #include "cos_sys_config.h" #include "cos_defines.h" +#include "Poco/SharedPtr.h" + using namespace qcloud_cos; void PrintResult(const qcloud_cos::CosResult& result, const qcloud_cos::BaseResp& resp) { if (result.IsSucc()) { @@ -431,6 +434,7 @@ void CompleteMultiUpload(qcloud_cos::CosAPI& cos, const std::string& bucket_name std::cout << "========================================================" << std::endl; } +// Upload object without handler void MultiUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, const std::string& object_name, const std::string& local_file) { qcloud_cos::MultiUploadObjectReq req(bucket_name, @@ -462,6 +466,43 @@ void MultiUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, std::cout << "========================================================" << std::endl; } +// Upload object with handler +void TransferUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, + const std::string& object_name, const std::string& local_file) { + qcloud_cos::MultiUploadObjectReq req(bucket_name, + object_name, local_file); + req.SetRecvTimeoutInms(1000 * 60); + qcloud_cos::MultiUploadObjectResp resp; + + Poco::SharedPtr handler = cos.TransferUploadObject(req, &resp); + // The TransferUploadObject is the asynchronization api, can use WaitUntilFinish to block until finish. + // At the same time the handler support the GetTotalSize(), GetProgress(), GetStatus(), Cancel() etc. + handler->WaitUntilFinish(); + + // Notice when not block with the WaitUntilFinish() the result might not get soon. + if (handler->m_result.IsSucc()) { + std::cout << "MultiUpload Succ." << std::endl; + std::cout << resp.GetLocation() << std::endl; + std::cout << resp.GetKey() << std::endl; + std::cout << resp.GetBucket() << std::endl; + std::cout << resp.GetEtag() << std::endl; + } else { + std::cout << "MultiUpload Fail." << std::endl; + // 获取具体失败在哪一步 + std::string resp_tag = resp.GetRespTag(); + if ("Init" == resp_tag) { + // print result + } else if ("Upload" == resp_tag) { + // print result + } else if ("Complete" == resp_tag) { + // print result + } + } + std::cout << "===================MultiUpload=============================" << std::endl; + PrintResult(handler->m_result, resp); + std::cout << "========================================================" << std::endl; +} + void ListParts(qcloud_cos::CosAPI& cos, const std::string& bucket_name, const std::string& object_name, const std::string& upload_id) { qcloud_cos::ListPartsReq req(bucket_name, object_name, upload_id); @@ -654,6 +695,8 @@ int main(int argc, char** argv) { // GetObjectByFile(cos, bucket_name, "sevenyou_e2_abc", "/data/sevenyou/temp/sevenyou_10m_download_03"); //GetObjectByStream(cos, bucket_name, "sevenyou_e2_abc"); // MultiGetObject(cos, bucket_name, "sevenyou_1102_south_multi", "/data/sevenyou/temp/sevenyou_10m_download_03"); + // MultiGetObject(cos, bucket_name, "test000part", "./multiget"); + // TransferUploadObject(cos, bucket_name, "transfer", "./test6M1"); // { // std::string upload_id; @@ -797,5 +840,9 @@ int main(int argc, char** argv) { // PrintResult(result, resp); // std::cout << "=========================================================" << std::endl; // } - +#if defined(WIN32) + system("pause"); +#endif } + + diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h index 673bdd3..cad95b7 100644 --- a/include/trsf/transfer_handler.h +++ b/include/trsf/transfer_handler.h @@ -83,19 +83,27 @@ namespace qcloud_cos{ // Notice there can not backwards void UpdateProgress(uint64_t update_prog); + // Get the current upload size(B). uint64_t GetProgress() const; void UpdateStatus(TransferStatus status); + // Get the current status of process, detail see the enum TransferStatus. TransferStatus GetStatus() const; void SetUploadID(const std::string& uploadid) { m_uploadid = uploadid; } + // Get the init or resumed uploadid. std::string GetUploadID() const { return m_uploadid; } + // Cancel the process of interface the uploadid can reuse. void Cancel(); + bool ShouldContinue() const; bool IsFinishStatus(TransferStatus status) const; + bool IsAllowTransition(TransferStatus org, TransferStatus dst) const; + + // Block until finish. void WaitUntilFinish(); public: diff --git a/src/cos_api.cpp b/src/cos_api.cpp index b33624e..e77b5ec 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -251,14 +251,13 @@ CosResult CosAPI::CompleteMultiUpload(const CompleteMultiUploadReq& request, return m_object_op.CompleteMultiUpload(request, response); } -void TransferSubmit(TransferAsynArgs args) { - ObjectOp *op = args.m_op; - MultiUploadObjectReq req = args.m_req; - MultiUploadObjectResp* resp = args.m_resp; - Poco::SharedPtr handler = args.m_handler; +void TransferSubmitTest(ObjectOp* op, const MultiUploadObjectReq& req, + Poco::SharedPtr& handler, + MultiUploadObjectResp* resp) { + if(op){ + op->MultiUploadObject(req, handler, resp); + } - // task is m_object_op.MultiUploadObject(request, response, handler); - op->MultiUploadObject(req, handler, resp); } // Async to transfer @@ -267,11 +266,9 @@ Poco::SharedPtr CosAPI::TransferUploadObject(const MultiUploadO // Create the handler Poco::SharedPtr handler = CreateUploadHandler(request.GetBucketName(), request.GetObjectName(), request.GetLocalFilePath()); - TransferAsynArgs args(&m_object_op, request, response, handler); - // Use the cos's boost thread pool to submit the task if(g_threadpool) { - g_threadpool->schedule(boost::bind(&TransferSubmit, args)); + g_threadpool->schedule(boost::bind(&TransferSubmitTest, &m_object_op, request, handler, response)); }else { handler->UpdateStatus(TransferStatus::FAILED); } From 092146048fb9b80a7b00e57112e69b637f8cb49d Mon Sep 17 00:00:00 2001 From: toranger Date: Wed, 3 Apr 2019 18:27:29 +0800 Subject: [PATCH 06/16] fix the transfersubmit name --- src/cos_api.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cos_api.cpp b/src/cos_api.cpp index e77b5ec..ea61852 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -251,7 +251,7 @@ CosResult CosAPI::CompleteMultiUpload(const CompleteMultiUploadReq& request, return m_object_op.CompleteMultiUpload(request, response); } -void TransferSubmitTest(ObjectOp* op, const MultiUploadObjectReq& req, +void TransferSubmit(ObjectOp* op, const MultiUploadObjectReq& req, Poco::SharedPtr& handler, MultiUploadObjectResp* resp) { if(op){ @@ -268,7 +268,7 @@ Poco::SharedPtr CosAPI::TransferUploadObject(const MultiUploadO request.GetLocalFilePath()); // Use the cos's boost thread pool to submit the task if(g_threadpool) { - g_threadpool->schedule(boost::bind(&TransferSubmitTest, &m_object_op, request, handler, response)); + g_threadpool->schedule(boost::bind(&TransferSubmit, &m_object_op, request, handler, response)); }else { handler->UpdateStatus(TransferStatus::FAILED); } From bf2d0dffde062da6dff60fc6e9248d06eddf4d2b Mon Sep 17 00:00:00 2001 From: toranger Date: Mon, 15 Apr 2019 19:30:38 +0800 Subject: [PATCH 07/16] Add upload progress callback in multiupload request --- demo/cos_demo.cpp | 7 ++++++- include/op/file_upload_task.h | 2 ++ include/request/object_req.h | 24 ++++++++++++++++++++++++ include/trsf/transfer_handler.h | 9 +++++++-- include/util/http_sender.h | 7 +++++-- src/op/file_upload_task.cpp | 6 +++--- src/op/object_op.cpp | 3 +++ src/trsf/transfer_handler.cpp | 7 +++++-- src/util/http_sender.cpp | 11 +++++++---- 9 files changed, 62 insertions(+), 14 deletions(-) diff --git a/demo/cos_demo.cpp b/demo/cos_demo.cpp index b04129a..d7c31af 100644 --- a/demo/cos_demo.cpp +++ b/demo/cos_demo.cpp @@ -466,14 +466,19 @@ void MultiUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, std::cout << "========================================================" << std::endl; } +void uploadprogress(const MultiUploadObjectReq *req, Poco::SharedPtr &handler) { + std::cout << "callback data is :" << handler->GetProgress() << std::endl; + +} + // Upload object with handler void TransferUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, const std::string& object_name, const std::string& local_file) { qcloud_cos::MultiUploadObjectReq req(bucket_name, object_name, local_file); req.SetRecvTimeoutInms(1000 * 60); + req.SetUploadProgressCallback(uploadprogress); qcloud_cos::MultiUploadObjectResp resp; - Poco::SharedPtr handler = cos.TransferUploadObject(req, &resp); // The TransferUploadObject is the asynchronization api, can use WaitUntilFinish to block until finish. // At the same time the handler support the GetTotalSize(), GetProgress(), GetStatus(), Cancel() etc. diff --git a/include/op/file_upload_task.h b/include/op/file_upload_task.h index 1e1a9db..4783f10 100644 --- a/include/op/file_upload_task.h +++ b/include/op/file_upload_task.h @@ -9,6 +9,7 @@ #include "cos_params.h" #include "cos_sys_config.h" #include "op/base_op.h" +#include "request/object_req.h" #include "util/codec_util.h" #include "util/file_util.h" #include "util/http_sender.h" @@ -73,6 +74,7 @@ class FileUploadTask { public: Poco::SharedPtr m_handler; + MultiUploadObjectReq *m_req; private: std::string m_full_url; diff --git a/include/request/object_req.h b/include/request/object_req.h index 13bfe3d..d1428fe 100644 --- a/include/request/object_req.h +++ b/include/request/object_req.h @@ -17,9 +17,15 @@ #include "cos_defines.h" #include "cos_sys_config.h" +#include "trsf/transfer_handler.h" + +#include "Poco/SharedPtr.h" + namespace qcloud_cos { +class TransferHandler; + class ObjectReq : public BaseReq { public: ObjectReq(const std::string& bucket_name, const std::string& object_name) @@ -659,6 +665,8 @@ class CompleteMultiUploadReq : public ObjectReq { }; class MultiUploadObjectReq : public ObjectReq { +public: + typedef void (*UploadProgressCallback)(const MultiUploadObjectReq *req, Poco::SharedPtr& handler); public: MultiUploadObjectReq(const std::string& bucket_name, const std::string& object_name, const std::string& local_file_path = "") @@ -667,6 +675,7 @@ class MultiUploadObjectReq : public ObjectReq { m_part_size = CosSysConfig::GetUploadPartSize(); m_thread_pool_size = CosSysConfig::GetUploadThreadPoolSize(); mb_set_meta = false; + m_progress_callback = NULL; // 默认打开当前路径下object的同名文件 if (local_file_path.empty()) { @@ -733,6 +742,17 @@ class MultiUploadObjectReq : public ObjectReq { return m_uploadid; } + void SetUploadProgressCallback(UploadProgressCallback callback) { + m_progress_callback = callback; + } + + void TriggerUploadProgressCallback(Poco::SharedPtr& handler) const { + if(m_progress_callback) { + m_progress_callback(this, handler); + } + } + + private: std::string m_local_file_path; uint64_t m_part_size; @@ -740,6 +760,10 @@ class MultiUploadObjectReq : public ObjectReq { std::map m_xcos_meta; bool mb_set_meta; std::string m_uploadid; + +public: + UploadProgressCallback m_progress_callback; + }; class AbortMultiUploadReq : public ObjectReq { diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h index cad95b7..7a1e8c7 100644 --- a/include/trsf/transfer_handler.h +++ b/include/trsf/transfer_handler.h @@ -10,8 +10,12 @@ #include "Poco/SharedPtr.h" #include "op/cos_result.h" +#include "request/object_req.h" + namespace qcloud_cos{ + class MultiUploadObjectReq; + class PartState { public: PartState(); @@ -137,8 +141,9 @@ namespace qcloud_cos{ class HandleStreamCopier { public: - static std::streamsize handleCopyStream(std::istream& istr, std::ostream& ostr, Poco::SharedPtr& handler, - std::size_t bufferSize = 8192); + static std::streamsize handleCopyStream(const MultiUploadObjectReq *req, std::istream& istr, std::ostream& ostr, + Poco::SharedPtr& handler, std::size_t bufferSize = 8192); + }; diff --git a/include/util/http_sender.h b/include/util/http_sender.h index d8aff9d..f012a43 100644 --- a/include/util/http_sender.h +++ b/include/util/http_sender.h @@ -20,12 +20,14 @@ #include "Poco/SharedPtr.h" #include "trsf/transfer_handler.h" + namespace qcloud_cos { class HttpSender { public: // trsf handler - static int SendRequest(const std::string& http_method, + static int SendRequest(const MultiUploadObjectReq *req, + const std::string& http_method, const std::string& url_str, const std::map& req_params, const std::map& req_headers, @@ -39,7 +41,8 @@ class HttpSender { bool is_check_md5 = false); // real trsf handler process - static int SendRequest(const std::string& http_method, + static int SendRequest(const MultiUploadObjectReq *req, + const std::string& http_method, const std::string& url_str, const std::map& req_params, const std::map& req_headers, diff --git a/src/op/file_upload_task.cpp b/src/op/file_upload_task.cpp index 2f51885..22dea78 100644 --- a/src/op/file_upload_task.cpp +++ b/src/op/file_upload_task.cpp @@ -19,7 +19,7 @@ FileUploadTask::FileUploadTask(const std::string& full_url, uint64_t recv_timeout_in_ms, unsigned char* pbuf, const size_t data_len) - : m_full_url(full_url), m_data_buf_ptr(pbuf), m_data_len(data_len), + : m_req(NULL), m_full_url(full_url), m_data_buf_ptr(pbuf), m_data_len(data_len), m_conn_timeout_in_ms(conn_timeout_in_ms), m_recv_timeout_in_ms(recv_timeout_in_ms), m_resp(""), m_is_task_success(false), m_is_resume(false), m_is_handler(false) { } @@ -31,7 +31,7 @@ FileUploadTask::FileUploadTask(const std::string& full_url, uint64_t recv_timeout_in_ms, unsigned char* pbuf, const size_t data_len) - : m_full_url(full_url), m_headers(headers), m_params(params), + : m_req(NULL), m_full_url(full_url), m_headers(headers), m_params(params), m_conn_timeout_in_ms(conn_timeout_in_ms), m_recv_timeout_in_ms(recv_timeout_in_ms), m_data_buf_ptr(pbuf), m_data_len(data_len), m_resp(""), m_is_task_success(false), m_is_resume(false), m_is_handler(false) { @@ -92,7 +92,7 @@ void FileUploadTask::UploadTask() { m_resp = ""; if(IsHandler()) { - m_http_status = HttpSender::SendRequest("PUT", m_full_url, m_params, m_headers, + m_http_status = HttpSender::SendRequest(m_req, "PUT", m_full_url, m_params, m_headers, body, m_handler, m_conn_timeout_in_ms, m_recv_timeout_in_ms, &m_resp_headers, &m_resp, &m_err_msg); }else { diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 3252c85..40b2f6f 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -1386,6 +1386,7 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, // Check the resume FileUploadTask* ptask = pptaskArr[task_index]; + ptask->m_req = const_cast(&req); ptask->SetHandler(true); if (resume_flag && !already_exist_parts[part_number].empty()) { @@ -1397,6 +1398,8 @@ CosResult ObjectOp::MultiThreadUpload(const MultiUploadObjectReq& req, SDK_LOG_INFO("upload data part:%lld has resumed", part_number); handler->UpdateProgress(read_len); + req.TriggerUploadProgressCallback(handler); + }else { ptask->m_handler = handler; FillUploadTask(upload_id, host, path, file_content_buf[task_index], read_len, diff --git a/src/trsf/transfer_handler.cpp b/src/trsf/transfer_handler.cpp index b6a0540..f0ce91f 100644 --- a/src/trsf/transfer_handler.cpp +++ b/src/trsf/transfer_handler.cpp @@ -114,8 +114,8 @@ namespace qcloud_cos { } - std::streamsize HandleStreamCopier::handleCopyStream(std::istream& istr, std::ostream& ostr, Poco::SharedPtr& handler, - std::size_t bufferSize) { + std::streamsize HandleStreamCopier::handleCopyStream(const MultiUploadObjectReq *req, std::istream& istr, std::ostream& ostr, + Poco::SharedPtr& handler, std::size_t bufferSize) { poco_assert (bufferSize > 0); Poco::Buffer buffer(bufferSize); @@ -130,6 +130,9 @@ namespace qcloud_cos { ostr.write(buffer.begin(), n); // update progress handler->UpdateProgress(n); + if(req) { + req->TriggerUploadProgressCallback(handler); + } if (istr && ostr) { istr.read(buffer.begin(), bufferSize); n = istr.gcount(); diff --git a/src/util/http_sender.cpp b/src/util/http_sender.cpp index 43e5b93..e03c7bc 100644 --- a/src/util/http_sender.cpp +++ b/src/util/http_sender.cpp @@ -36,7 +36,8 @@ namespace qcloud_cos { // Trsf handler -int HttpSender::SendRequest(const std::string& http_method, +int HttpSender::SendRequest(const MultiUploadObjectReq *req, + const std::string& http_method, const std::string& url_str, const std::map& req_params, const std::map& req_headers, @@ -50,7 +51,8 @@ int HttpSender::SendRequest(const std::string& http_method, bool is_check_md5) { std::istringstream is(req_body); std::ostringstream oss; - int ret = SendRequest(http_method, + int ret = SendRequest(req, + http_method, url_str, req_params, req_headers, @@ -449,7 +451,8 @@ int HttpSender::SendRequest(const std::string& http_method, } // Real trsf handler process -int HttpSender::SendRequest(const std::string& http_method, +int HttpSender::SendRequest(const MultiUploadObjectReq *objreq, + const std::string& http_method, const std::string& url_str, const std::map& req_params, const std::map& req_headers, @@ -523,7 +526,7 @@ int HttpSender::SendRequest(const std::string& http_method, // According to the copyStream to handle the process // TODO overwrite the copyStream insider to record // Poco::StreamCopier::copyStream(is, os); - HandleStreamCopier::handleCopyStream(is, os, handler); + HandleStreamCopier::handleCopyStream(objreq, is, os, handler); // 5. 接收返回 Poco::Net::StreamSocket& ss = session->socket(); From 1e1f42d5303e1540957825cb822f3121320f88d2 Mon Sep 17 00:00:00 2001 From: toranger Date: Tue, 16 Apr 2019 11:56:37 +0800 Subject: [PATCH 08/16] Add the status update callback --- demo/cos_demo.cpp | 4 ++++ include/request/object_req.h | 16 +++++++++++++++- include/trsf/transfer_handler.h | 2 ++ src/cos_api.cpp | 1 + src/op/object_op.cpp | 6 ++++++ src/trsf/transfer_handler.cpp | 25 +++++++++++++++++++++++-- 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/demo/cos_demo.cpp b/demo/cos_demo.cpp index d7c31af..136fe3b 100644 --- a/demo/cos_demo.cpp +++ b/demo/cos_demo.cpp @@ -468,7 +468,10 @@ void MultiUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_name, void uploadprogress(const MultiUploadObjectReq *req, Poco::SharedPtr &handler) { std::cout << "callback data is :" << handler->GetProgress() << std::endl; +} +void statusprogress(const MultiUploadObjectReq *req, Poco::SharedPtr &handler) { + std::cout << "callback status is :" << handler->GetStatusString() << std::endl; } // Upload object with handler @@ -478,6 +481,7 @@ void TransferUploadObject(qcloud_cos::CosAPI& cos, const std::string& bucket_nam object_name, local_file); req.SetRecvTimeoutInms(1000 * 60); req.SetUploadProgressCallback(uploadprogress); + req.SetTransferStatusUpdateCallback(statusprogress); qcloud_cos::MultiUploadObjectResp resp; Poco::SharedPtr handler = cos.TransferUploadObject(req, &resp); // The TransferUploadObject is the asynchronization api, can use WaitUntilFinish to block until finish. diff --git a/include/request/object_req.h b/include/request/object_req.h index d1428fe..fd5132f 100644 --- a/include/request/object_req.h +++ b/include/request/object_req.h @@ -667,6 +667,8 @@ class CompleteMultiUploadReq : public ObjectReq { class MultiUploadObjectReq : public ObjectReq { public: typedef void (*UploadProgressCallback)(const MultiUploadObjectReq *req, Poco::SharedPtr& handler); + typedef void (*TransferStatusUpdateCallback)(const MultiUploadObjectReq *req, Poco::SharedPtr& handler); + public: MultiUploadObjectReq(const std::string& bucket_name, const std::string& object_name, const std::string& local_file_path = "") @@ -676,6 +678,7 @@ class MultiUploadObjectReq : public ObjectReq { m_thread_pool_size = CosSysConfig::GetUploadThreadPoolSize(); mb_set_meta = false; m_progress_callback = NULL; + m_status_callback = NULL; // 默认打开当前路径下object的同名文件 if (local_file_path.empty()) { @@ -745,13 +748,22 @@ class MultiUploadObjectReq : public ObjectReq { void SetUploadProgressCallback(UploadProgressCallback callback) { m_progress_callback = callback; } - + + void SetTransferStatusUpdateCallback(TransferStatusUpdateCallback callback) { + m_status_callback = callback; + } + void TriggerUploadProgressCallback(Poco::SharedPtr& handler) const { if(m_progress_callback) { m_progress_callback(this, handler); } } + void TriggerTransferStatusUpdateCallback(Poco::SharedPtr& handler) const { + if(m_status_callback) { + m_status_callback(this, handler); + } + } private: std::string m_local_file_path; @@ -762,7 +774,9 @@ class MultiUploadObjectReq : public ObjectReq { std::string m_uploadid; public: + // These callback only used in the sync function TransferUploadObject UploadProgressCallback m_progress_callback; + TransferStatusUpdateCallback m_status_callback; }; diff --git a/include/trsf/transfer_handler.h b/include/trsf/transfer_handler.h index 7a1e8c7..9d2fbc8 100644 --- a/include/trsf/transfer_handler.h +++ b/include/trsf/transfer_handler.h @@ -94,6 +94,8 @@ namespace qcloud_cos{ // Get the current status of process, detail see the enum TransferStatus. TransferStatus GetStatus() const; + std::string GetStatusString() const; + void SetUploadID(const std::string& uploadid) { m_uploadid = uploadid; } // Get the init or resumed uploadid. std::string GetUploadID() const { return m_uploadid; } diff --git a/src/cos_api.cpp b/src/cos_api.cpp index ea61852..4f84d07 100644 --- a/src/cos_api.cpp +++ b/src/cos_api.cpp @@ -271,6 +271,7 @@ Poco::SharedPtr CosAPI::TransferUploadObject(const MultiUploadO g_threadpool->schedule(boost::bind(&TransferSubmit, &m_object_op, request, handler, response)); }else { handler->UpdateStatus(TransferStatus::FAILED); + request.TriggerTransferStatusUpdateCallback(handler); } // Return the handler outside. return handler; diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 40b2f6f..55ec6f6 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -499,6 +499,7 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, resp->CopyFrom(init_resp); handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = result; + req.TriggerTransferStatusUpdateCallback(handler); return result; } resume_uploadid = init_resp.GetUploadId(); @@ -507,6 +508,7 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, resp->CopyFrom(init_resp); handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = result; + req.TriggerTransferStatusUpdateCallback(handler); return result; } } @@ -525,6 +527,7 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, if (!handler->ShouldContinue()) { SDK_LOG_INFO("Multi upload object, canceled"); handler->UpdateStatus(TransferStatus::CANCELED); + req.TriggerTransferStatusUpdateCallback(handler); return result; } @@ -542,10 +545,12 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, ", resume_uploadid=%s", resume_uploadid.c_str()); handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = abort_result; + req.TriggerTransferStatusUpdateCallback(handler); return abort_result; } handler->UpdateStatus(TransferStatus::ABORTED); handler->m_result = result; + req.TriggerTransferStatusUpdateCallback(handler); return result; } @@ -566,6 +571,7 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, } resp->CopyFrom(comp_resp); handler->m_result = result; + req.TriggerTransferStatusUpdateCallback(handler); return result; } diff --git a/src/trsf/transfer_handler.cpp b/src/trsf/transfer_handler.cpp index f0ce91f..d7e99f6 100644 --- a/src/trsf/transfer_handler.cpp +++ b/src/trsf/transfer_handler.cpp @@ -28,6 +28,24 @@ namespace qcloud_cos { m_uploadid(""), m_cancel(false) {} + + static std::string GetNameForStatus(TransferStatus status) { + switch (status) { + case TransferStatus::NOT_START: + return "NOT_START"; + case TransferStatus::IN_PROGRESS: + return "IN_PROGRESS"; + case TransferStatus::CANCELED: + return "CANCELED"; + case TransferStatus::FAILED: + return "FAILED"; + case TransferStatus::COMPLETED: + return "COMPLETED"; + case TransferStatus::ABORTED: + return "ABORTED"; + } + } + void TransferHandler::UpdateProgress(uint64_t update_prog) { boost::lock_guard locker(m_lock_prog); @@ -105,14 +123,17 @@ namespace qcloud_cos { return !m_cancel; } - void TransferHandler::WaitUntilFinish() { boost::unique_lock locker(m_lock_stat); while (!IsFinishStatus(m_status)) { m_cond.wait(locker); } } - + + + std::string TransferHandler::GetStatusString() const { + return GetNameForStatus(m_status); + } std::streamsize HandleStreamCopier::handleCopyStream(const MultiUploadObjectReq *req, std::istream& istr, std::ostream& ostr, Poco::SharedPtr& handler, std::size_t bufferSize) { From 652e31ffed044ee0b5c378f4ef88b8502854ba28 Mon Sep 17 00:00:00 2001 From: toranger Date: Tue, 16 Apr 2019 16:41:45 +0800 Subject: [PATCH 09/16] rm the cout which used in test --- src/op/object_op.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 55ec6f6..7c3cd90 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -90,7 +90,6 @@ bool ObjectOp::check_single_part(const std::string& local_file_path, uint64_t of // Read data as a block: fin.read (data,local_part_size); - std::cout << "gount is :" << fin.gcount() << std::endl; fin.seekg (0, fin.beg); fin.close(); From bdaea0142ce65870f2f445c0fc26061ccf7f5f6b Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 18 Apr 2019 11:44:04 +0800 Subject: [PATCH 10/16] Fix the result status and the normal multiupload retry nil ptr --- src/op/file_upload_task.cpp | 2 +- src/op/object_op.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/op/file_upload_task.cpp b/src/op/file_upload_task.cpp index 22dea78..0629d46 100644 --- a/src/op/file_upload_task.cpp +++ b/src/op/file_upload_task.cpp @@ -117,7 +117,7 @@ void FileUploadTask::UploadTask() { } m_is_task_success = true; - } while (!m_is_task_success && loop <= kMaxRetryTimes && m_handler->ShouldContinue()); + } while (!m_is_task_success && loop <= kMaxRetryTimes); return; } diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 7c3cd90..7453e08 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -1087,6 +1087,9 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb task_fail_flag = true; break; } else { + // Notice the lseek when used in 32bit plat has the biggest offset limit 2G + // It fs can not seek over it and return NO_VAL(errno = 22 in windows), + // There use the _lseeki64() instead in windows and the * in linux. if (-1 == lseek(fd, vec_offset[task_index], SEEK_SET)) { std::string err_info = "down data, lseek ret=" + StringUtil::IntToString(errno) + ", offset=" @@ -1119,6 +1122,8 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb } if (task_fail_flag) { + // The result reused need to set status last + result.SetFail(); break; } } From 88fdf7d89d650cc1d4362d3e67eddb87b82dadb4 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 18 Apr 2019 15:36:24 +0800 Subject: [PATCH 11/16] Fix the mem leak history problem --- src/op/cos_result.cpp | 6 ++--- src/response/base_resp.cpp | 6 ++--- src/response/bucket_resp.cpp | 48 +++++++++++++++++------------------ src/response/object_resp.cpp | 36 +++++++++++++------------- src/response/service_resp.cpp | 6 ++--- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/op/cos_result.cpp b/src/op/cos_result.cpp index a6dd3d7..aac6a20 100644 --- a/src/op/cos_result.cpp +++ b/src/op/cos_result.cpp @@ -41,7 +41,7 @@ bool CosResult::ParseFromHttpResponse(const std::map& if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_INFO("Parse string to xml doc error, xml_body=%s", body.c_str()); SetErrorMsg(body); - delete cstr; + delete[] cstr; return false; } @@ -49,7 +49,7 @@ bool CosResult::ParseFromHttpResponse(const std::map& if (NULL == root) { SDK_LOG_INFO("Miss root node=Error, xml_body=%s", body.c_str()); SetErrorMsg(body); - delete cstr; + delete[] cstr; return false; } @@ -71,7 +71,7 @@ bool CosResult::ParseFromHttpResponse(const std::map& node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } diff --git a/src/response/base_resp.cpp b/src/response/base_resp.cpp index 3f84b15..aafe5ac 100644 --- a/src/response/base_resp.cpp +++ b/src/response/base_resp.cpp @@ -89,14 +89,14 @@ bool BaseResp::ParseFromACLXMLString(const std::string& body, if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("AccessControlPolicy"); if (NULL == root) { SDK_LOG_ERR("Miss root node=AccessControlPolicy, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -165,7 +165,7 @@ bool BaseResp::ParseFromACLXMLString(const std::string& body, } } - delete cstr; + delete[] cstr; return true; } diff --git a/src/response/bucket_resp.cpp b/src/response/bucket_resp.cpp index ee7f932..ff58d49 100644 --- a/src/response/bucket_resp.cpp +++ b/src/response/bucket_resp.cpp @@ -28,14 +28,14 @@ bool GetBucketResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node(kGetBucketRoot.c_str()); if (NULL == root) { SDK_LOG_ERR("Miss root node=kGetBucketRoot, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -98,7 +98,7 @@ bool GetBucketResp::ParseFromXmlString(const std::string& body) { node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -110,14 +110,14 @@ bool ListMultipartUploadResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node(kListMultipartUploadRoot.c_str()); if (NULL == root) { SDK_LOG_ERR("Miss root node=kListMultipartUploadRoot, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -204,7 +204,7 @@ bool ListMultipartUploadResp::ParseFromXmlString(const std::string& body) { node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -216,14 +216,14 @@ bool GetBucketReplicationResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node(kBucketReplicationRoot.c_str()); if (NULL == root) { SDK_LOG_ERR("Miss root node=BucketReplicationRoot, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -272,7 +272,7 @@ bool GetBucketReplicationResp::ParseFromXmlString(const std::string& body) { node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -284,14 +284,14 @@ bool GetBucketLifecycleResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("LifecycleConfiguration"); if (NULL == root) { SDK_LOG_ERR("Miss root node=LifecycleConfiguration, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -455,7 +455,7 @@ bool GetBucketLifecycleResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } @@ -471,14 +471,14 @@ bool GetBucketCORSResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("CORSConfiguration"); if (NULL == root) { SDK_LOG_ERR("Miss root node=CORSConfiguration, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -514,7 +514,7 @@ bool GetBucketCORSResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } @@ -526,14 +526,14 @@ bool GetBucketVersioningResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("VersioningConfiguration"); if (NULL == root) { SDK_LOG_ERR("Miss root node=VersioningConfiguration, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -555,7 +555,7 @@ bool GetBucketVersioningResp::ParseFromXmlString(const std::string& body) { node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -567,20 +567,20 @@ bool GetBucketLocationResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("LocationConstraint"); if (NULL == root) { SDK_LOG_ERR("Miss root node=LocationConstraint, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } m_location = root->value(); - delete cstr; + delete[] cstr; return true; } @@ -592,14 +592,14 @@ bool GetBucketObjectVersionsResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("ListVersionsResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=ListVersionsResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -670,7 +670,7 @@ bool GetBucketObjectVersionsResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } diff --git a/src/response/object_resp.cpp b/src/response/object_resp.cpp index a67aa26..420dcff 100644 --- a/src/response/object_resp.cpp +++ b/src/response/object_resp.cpp @@ -28,14 +28,14 @@ bool InitMultiUploadResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node(kInitiateMultipartUploadRoot.c_str()); if (NULL == root) { SDK_LOG_ERR("Miss root node=InitiateMultipartUploadResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -51,7 +51,7 @@ bool InitMultiUploadResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } @@ -62,14 +62,14 @@ bool CompleteMultiUploadResp::ParseFromXmlString(const std::string& body) { cstr[body.size()] = '\0'; if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=[%s]", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node(kCompleteMultiUploadRoot.c_str()); if (NULL == root) { SDK_LOG_ERR("Miss root node=ListBucketsResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -87,7 +87,7 @@ bool CompleteMultiUploadResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } @@ -158,14 +158,14 @@ bool ListPartsResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("ListPartsResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=ListPartsResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -243,7 +243,7 @@ bool ListPartsResp::ParseFromXmlString(const std::string& body) { } } - delete cstr; + delete[] cstr; return true; } @@ -259,14 +259,14 @@ bool PutObjectCopyResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("CopyObjectResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=CopyObjectResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -284,7 +284,7 @@ bool PutObjectCopyResp::ParseFromXmlString(const std::string& body) { node_name.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -296,14 +296,14 @@ bool UploadPartCopyDataResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("CopyPartResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=CopyObjectResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -319,7 +319,7 @@ bool UploadPartCopyDataResp::ParseFromXmlString(const std::string& body) { node_name.c_str()); } } - delete cstr; + delete[] cstr; return true; } @@ -349,14 +349,14 @@ bool DeleteObjectsResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("DeleteResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=DeleteResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -407,7 +407,7 @@ bool DeleteObjectsResp::ParseFromXmlString(const std::string& body) { node_name.c_str()); } } - delete cstr; + delete[] cstr; return true; } diff --git a/src/response/service_resp.cpp b/src/response/service_resp.cpp index b00c2a4..a06e567 100644 --- a/src/response/service_resp.cpp +++ b/src/response/service_resp.cpp @@ -28,14 +28,14 @@ bool GetServiceResp::ParseFromXmlString(const std::string& body) { if (!StringUtil::StringToXml(cstr, &doc)) { SDK_LOG_ERR("Parse string to xml doc error, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } rapidxml::xml_node<>* root = doc.first_node("ListAllMyBucketsResult"); if (NULL == root) { SDK_LOG_ERR("Miss root node=ListAllMyBucketsResult, xml_body=%s", body.c_str()); - delete cstr; + delete[] cstr; return false; } @@ -86,7 +86,7 @@ bool GetServiceResp::ParseFromXmlString(const std::string& body) { node_name.c_str(), body.c_str()); } } - delete cstr; + delete[] cstr; return true; } From d52bb5a6d1e40f338a8749787d7634a8ea5b0407 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 18 Apr 2019 16:36:03 +0800 Subject: [PATCH 12/16] Change the order to update status prevent the core of outside release resp --- src/op/object_op.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 7453e08..69baf3c 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -454,6 +454,8 @@ Poco::SharedPtr ObjectOp::CreateUploadHandler(const std::string } // Transfer call +// Be careful with the order to call update status, the resp is outside pointer, +// when outside resp is the temp param it will release lead the opreation core. CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, Poco::SharedPtr& handler, MultiUploadObjectResp* resp) { @@ -496,8 +498,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, if (!result.IsSucc()) { SDK_LOG_ERR("Multi upload object fail, check init mutli result."); resp->CopyFrom(init_resp); - handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = result; + handler->UpdateStatus(TransferStatus::FAILED); req.TriggerTransferStatusUpdateCallback(handler); return result; } @@ -505,8 +507,8 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, if (resume_uploadid.empty()) { SDK_LOG_ERR("Multi upload object fail, upload id is empty."); resp->CopyFrom(init_resp); - handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = result; + handler->UpdateStatus(TransferStatus::FAILED); req.TriggerTransferStatusUpdateCallback(handler); return result; } @@ -542,13 +544,13 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, if (!abort_result.IsSucc()) { SDK_LOG_ERR("Upload failed, and abort muliti upload also failed" ", resume_uploadid=%s", resume_uploadid.c_str()); - handler->UpdateStatus(TransferStatus::FAILED); handler->m_result = abort_result; + handler->UpdateStatus(TransferStatus::FAILED); req.TriggerTransferStatusUpdateCallback(handler); return abort_result; } - handler->UpdateStatus(TransferStatus::ABORTED); handler->m_result = result; + handler->UpdateStatus(TransferStatus::ABORTED); req.TriggerTransferStatusUpdateCallback(handler); return result; } @@ -563,13 +565,13 @@ CosResult ObjectOp::MultiUploadObject(const MultiUploadObjectReq& req, comp_req.SetPartNumbers(part_numbers); result = CompleteMultiUpload(comp_req, &comp_resp); + resp->CopyFrom(comp_resp); + handler->m_result = result; if (!result.IsSucc()) { handler->UpdateStatus(TransferStatus::FAILED); }else { handler->UpdateStatus(TransferStatus::COMPLETED); } - resp->CopyFrom(comp_resp); - handler->m_result = result; req.TriggerTransferStatusUpdateCallback(handler); return result; From b7ece19c9b6e13b100d9c70d2c9538800fd97ed7 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 18 Apr 2019 19:59:33 +0800 Subject: [PATCH 13/16] Add the binrary mode in windows open, fix the 0x0A into 0x0D 0x0A problem and the max seek offset 2G in 32 windows plat --- src/op/object_op.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 69baf3c..19fd726 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -36,6 +36,11 @@ #include "Poco/DigestStream.h" #include "Poco/StreamCopier.h" +#if defined(WIN32) +#define lseek _lseeki64 +#define write _write +#endif + namespace qcloud_cos { bool ObjectOp::IsObjectExist(const std::string& bucket_name, const std::string& object_name) { @@ -1007,7 +1012,8 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb // 3. 打开本地文件 std::string local_path = req.GetLocalFilePath(); #if defined(WIN32) - int fd = _open(local_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, + // The _O_BINARY is need by windows otherwise the x0A might change into x0D x0A + int fd = _open(local_path.c_str(), _O_BINARY | O_WRONLY | O_CREAT | O_TRUNC, _S_IREAD | _S_IWRITE); #else int fd = open(local_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, From 4045e2d729bd9e799da21565af39e918a3fcf5c4 Mon Sep 17 00:00:00 2001 From: toranger Date: Fri, 19 Apr 2019 12:19:16 +0800 Subject: [PATCH 14/16] Use _close with other windows api, testing --- src/op/object_op.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index 19fd726..f319f3a 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -38,7 +38,6 @@ #if defined(WIN32) #define lseek _lseeki64 -#define write _write #endif namespace qcloud_cos { @@ -1108,7 +1107,12 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb break; } +#if defined(WIN32) + if (-1 == _write(fd, file_content_buf[task_index], ptask->GetDownLoadLen())) { + +#else if (-1 == write(fd, file_content_buf[task_index], ptask->GetDownLoadLen())) { +#endif std::string err_info = "down data, write ret=" + StringUtil::IntToString(errno) + ", len=" + StringUtil::Uint64ToString(ptask->GetDownLoadLen()); @@ -1143,8 +1147,15 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb resp->SetEtag(head_resp.GetEtag()); } - // 4. 释放所有资源 + // Release resource + // The _close must be with the _open _write api in windows, otherwise there will occure the wrong content. + // Keep testing. Complete me. +#if defined(WIN32) + _close(fd); +#else close(fd); +#endif + for(unsigned i = 0; i < pool_size; i++){ delete [] file_content_buf[i]; delete pptaskArr[i]; From 23b4c30fa205837c675a80d396be84fd49f2036a Mon Sep 17 00:00:00 2001 From: toranger Date: Wed, 24 Apr 2019 16:33:42 +0800 Subject: [PATCH 15/16] Check each range size, in case of incomplete file with no error --- include/op/file_download_task.h | 3 ++- src/op/file_download_task.cpp | 18 ++++++++++++++++-- src/op/object_op.cpp | 10 ++++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/include/op/file_download_task.h b/include/op/file_download_task.h index 799e580..05bf5bc 100644 --- a/include/op/file_download_task.h +++ b/include/op/file_download_task.h @@ -40,7 +40,7 @@ class FileDownTask { void DownTask(); - void SetDownParams(unsigned char* pdatabuf, size_t datalen, uint64_t offset); + void SetDownParams(unsigned char* pdatabuf, size_t datalen, uint64_t offset, uint64_t target_size); std::string GetTaskResp(); @@ -69,6 +69,7 @@ class FileDownTask { int m_http_status; std::map m_resp_headers; std::string m_err_msg; + uint64_t m_target_size; }; } // namespace qcloud_cos diff --git a/src/op/file_download_task.cpp b/src/op/file_download_task.cpp index 9e7b829..14163cc 100644 --- a/src/op/file_download_task.cpp +++ b/src/op/file_download_task.cpp @@ -19,7 +19,7 @@ FileDownTask::FileDownTask(const std::string& full_url, m_conn_timeout_in_ms(conn_timeout_in_ms), m_recv_timeout_in_ms(recv_timeout_in_ms), m_offset(offset), m_data_buf_ptr(pbuf), - m_data_len(data_len), m_resp(""), m_is_task_success(false), m_real_down_len(0) { + m_data_len(data_len), m_resp(""), m_is_task_success(false), m_real_down_len(0), m_target_size(0) { } void FileDownTask::Run() { @@ -28,10 +28,11 @@ void FileDownTask::Run() { DownTask(); } -void FileDownTask::SetDownParams(unsigned char* pbuf, size_t data_len, uint64_t offset) { +void FileDownTask::SetDownParams(unsigned char* pbuf, size_t data_len, uint64_t offset, uint64_t target_size) { m_data_buf_ptr = pbuf; m_data_len = data_len; m_offset = offset; + m_target_size = target_size; } size_t FileDownTask::GetDownLoadLen() { @@ -80,6 +81,19 @@ void FileDownTask::DownTask() { size_t len = MIN(m_resp.length(), buf_max_size); memcpy(m_data_buf_ptr, m_resp.c_str(), len); m_real_down_len = len; + + // Must notice the receive timeout in Poco socket in SendRequest() + // The receiveResponse does not throw the expection of timeout, + // so there check the each part size outside, in case of the file incomplete. + if (len != m_target_size) { + SDK_LOG_ERR("FileDownload: url(%s) fail, might be reseted connection, offset:%lld, received:%d, expected: %lld", + m_full_url.c_str(), m_offset, len, m_target_size); + m_is_task_success = false; + // Clear the resp + m_resp = ""; + return; + } + m_is_task_success = true; m_resp = ""; return; diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index f319f3a..a241ea7 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -1064,8 +1064,14 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb SDK_LOG_DBG("down data, task_index=%d, file_size=%llu, offset=%llu", task_index, file_size, offset); FileDownTask* ptask = pptaskArr[task_index]; - - ptask->SetDownParams(file_content_buf[task_index], slice_size, offset); + uint64_t target_size = 0; + if (file_size - offset < slice_size) { + target_size = file_size - offset; + } else { + target_size = slice_size; + } + + ptask->SetDownParams(file_content_buf[task_index], slice_size, offset, target_size); tp.schedule(boost::bind(&FileDownTask::Run, ptask)); vec_offset[task_index] = offset; offset += slice_size; From eef3b6081f63d4fc1374910a56ed71e98f69ac12 Mon Sep 17 00:00:00 2001 From: toranger Date: Fri, 26 Apr 2019 14:27:27 +0800 Subject: [PATCH 16/16] Fix the macro with _WIN32 --- demo/cos_demo.cpp | 2 +- include/cos_defines.h | 4 ++-- include/util/simple_mutex.h | 24 ++++++++++++------------ src/op/object_op.cpp | 10 +++++----- src/util/auth_tool.cpp | 2 +- src/util/http_sender.cpp | 4 ++-- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/demo/cos_demo.cpp b/demo/cos_demo.cpp index 136fe3b..9dafb17 100644 --- a/demo/cos_demo.cpp +++ b/demo/cos_demo.cpp @@ -849,7 +849,7 @@ int main(int argc, char** argv) { // PrintResult(result, resp); // std::cout << "=========================================================" << std::endl; // } -#if defined(WIN32) +#if defined(_WIN32) system("pause"); #endif } diff --git a/include/cos_defines.h b/include/cos_defines.h index aa8e9f0..b8051e8 100644 --- a/include/cos_defines.h +++ b/include/cos_defines.h @@ -3,7 +3,7 @@ #include #include -#if !defined(WIN32) +#if !defined(_WIN32) #include #endif @@ -66,7 +66,7 @@ typedef enum cos_log_level { (level == COS_LOG_ERR) ? "[ERR] " : "[CRIT]") -#if defined(WIN32) +#if defined(_WIN32) #define COS_LOW_LOGPRN(level, fmt, ...) \ if (level <= CosSysConfig::GetLogLevel()) { \ if (CosSysConfig::GetLogOutType()== COS_LOG_STDOUT) { \ diff --git a/include/util/simple_mutex.h b/include/util/simple_mutex.h index 428dd99..8a0aac3 100644 --- a/include/util/simple_mutex.h +++ b/include/util/simple_mutex.h @@ -1,7 +1,7 @@ #ifndef SIMPLE_MUTEX_H #define SIMPLE_MUTEX_H -#if defined(WIN32) +#if defined(_WIN32) #include #include #else @@ -16,7 +16,7 @@ class SimpleMutex { public: SimpleMutex() { -#if defined(WIN32) +#if defined(_WIN32) m_mutex = CreateMutexA(NULL, FALSE, NULL); #else pthread_mutex_init(&m_mutex, NULL); @@ -24,7 +24,7 @@ class SimpleMutex { } ~SimpleMutex() { -#if defined(WIN32) +#if defined(_WIN32) CloseHandle(m_mutex); #else pthread_mutex_destroy(&m_mutex); @@ -32,14 +32,14 @@ class SimpleMutex { } void Lock() { -#if defined(WIN32) +#if defined(_WIN32) DWORD d = WaitForSingleObject(m_mutex, INFINITE); #else pthread_mutex_lock(&m_mutex); #endif } void Unlock() { -#if defined(WIN32) +#if defined(_WIN32) ReleaseMutex(m_mutex); #else pthread_mutex_unlock(&m_mutex); @@ -47,7 +47,7 @@ class SimpleMutex { } private: -#if defined(WIN32) +#if defined(_WIN32) HANDLE m_mutex; #else pthread_mutex_t m_mutex; @@ -76,7 +76,7 @@ class SimpleMutexLocker { class SimpleRWLock { public: SimpleRWLock() { -#if defined(WIN32) +#if defined(_WIN32) m_lock = CreateMutexA(NULL, FALSE, NULL); #else pthread_rwlock_init(&m_lock, NULL); @@ -84,7 +84,7 @@ class SimpleRWLock { } ~SimpleRWLock() { -#if defined(WIN32) +#if defined(_WIN32) CloseHandle(m_lock); #else pthread_rwlock_destroy(&m_lock); @@ -92,7 +92,7 @@ class SimpleRWLock { } void WriteLock() { -#if defined(WIN32) +#if defined(_WIN32) DWORD d = WaitForSingleObject(m_lock, INFINITE); #else pthread_rwlock_wrlock(&m_lock); @@ -101,7 +101,7 @@ class SimpleRWLock { } void ReadLock() { -#if defined(WIN32) +#if defined(_WIN32) DWORD d = WaitForSingleObject(m_lock, INFINITE); #else pthread_rwlock_rdlock(&m_lock); @@ -110,7 +110,7 @@ class SimpleRWLock { } void Unlock() { -#if defined(WIN32) +#if defined(_WIN32) ReleaseMutex(m_lock); #else pthread_rwlock_unlock(&m_lock); @@ -118,7 +118,7 @@ class SimpleRWLock { } private: -#if defined(WIN32) +#if defined(_WIN32) HANDLE m_lock; #else pthread_rwlock_t m_lock; diff --git a/src/op/object_op.cpp b/src/op/object_op.cpp index a241ea7..84f3d35 100644 --- a/src/op/object_op.cpp +++ b/src/op/object_op.cpp @@ -16,7 +16,7 @@ #include "threadpool/boost/threadpool.hpp" #include -#if defined(WIN32) +#if defined(_WIN32) #include #endif @@ -36,7 +36,7 @@ #include "Poco/DigestStream.h" #include "Poco/StreamCopier.h" -#if defined(WIN32) +#if defined(_WIN32) #define lseek _lseeki64 #endif @@ -1010,7 +1010,7 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb // 3. 打开本地文件 std::string local_path = req.GetLocalFilePath(); -#if defined(WIN32) +#if defined(_WIN32) // The _O_BINARY is need by windows otherwise the x0A might change into x0D x0A int fd = _open(local_path.c_str(), _O_BINARY | O_WRONLY | O_CREAT | O_TRUNC, _S_IREAD | _S_IWRITE); @@ -1113,7 +1113,7 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb break; } -#if defined(WIN32) +#if defined(_WIN32) if (-1 == _write(fd, file_content_buf[task_index], ptask->GetDownLoadLen())) { #else @@ -1156,7 +1156,7 @@ CosResult ObjectOp::MultiThreadDownload(const MultiGetObjectReq& req, MultiGetOb // Release resource // The _close must be with the _open _write api in windows, otherwise there will occure the wrong content. // Keep testing. Complete me. -#if defined(WIN32) +#if defined(_WIN32) _close(fd); #else close(fd); diff --git a/src/util/auth_tool.cpp b/src/util/auth_tool.cpp index 5a6fdab..1085bed 100644 --- a/src/util/auth_tool.cpp +++ b/src/util/auth_tool.cpp @@ -13,7 +13,7 @@ #include "util/http_sender.h" #include "cos_sys_config.h" -#if defined(WIN32) +#if defined(_WIN32) #define strncasecmp _strnicmp #define strcasecmp _stricmp #endif diff --git a/src/util/http_sender.cpp b/src/util/http_sender.cpp index e03c7bc..e2086e1 100644 --- a/src/util/http_sender.cpp +++ b/src/util/http_sender.cpp @@ -7,7 +7,7 @@ #include "util/http_sender.h" -#if defined(WIN32) +#if defined(_WIN32) #include #else #include @@ -617,7 +617,7 @@ int HttpSender::SendRequest(const MultiUploadObjectReq *objreq, // This differs significantly from the unix implementations which are accurate close to the microsecond. uint64_t HttpSender::GetTimeStampInUs() { // 构造时间 -#if defined(WIN32) +#if defined(_WIN32) time_t ltime; time(<ime); return (uint64_t)(ltime * 1000000);