jayden/LayaNative2.0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source

Browse Source
This commit is contained in:
lvfulong
2020-11-11 16:17:13 +08:00
parent 4d989f3ecb
commit bc4ca748de
2441 changed files with 623057 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Conch/source/common/downloadMgr/JCCurlWrap.cpp
+753
View File
@@ -0,0 +1,753 @@
/**
@file JCCurlWrap.cpp
@brief
@author hugao
@version 1.0
@date 2016_5_11
*/
#include "../downloadMgr/JCCurlWrap.h"
#include "../misc/JCGetClockExact.h"
#include "../util/Log.h"
#include "../util/JCCommonMethod.h"
#include "../downloadMgr/JCDownloadMgr.h"
#include "../util/JCLayaUrl.h"
#include <cmath>
#ifdef WIN32
#ifdef _DEBUG
#pragma comment(lib,"libcurld.lib")
#else
#pragma comment(lib,"libcurl.lib")
#endif
#pragma comment(lib,"WS2_32.lib")
#pragma comment(lib,"wldap32.lib")
#endif
/*
Curl 多线程应用：
1. curl_global_init 函数不是线程安全的，所以必须首先在主线程中调用，避免多线程 curl_easy_init
引起调用 curl_global_init 产生重入
2. 必须 curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L) 禁用掉 alarm 的使用，再多线程中 alarm + siglongjmp
会产生莫名奇妙的崩溃
3. curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1); 避免连接重用导致可能的 CLOSE_WAIT 连接过多造成效率问题。
*/
#define DEF_OPTTIMEOUT 1800
#define P_NOPOSTDATA nullptr
#define P_NOTIMEOUT 0
#define P_NOCONNTIMEOUT 0
#define P_PRIORITY_NORMAL 0
#define P_PRIORITY_LOW 1
#define P_NOLOCALFILE nullptr
namespace laya{
bool Curl::s_bUseCurlCookie = true;
std::vector<std::string> Curl::NoHeader;
static size_t save_header(void *ptr, size_t size, size_t nmemb, void *data) {
std::string* pRecv = (std::string*)data;
int sz = size*nmemb;
if (pRecv) {
(*pRecv).append((const char*)ptr, sz);
}
return (size_t)sz;
}
Curl::Curl(){
m_pCurlHandle = 0;
m_nOptTimeout = 0;
m_tmTaskBegin = 0;
m_tmLastProg = 0;
m_tmLastNotify = 0;
m_bStopAndRetry = false;
m_tmLastHasData = 0;
m_nResponseCode = 0;
}
Curl::~Curl()
{
Release();
}
void Curl::global_init() {
}
bool Curl::Init( ){
if( 0 == m_pCurlHandle )
{
m_pCurlHandle = curl_easy_init();
if( 0 == m_pCurlHandle )
return false;
}
if (JCDownloadMgr::s_curlProxyString.length() > 0) {
CURLcode ret = curl_easy_setopt(m_pCurlHandle, CURLOPT_PROXY, JCDownloadMgr::s_curlProxyString.c_str());
if (ret!= CURLE_OK) {
LOGE("setcurlproxy error");
}
}
return true;
}
void Curl::Release()
{
if( 0 != m_pCurlHandle )
{
curl_easy_cleanup( m_pCurlHandle );
m_pCurlHandle = 0;
}
}
bool Curl::_Prepare(){
if( 0 == m_pCurlHandle ){
m_pCurlHandle = curl_easy_init();
if( 0 == m_pCurlHandle )
return false;
}
//设置缺省值。
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(m_pCurlHandle, CURLOPT_ACCEPT_ENCODING, "");
curl_easy_setopt(m_pCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(m_pCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(m_pCurlHandle, CURLOPT_FOLLOWLOCATION, 1L);
//header是必须保存的
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADERFUNCTION, save_header);
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEHEADER, &m_strResponseHead);
//缺省打开进度回调
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(m_pCurlHandle, CURLOPT_XFERINFOFUNCTION, Curl::_ProgressCallback);
curl_easy_setopt(m_pCurlHandle, CURLOPT_XFERINFODATA, this);
return true;
}
int Curl::_WriteCallback( void *p_pDataBuffer, size_t p_size, size_t p_nmemb, void *p_pUserData ){
Curl *pCurl= (Curl*)p_pUserData;
if( 0 == pCurl )
return 0;
__Buffer *pBuffer = &pCurl->m_Buffer;
size_t iSize = p_size*p_nmemb;
pBuffer->AddData( p_pDataBuffer, iSize );
return iSize;
}
int Curl::_ProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow ){
//终止当前下载
if (JCDownloadMgr::m_bCancelTask)
return 1;
Curl *pCurl = (Curl*)clientp;
if( 0 == pCurl)
return 0;//没有数据也没事。
//std::uint32_t ulCur = GetClockExact::getInstance()->GetTimeMs();
double curtm = laya::tmGetCurms();
int iRet = 0;
if( 0 != pCurl ){
int dt = (int)(curtm - pCurl->m_tmTaskBegin);
//float speed = ((nWriteSz-pCurl->m_dLastDownedSize)/1024.0f);
float speed = dlnow / 1.024f / dt;
//因为dlnow可能一直为0，所以采用buffer实际接收到的数据为准
//LOGE("%f t:%f,n:%f,curlen:%d",(dlnow+ pContext->nLocalLen)/(dltotal+1+ pContext->nLocalLen),(float)dltotal,(float)dlnow,pContext->nLocalLen);
int nWriteSz = pCurl->m_Buffer.GetDataSize();
nWriteSz = nWriteSz > dlnow?nWriteSz :(int) dlnow;
if (nWriteSz <= 0)
return 0;
if( pCurl->m_dLastDownedSize==0 )
pCurl->m_dLastDownedSize = nWriteSz;
bool bNeedProg = false;
if( fabsf((pCurl->m_dDownedSize-nWriteSz))>1){
pCurl->m_tmLastHasData = curtm;
pCurl->m_dDownedSize = nWriteSz;
bNeedProg = true;
}
else {
//LOGE("nodata:%d", (int)(curtm - pCurl->m_tmLastHasData));
}
//如果超过10秒还没有任何反应，则认为断线了，需要重来。
if( laya::JCDownloadMgr::s_nNoResponseTimeout>0 && nWriteSz>0 &&
curtm-pCurl->m_tmLastHasData>laya::JCDownloadMgr::s_nNoResponseTimeout ){
LOGE("no received data over %d second,retry",laya::JCDownloadMgr::s_nNoResponseTimeout);
pCurl->m_bStopAndRetry=true;
return 1; //取消
}
//与大小相关的超时。这个每隔2秒提醒一次
int downloadtm = (int)((curtm-pCurl->m_tmTaskBegin)/1000);
int maxtm = (int)(dltotal/(20*1024));//按照20k的速度下载所需要的时间。
maxtm = maxtm<10?10:maxtm;
//LOGE("dltotal=%d,downloadtm=%d,slowtm=%d",(int)dltotal, downloadtm, maxtm);
if((dltotal>0||nWriteSz>0) && downloadtm >maxtm && (curtm-pCurl->m_tmLastNotify)>2000 ){//按照每秒10k的速度计算的话，就是网络慢
pCurl->m_tmLastNotify = curtm;
//TODO 超时提醒
}
if( bNeedProg || curtm-pCurl->m_tmLastProg>2000){
pCurl->m_dLastDownedSize= nWriteSz;
//如果dlnow=0，可能有问题，例如设置续传的时候如果值不对，失败了，还会调用这个函数，但是，
//total还是从0算，now 为0， 为了避免这种情况下的计算错误，直接忽略now为0的情况。
if(dltotal>0 && dlnow>0 && pCurl->m_pExtOnProg){
return pCurl->m_pExtOnProg((unsigned int)dltotal+ pCurl->m_nLocalFileLen,
(unsigned int)dlnow+pCurl->m_nLocalFileLen, speed, pCurl->m_pExtOnProgData);
}
}
/*
//如果已经3秒了还是0%
if( 0.0 == dlnow ){
if( ulCur - pCurl->m_ulTaskBegin > 3*1000 )
{
iRet = 1;
}
}
else if( dlnow > pCurl->m_dDownedSize )
{
pCurl->m_dDownedSize = dlnow;
pCurl->m_ulTaskBegin = ulCur;
}
*/
}
return iRet;
}
bool Curl::checkResult(const char *p_pszUrl) {
int opttimeout = m_nOptTimeout == 0 ? DEF_OPTTIMEOUT : m_nOptTimeout;
switch (m_nCurlRet) {
case CURLE_OPERATION_TIMEDOUT: // 操作超时，是不是应该重新提交？
{
auto curtm = laya::tmGetCurms();
int dt = (int)(curtm - m_tmTaskBegin);
LOGW("download error: timeout, dt=%d,opttimeout=%d", dt / 1000, (opttimeout - 2));
}
break;
case CURLE_PARTIAL_FILE: //可能是服务器关闭当前tcp连接了。
LOGW("download error:PARTIAL_FILE");
break;
case CURLE_COULDNT_CONNECT: //无法连接到服务器，可能服务器没开。
LOGW("download error: can't connect the server:%s", p_pszUrl);
break;
case CURLE_GOT_NOTHING:
LOGW("download error: the server has nothing responce %s",p_pszUrl);
break;
case CURLE_ABORTED_BY_CALLBACK:
if (m_bStopAndRetry) {
//TODO 重新尝试
m_bStopAndRetry = false;
}
break;
case CURLE_OK:break;
default:
LOGW("curl_easy_perform failed, code=%d\nsrc=%s", m_nCurlRet, p_pszUrl ? p_pszUrl : "");
break;
}
switch (m_nCurlRet){
case CURLE_OPERATION_TIMEDOUT: // 操作超时，是不是应该重新提交？
case CURLE_PARTIAL_FILE: //可能是服务器关闭当前tcp连接了。
case CURLE_GOT_NOTHING:
case CURLE_ABORTED_BY_CALLBACK:
case CURLE_OK:
{
bool httpok = false;
CURLcode iCode = curl_easy_getinfo(m_pCurlHandle, CURLINFO_RESPONSE_CODE, &m_nResponseCode);
if (CURLE_OK == iCode) {
if (m_nResponseCode >= 200 && m_nResponseCode<300) {
//OK
httpok = true;
}
}
char* pSvIP = NULL;
char* pLocIP = NULL;
curl_easy_getinfo(m_pCurlHandle, CURLINFO_PRIMARY_IP, &pSvIP);
curl_easy_getinfo(m_pCurlHandle, CURLINFO_LOCAL_IP, &pLocIP);
m_strSvAddr = pSvIP ? pSvIP : "";
m_strLocalAddr = pLocIP ? pLocIP : "";
if (Curl::s_bUseCurlCookie && m_nCurlRet== CURLE_OK) {
curl_slist* plistdata = NULL;
if (CURLE_OK == curl_easy_getinfo(m_pCurlHandle, CURLINFO_COOKIELIST, &plistdata)) {
if (plistdata) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_COOKIELIST, "FLUSH");
curl_slist_free_all(plistdata);
}
}
}
return httpok;
}
break;
default:break;
}
return false;
}
Curl& Curl::begin() {
//每次都要调用的
if (s_bUseCurlCookie) {
//这个必须每次都设置，因为curl内部有个data->change.cookielist
curl_easy_setopt(m_pCurlHandle, CURLOPT_COOKIELIST, "SESS");
curl_easy_setopt(m_pCurlHandle, CURLOPT_COOKIEFILE, m_strCookieFile.c_str());
}
m_nCurlRet = CURLE_FAILED_INIT;
m_nLocalFileLen = 0;
//这个必须在begin做，不能在end做。
m_strResponseHead.clear();
m_Buffer.ClearData();
m_tmTaskBegin = laya::tmGetCurms();
m_tmLastHasData = m_tmTaskBegin;
m_dDownedSize = 0.0;
m_dLastDownedSize = 0.0;
return *this;
}
Curl& Curl::end() {
if (m_pslist)
curl_slist_free_all(m_pslist);
m_pslist = nullptr;
return *this;
}
Curl& Curl::set_Url(const char* pUrl) {
//JCUrl url(pUrl);
//std::string encodeUrl = url.encode();
if (JCDownloadMgr::s_bEncodeURI) {
std::string uri = encodeURI(pUrl);
curl_easy_setopt(m_pCurlHandle, CURLOPT_URL, uri.c_str());
}
else
curl_easy_setopt(m_pCurlHandle, CURLOPT_URL, pUrl);
return *this;
}
Curl& Curl::set_CookieList(const char* pC) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_COOKIELIST, pC);
return *this;
}
Curl& Curl::set_CookieFile(const char* pFile) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_COOKIEFILE, pFile);
return *this;
}
Curl& Curl::set_GET() {
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPGET, 1L);
return *this;
}
Curl& Curl::set_POST() {
return *this;
}
void Curl::ApplyHeaders() {
if (m_pslist)
curl_slist_free_all(m_pslist);
m_pslist = nullptr;
int sz = m_headers.size();
if (sz <= 0) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, nullptr);
return ;
}
for (int i = 0; i<sz; i++) {
m_pslist = curl_slist_append(m_pslist, m_headers[i].c_str());
}
//CURLcode ret = curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, slist);
//curl_slist_free_all(slist);
m_headers.clear();
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, m_pslist);
}
Curl& Curl::set_FollowLocation(long v) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_FOLLOWLOCATION, v);
return *this;
}
Curl& Curl::set_Timeout(long v) {
m_nOptTimeout = v;
curl_easy_setopt(m_pCurlHandle, CURLOPT_TIMEOUT, v);
return *this;
}
Curl& Curl::set_ConnectTimeout(long v) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_CONNECTTIMEOUT, v);
return *this;
}
Curl& Curl::set_OnData(Curl::WRITECALLBACK func, void* pUserData) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEFUNCTION, func);
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEDATA, pUserData);
return *this;
}
Curl& Curl::set_EnableProg(bool b) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOPROGRESS, b?0:1L);
return *this;
}
Curl& Curl::set_PostData(const char* pPostData, int nPostLen) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_POSTFIELDS, pPostData);
curl_easy_setopt(m_pCurlHandle, CURLOPT_POSTFIELDSIZE, nPostLen);
curl_easy_setopt(m_pCurlHandle, CURLOPT_POST, 1L);
return *this;
}
static int downLoadPackage(void *ptr, size_t size, size_t nmemb, void *userdata) {
FILE *fp = (FILE*)userdata;
size_t written = fwrite(ptr, size, nmemb, fp);
return written;
}
static size_t NoWrite(void *ptr, size_t size, size_t nmemb, void *userdata) {
return size*nmemb;
}
long GetLocalFileLenth(const char* fileName) {
FILE* fp = fopen(fileName, "rb");
if (fp != NULL) {
fseek(fp, 0, SEEK_END);
long len = ftell(fp);
fclose(fp);
return len;
}
return 0;
}
std::string getHeadInfo(std::string& head, const char* InfoKey) {
std::string ret;
int st = head.find(InfoKey);
if (st <= 0)return ret;
st += strlen(InfoKey);
int ed = head.find("\r\n", st);
if (ed > st) {
ret.append(head.c_str() + st, ed - st);
}
return ret;
}
bool getRemoteFileInfo(CURL *m_pCurlHandle, const char* url, unsigned int& len, std::string& lastModified, std::string& etag) {
bool retval = false;
double dlen = 0.0;
JCUrl _url(url);
std::string encodeUrl = JCDownloadMgr::s_bEncodeURI ? encodeURI(url) : url;// _url.encode();
len = 0;
lastModified = "";
etag = "";
std::string headRet;
curl_easy_setopt(m_pCurlHandle, CURLOPT_URL, encodeUrl.c_str());
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADER, 1); //只要求header头
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOBODY, 1); //不需求body
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEHEADER, &headRet);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADERFUNCTION, save_header);
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEFUNCTION, NoWrite);
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEDATA, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L);//注意与Query的相互影响
curl_easy_setopt(m_pCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L);
//不要破坏下载进度
//curl_easy_setopt(m_pCurlHandle, CURLOPT_PROGRESSDATA, 0);
//curl_easy_setopt(m_pCurlHandle, CURLOPT_NOPROGRESS, 0);
//curl_easy_setopt(m_pCurlHandle, CURLOPT_PROGRESSFUNCTION, Curl::_ProgressCallback);
CURLcode ret = curl_easy_perform(m_pCurlHandle);
if (ret == CURLE_OK) {
if (CURLE_OK == curl_easy_getinfo(m_pCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dlen)) {
len = (unsigned int)dlen;
}
else {
LOGW("curl_easy_getinfo failed!\n");
}
long filetime = 0;
if (curl_easy_getinfo(m_pCurlHandle, CURLINFO_FILETIME, &filetime) == CURLE_OK) {
}
if (filetime <= 0) {
lastModified = getHeadInfo(headRet, "Last-Modified:");
}
etag = getHeadInfo(headRet, "ETag:");
//LOGE("%s%s", lastModified.c_str(), etag.c_str());
retval = true;
}
else {
retval = false;
}
//reset
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADER, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOBODY, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEHEADER, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADERFUNCTION, 0);
return retval;
}
/*
* 如果只是头的话，p_ppResBuff 也是头的内容。
*/
void Curl::query(const char *p_pszUrl, __Buffer **p_ppResBuff,
const char* p_pPostData, int p_nPostLen, //可以为0
bool p_bOnlyHeader, //一般是false
int p_nTimeout, int p_nConnTimeout, //为0则缺省
const std::vector<std::string>& p_vHeaders, //size()==0则忽略
const char* p_pLocalFile, //一边下载，一边保存，一般用在大文件下载，0则忽略
bool p_bChkRemoteChange //检查远端文件是否改变了，大文件用
){
if(p_ppResBuff)
*p_ppResBuff = 0;
m_nResponseCode = 0;
FILE* fp = nullptr;
do {
if (!_Prepare())
break;
begin();
//如果要保存到本地的处理
if (p_pLocalFile) {
m_nLocalFileLen = GetLocalFileLenth(p_pLocalFile);
if (p_bChkRemoteChange) {
//要检查远程是否改变了
unsigned int remoteLen=0;
std::string lastModified, etag;
//注意：这个会改变很多状态，所以需要在最前面执行。
bool br = getRemoteFileInfo(m_pCurlHandle, p_pszUrl, remoteLen, lastModified, etag);
//TODO 检查文件是否改变了，这个也可以在脚本中做
if (m_nLocalFileLen > 0 && (long)remoteLen == m_nLocalFileLen) {
m_nCurlRet = CURLE_OK;
m_nResponseCode = 200;
break;
}
}
fp = fopen(p_pLocalFile, "a+b");
if (!fp) {
LOGW("Open file error:%s", p_pLocalFile);
m_nCurlRet = CURLE_GOT_NOTHING;
break;
}
fseek(fp, 0, SEEK_END);
set_OnData(downLoadPackage,fp);
//if(m_nLocalFileLen>0)
curl_easy_setopt(m_pCurlHandle, CURLOPT_RESUME_FROM, m_nLocalFileLen);
}
else {
set_OnData(Curl::_WriteCallback, this);
curl_easy_setopt(m_pCurlHandle, CURLOPT_RESUME_FROM, 0);
}
m_nOptTimeout = p_nTimeout;
int opttimeout = p_nTimeout == 0 ? DEF_OPTTIMEOUT : p_nTimeout;
set_Url(p_pszUrl);
set_Header(p_vHeaders);
ApplyHeaders();
if (p_pPostData && p_nPostLen > 0) {
set_PostData(p_pPostData, p_nPostLen);
}
else {
set_GET();
}
set_OnlyHead(p_bOnlyHeader);
set_Timeout(opttimeout);
int connTimeout = p_nConnTimeout==0? 8: p_nConnTimeout;
set_ConnectTimeout(connTimeout);
m_nCurlRet = curl_easy_perform(m_pCurlHandle);
if (!checkResult(p_pszUrl)) {
m_Buffer.ClearData();
}
else if (p_bOnlyHeader) {
m_Buffer.ClearData();
m_Buffer.AddData(m_strResponseHead.c_str(), m_strResponseHead.length());
}
if(p_ppResBuff)
*p_ppResBuff = &m_Buffer;
//curl_easy_setopt(m_pCurlHandle, CURLOPT_WRITEHEADER, 0);
} while (false);
end();
if (fp)
fclose(fp);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_POSTFIELDS, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_POSTFIELDSIZE, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_POST, 0L);
//不能调用这个。会丢失之前的有效设置。例如session
//curl_easy_reset(m_pCurlHandle);
}
void Curl::query( const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pPostData, int p_nPostLen){
query(p_pszUrl, p_ppResBuff, p_pPostData, p_nPostLen,
false, P_NOTIMEOUT, P_NOCONNTIMEOUT, NoHeader, P_NOLOCALFILE, false);
}
Curl& Curl::set_OnlyHead(bool b) {
if (b) {
curl_easy_setopt(m_pCurlHandle, CURLOPT_HEADER, 1); //只要求header头
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOBODY, 1); //不需求body
set_EnableProg(false);
}
else {
curl_easy_setopt(m_pCurlHandle, CURLOPT_NOBODY, 0);
set_EnableProg(true);
}
return *this;
}
void Curl::downloadHeader(const char *p_pszUrl, __Buffer **p_ppResBuff, int p_nOptTimeout) {
query(p_pszUrl, p_ppResBuff, P_NOPOSTDATA, 0,
true, p_nOptTimeout, P_NOCONNTIMEOUT, NoHeader, P_NOLOCALFILE, false);
}
void Curl::downloadBigFile(const char *p_pszUrl, const char* p_pszLocal, int p_nOptTimeout) {
query(p_pszUrl, nullptr, P_NOPOSTDATA, 0,
false, p_nOptTimeout, P_NOCONNTIMEOUT, NoHeader, p_pszLocal, true);
}
void Curl::PostMultipart( const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pUserName, const char* p_pPostData, int p_nPostDataLen ){
std::vector<std::string> formdata;
formdata.push_back("username");
formdata.push_back(p_pUserName);
_PostMultipart(p_pszUrl,p_ppResBuff,formdata,"data",p_pPostData,p_nPostDataLen);
}
void Curl::_PostMultipart( const char *p_pszUrl, __Buffer **p_ppResBuff, std::vector<std::string>& formdata, const char* p_pDataName,const char* p_pPostData, int p_nPostDataLen){
{
int sz = formdata.size();
if(sz%2!=0){
LOGW("Curl::_PostMultipart transfer parameter is incorrect，formdata should be an even number，actually number is %d.",sz);
return;
}
}
*p_ppResBuff = 0;
/*
--------------------------d04f9c242ff033d8
Content-Disposition: form-data; name="format"
bin
--------------------------d04f9c242ff033d8
Content-Disposition: form-data; name="data"
?????????????????????????????????????????????????????
????????????????????
--------------------------d04f9c242ff033d8--
*/
curl_httppost *pPost = 0;
curl_httppost *lastptr = 0;
do{
if( !_Prepare() )
break;
begin();
{
int sz = formdata.size();
if (sz>0) {
for (int i = 0; i<sz / 2; i++) {
curl_formadd(&pPost, &lastptr,
CURLFORM_PTRNAME, formdata[i * 2].c_str(),
CURLFORM_PTRCONTENTS, formdata[i * 2 + 1].c_str(),
CURLFORM_END);
}
}
}
if (p_pPostData)
curl_formadd(&pPost, &lastptr, CURLFORM_PTRNAME, p_pDataName, CURLFORM_PTRCONTENTS, p_pPostData, CURLFORM_CONTENTSLENGTH, p_nPostDataLen, CURLFORM_END);
CURLcode iCode = CURLE_FAILED_INIT;
ApplyHeaders();
iCode = curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPPOST, pPost);
int opttimeout = m_nOptTimeout==0?DEF_OPTTIMEOUT:m_nOptTimeout;
set_Timeout(opttimeout);
set_ConnectTimeout(2L);
set_Url(p_pszUrl);
set_OnData(Curl::_WriteCallback, this);
set_EnableProg(true);
m_Buffer.ClearData();
m_tmTaskBegin = laya::tmGetCurms();// GetClockExact::getInstance()->GetTimeMs();
m_tmLastHasData = m_tmTaskBegin;
m_dDownedSize = 0.0;
m_dLastDownedSize = 0.0;
m_nCurlRet=curl_easy_perform(m_pCurlHandle);
if (!checkResult(p_pszUrl)) {
m_Buffer.ClearData();
}
*p_ppResBuff = &m_Buffer;
} while(0);
curl_formfree( pPost );
end();
//curl_easy_reset(m_pCurlHandle);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPPOST, 0);
}
//上传文件的回调函数
size_t put_data(void *ptr,size_t size,size_t nmemb,void *userdata){
return fread(ptr,size,nmemb,(FILE *)userdata);
}
void Curl::upload(const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pUserName,const char* p_pszFile){
*p_ppResBuff = 0;
curl_httppost *pPost = 0;
curl_httppost *lastptr = 0;
do{
if( !_Prepare() )
break;
CURLcode iCode = CURLE_FAILED_INIT;
begin();
curl_formadd(&pPost, &lastptr, CURLFORM_PTRNAME, "username", CURLFORM_PTRCONTENTS, p_pUserName, CURLFORM_END);
curl_formadd(&pPost, &lastptr, CURLFORM_COPYNAME, "upload", CURLFORM_FILE, p_pszFile, CURLFORM_END);
ApplyHeaders();
int opttimeout = m_nOptTimeout==0?DEF_OPTTIMEOUT:m_nOptTimeout;
set_Timeout(opttimeout);
set_ConnectTimeout(2L);
set_Url(p_pszUrl);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPPOST, pPost);
//iCode=curl_easy_setopt(m_pCurlHandle,CURLOPT_READFUNCTION,put_data); //读文件的回调
set_EnableProg(false);
//iCode=curl_easy_setopt(m_pCurlHandle, CURLOPT_UPLOAD, 1L);
//iCode=curl_easy_setopt(m_pCurlHandle, CURLOPT_READDATA, pf);
//iCode=curl_easy_setopt(m_pCurlHandle, CURLOPT_INFILESIZE, fsz);
//iCode=curl_easy_setopt(m_pCurlHandle, CURLOPT_INFILESIZE_LARGE, fsz);
m_Buffer.ClearData();
m_tmTaskBegin = laya::tmGetCurms();// GetClockExact::getInstance()->GetTimeMs();
m_tmLastHasData = m_tmTaskBegin;
m_dDownedSize = 0.0;
m_dLastDownedSize = 0.0;
m_nCurlRet=curl_easy_perform(m_pCurlHandle);
if(!checkResult(p_pszUrl)){
m_Buffer.ClearData();
}
*p_ppResBuff = &m_Buffer;
} while(0);
end();
//curl_easy_reset(m_pCurlHandle);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPHEADER, 0);
curl_easy_setopt(m_pCurlHandle, CURLOPT_HTTPPOST, 0);
}
void Curl::setCookieFile( const char* p_pszCookieFile ){
m_strCookieFile = p_pszCookieFile;
if(!s_bUseCurlCookie)
return;
//curl_easy_setopt( m_pCurlHandle, CURLOPT_COOKIESESSION,true); 开了这个就不行了
curl_easy_setopt( m_pCurlHandle, CURLOPT_COOKIEJAR, p_pszCookieFile);
curl_easy_setopt( m_pCurlHandle, CURLOPT_COOKIELIST, "SESS");
//curl_easy_setopt( m_pCurlHandle, CURLOPT_COOKIEFILE, p_pszCookieFile);
}
void Curl::setProxyString(const char* p_pszProxy) {
if (p_pszProxy && strlen(p_pszProxy) > 0) {
CURLcode ret = curl_easy_setopt(m_pCurlHandle, CURLOPT_PROXY, p_pszProxy);
if (ret != CURLE_OK) {
LOGW("setcurlproxy error");
}
}
}
}
Conch/source/common/downloadMgr/JCCurlWrap.h
+222
View File
@@ -0,0 +1,222 @@
/**
@file JCCurlWrap.h
@brief
@author hugao
@version 1.0
@date 2016_5_11
*/
#ifndef __JCCurlWrap_H__
#define __JCCurlWrap_H__
// 这个宏指示是在使用 CURL 的静态库，必须要有
#define CURL_STATICLIB
#ifdef __cplusplus
extern "C" {
#endif
#include <curl/curl.h>
#ifdef __cplusplus
}
#endif
#include <memory.h>
#include <list>
#include <vector>
#include <string>
#include <functional>
namespace laya{
class __Buffer{
public:
__Buffer(){
m_pBuffer = 0;
m_iDataSize = m_iBufferSize = 0;
}
~__Buffer(){
if( 0 != m_pBuffer ){
delete[] m_pBuffer;
m_pBuffer = 0;
}
}
char *SwapBuff( char *p_pBuff, size_t p_iBuffSize, size_t p_iDataSize=0 ){
char *pRet = m_pBuffer;
m_pBuffer = p_pBuff;
m_iBufferSize = p_iBuffSize;
m_iDataSize = p_iDataSize;
return pRet;
}
size_t GetDataSize(){
return m_iDataSize;
}
char *GetData(){
if( 0 == m_iDataSize )
return 0;
else
return m_pBuffer;
}
void ClearData(){
m_iDataSize = 0;
}
size_t GetFreeSize(){
return (m_iBufferSize-m_iDataSize);
}
void AddData( const void *p_pData, size_t p_iSize ){
for(;GetFreeSize()<p_iSize;){
_Expand();
}
memcpy( m_pBuffer+m_iDataSize, p_pData, p_iSize );
m_iDataSize += p_iSize;
}
void Finish(){
AddData( "", 1 );
}
private:
void _Expand(){
if (0 == m_iBufferSize)
m_iBufferSize = 16 * 1024;
else
m_iBufferSize = m_iBufferSize << 1;
char *pBuff = new char[m_iBufferSize];
if (0 != m_pBuffer){
if (m_iDataSize > 0){
memcpy(pBuff, m_pBuffer, m_iDataSize);
}
delete[] m_pBuffer;
}
m_pBuffer = pBuff;
}
private:
char *m_pBuffer;
size_t m_iDataSize;
size_t m_iBufferSize;
};
class Curl{
protected:
typedef int (*WRITECALLBACK)(void *p_pDataBuffer, size_t p_size, size_t p_nmemb, void *p_pUserData);
typedef int (*PROGRESSCALLBACK)(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow);
typedef size_t(*HEADERCALLBACK)(void *ptr, size_t size, size_t nmemb, void *data);
public:
//进度的回调。返回1则中止。参数是total，now, speed, userdata
typedef int(*FuncOnProg)(unsigned int, unsigned int, float, void*);
Curl();
~Curl();
static void global_init();
bool Init( );
void Release();
void setProgressCB(FuncOnProg cb, void* userData) {
m_pExtOnProg = cb;
m_pExtOnProgData = userData;
}
Curl& begin();
Curl& end();
Curl& set_GET();
Curl& set_POST();
Curl& set_Url(const char* pUrl);
Curl& set_CookieList(const char* pC);
Curl& set_CookieFile(const char* pFile);
//是否自动重定向
Curl& set_FollowLocation(long v);
//设置超时。单位秒
Curl& set_Timeout(long v);
// 连接超时时间 秒
Curl& set_ConnectTimeout(long v);
Curl& set_OnData(WRITECALLBACK func, void* pUserData);
Curl& set_EnableProg(bool b);
Curl& set_UserAgent(const char* pAgent);//这个也可以通过header来设置?
Curl& set_PostData(const char* pPostData, int nPostLen);
//只要header?
Curl& set_OnlyHead(bool b);
void ApplyHeaders();
//设置header，每次用完后会自动清理
void set_Header( const std::vector<std::string>& p_headers ){m_headers = p_headers;};
//检查当前任务的返回值。触发对应的回调。
bool checkResult(const char *p_pszUrl);
//通用的下载请求
void query(const char *p_pszUrl, __Buffer **p_ppResBuff,
const char* p_pPostData, int p_nPostLen, //可以为0
bool p_bOnlyHeader, //一般是false
int p_nTimeout, int p_nConnTimeout, //为0则缺省
const std::vector<std::string>& p_vHeaders, //size()==0则忽略
const char* p_pLocalFile, //一边下载，一边保存，一般用在大文件下载，0则忽略
bool p_bChkRemoteChange //检查远端文件是否改变了，大文件用
);
//如果有 p_pPostData,则在 CURLOPT_POSTFIELDS 发送数据
void query( const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pPostData, int p_nPostLen);
void downloadBigFile(const char *p_pszUrl, const char* p_pszLocal, int p_nOptTimeout);
void downloadHeader(const char *p_pszUrl, __Buffer **p_ppResBuff, int p_nOptTimeout);
//发送一个只有一个 data 字段的数据
void PostMultipart( const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pUserName, const char* p_pPostData, int p_nPostDataLen );
//上传文件。
void upload(const char *p_pszUrl, __Buffer **p_ppResBuff, const char* p_pUserName, const char* p_pszFile );
//发送一个带一个二进制buffer的formdata。
//二进制数据对应的formdata的名字是 p_pDataName
//formdata 是其他字段，每两个为一对。
void _PostMultipart( const char *p_pszUrl, __Buffer **p_ppResBuff, std::vector<std::string>& formdata, const char* p_pDataName,const char* p_pPostData, int p_nPostDataLen);
//设置curl的cookie文件
void setCookieFile( const char* p_pszCookieFile );
void setProxyString(const char* p_pszProxy);
static int _ProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow);
protected:
bool _Prepare();
static int _WriteCallback(void *p_pDataBuffer, size_t p_size, size_t p_nmemb, void *p_pUserData);
public:
static std::vector<std::string> NoHeader;
static bool s_bUseCurlCookie;
long m_nLocalFileLen = 0; //本地文件的长度。用于断点续传。
//下面是返回结果
std::string m_strSvAddr; //服务器地址
std::string m_strLocalAddr;
int m_nCurlRet = 0;
long m_nResponseCode = 0;
std::string m_strResponseHead; //响应的header
protected:
CURL * m_pCurlHandle;
int m_nOptTimeout;
__Buffer m_Buffer;
double m_tmTaskBegin;
curl_off_t m_dDownedSize;
curl_off_t m_dLastDownedSize;
double m_tmLastProg;
double m_tmLastNotify;
bool m_bStopAndRetry; //停止当前任务，重新尝试
std::vector<std::string> m_headers; //每一个都是 key:value
//传了一部分后没有反应的检测
double m_tmLastHasData; //上次接收到数据的时间
std::string m_strCookieFile;
struct curl_slist *m_pslist = nullptr;
FuncOnProg m_pExtOnProg = nullptr;
void* m_pExtOnProgData = nullptr;
};
}
//------------------------------------------------------------------------------
#endif //__JCCurlWrap_H__
//-----------------------------END FILE--------------------------------
Conch/source/common/downloadMgr/JCDownloadMgr.cpp
+952
View File
@@ -0,0 +1,952 @@
/**
@file JCDownloadMgr.cpp
@brief
@author hugao
@version 1.0
@date 2016_5_11
*/
#include "../downloadMgr/JCDownloadMgr.h"
#include "../misc/JCThreadPool.h"
#include <ctime>
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
#include "../downloadMgr/JCCurlWrap.h"
#endif
#include "../util/Log.h"
#include <algorithm>
#include "../util/JCCommonMethod.h"
#include <thread>
#include "../util/JCLayaUrl.h"
#include <atomic>
#include <stdlib.h>
#define TIMEOUTTRYNUM 40
#define CONNECTERRTRYNUM 80
#define P_NOPOSTDATA nullptr
#define P_NOTIMEOUT 0
#define P_NOCONNTIMEOUT 0
#define P_PRIORITY_NORMAL 0
#define P_PRIORITY_LOW 1
#define P_NOLOCALFILE nullptr
#include <cctype>
#include <iomanip>
#include <sstream>
#include <string>
namespace laya{
typedef std::recursive_mutex _mutex_t;
static std::uint32_t _l_ServerStop = 0;
int JCDownloadMgr::s_nNoResponseTimeout = 15000;
int JCDownloadMgr::s_nConnTimeout = 0;
int JCDownloadMgr::s_nOptTimeout = 0;
bool JCDownloadMgr::s_bEncodeURI=true;
bool JCDownloadMgr::m_bCancelTask = false;
JCDownloadMgr* gDownloadMgr = NULL;
int _OnProgress(unsigned int total, unsigned int now, float p_fSpeed, void* p_userData);
std::string JCDownloadMgr::s_curlProxyString = "";
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
class _QueryBase {
public:
virtual ~_QueryBase() {
int a = 10;
};
//false则重试。自己内部做sleep
virtual bool run(Curl* pCurl) { return true; };
};
class _QueryDownload:public _QueryBase {
protected:
const char* mpPostData = nullptr;
int mnPostDataLen = 0;
public:
short mnOptTimeout=0;
short mnConnTimeout = 0;
unsigned int mnTryNum = 0;
bool mbOnlyHeader = false; //只下载header
bool mbChkRemote = false;
std::string mUrl;
std::vector<std::string> mHeaders;
JCDownloadMgr::onProgressFunc mOnProg;
JCDownloadMgr::onEndFunc mOnEnd; //注意：传过来的指针不可以保存。
std::string mstrLocalFile; //直接写文件
_QueryDownload(const char* pszUrl) {
if(pszUrl)
mUrl = pszUrl;
}
void setPostData(const char* pData, int nLen) {
if (mpPostData)delete[] mpPostData;
mpPostData = new char[nLen];
memcpy((void*)mpPostData, pData, nLen);
mnPostDataLen = nLen;
}
virtual bool run(Curl* pCurl) {
if (JCDownloadMgr::m_bCancelTask) {
return true;
}
pCurl->setProgressCB(_OnProgress, this);
unsigned char *pContentBuff = nullptr;
size_t iContentBuffSize = 0;
bool bBigFile = mstrLocalFile.length() > 0;
LOGI("Download [%c%c]:%s", mbOnlyHeader ? 'H' : ' ', bBigFile ? 'B' : ' ', mUrl.c_str());
// 下载文件
JCUrl url(mUrl.c_str());
std::string encodeUrl = mUrl;// url.encode();
bool bHasQuery = url.m_Query.length() > 0;
char *pszUrlBuff = gDownloadMgr->getFinalUrl(encodeUrl.c_str());
__Buffer* _pContentBuffer = NULL;
pCurl->query(pszUrlBuff, &_pContentBuffer,
mpPostData, mnPostDataLen,
mbOnlyHeader,
mnOptTimeout, mnConnTimeout,
mHeaders,
bBigFile ? mstrLocalFile.c_str() : nullptr,
bBigFile);
if (_pContentBuffer) {
iContentBuffSize = _pContentBuffer->GetDataSize();
pContentBuff = (unsigned char*)_pContentBuffer->SwapBuff(0, 0);
}
if(mpPostData)
delete mpPostData;
mpPostData = NULL;
if (mOnEnd) {
if (pCurl->m_nCurlRet != CURLE_OK) {
//curl 执行失败
static std::string nullstr;
JCBuffer jb;
mOnEnd(jb, pCurl->m_strLocalAddr, pCurl->m_strSvAddr, pCurl->m_nCurlRet, pCurl->m_nResponseCode, nullstr);
}
else {
//bool bHttpOK = pCurl->m_nResponseCode >= 200 && pCurl->m_nResponseCode < 300;
LOGI("Download end:%d", pCurl->m_nResponseCode);
if (bBigFile || iContentBuffSize <= 0) {//大文件没有buffer
JCBuffer jb;
mOnEnd(jb, pCurl->m_strLocalAddr, pCurl->m_strSvAddr,
CURLE_OK, pCurl->m_nResponseCode,
pCurl->m_strResponseHead);
}
else {
if(pContentBuff && iContentBuffSize>0){
int sz = iContentBuffSize;
unsigned char* posthandleData = pContentBuff;
gDownloadMgr->postDownload(pszUrlBuff, posthandleData, sz);
iContentBuffSize = sz;
if (posthandleData != pContentBuff) {
delete[] pContentBuff;
pContentBuff = posthandleData;
}
}
JCBuffer buf((void*)pContentBuff, iContentBuffSize, false, true);
mOnEnd(buf, pCurl->m_strLocalAddr, pCurl->m_strSvAddr,
CURLE_OK, pCurl->m_nResponseCode,
pCurl->m_strResponseHead);
}
}
}
delete[] pszUrlBuff;
return true;
}
};
int _OnProgress(unsigned int total, unsigned int now, float p_fSpeed, void* p_userData) {
if (!gDownloadMgr)
return 1;
if (gDownloadMgr->m_bCancelTask)
return 1;
_QueryDownload* pQ = (_QueryDownload*)p_userData;
if (pQ && pQ->mOnProg) {
return pQ->mOnProg(total, now, p_fSpeed);
}
return 0;
}
class _QueryFunction :public _QueryBase {
public:
std::function<int(void*, unsigned char**, size_t*)> mTask;
virtual bool run(Curl* pCurl) {
unsigned char *pContentBuff = nullptr;
size_t iContentBuffSize = 0;
mTask(pCurl, &pContentBuff, &iContentBuffSize);
//TODO 要回调么
return true;
}
};
//设置cookie file
class _QuerySetCookieFile :public _QueryBase {
public:
//设置cookie文件。注意不一定什么时候执行，最好是在所有下载开始之前
std::string m_strSetCookieFile;
_QuerySetCookieFile(const char* pszCF) {
m_strSetCookieFile = pszCF;
}
virtual bool run(Curl* pCurl) {
pCurl->setCookieFile(m_strSetCookieFile.c_str()); //请求设置cookie文件
return true;
}
};
//等待清空队列。
class _QueryWaitEmpty :public _QueryBase {
protected:
#ifdef WIN32
std::atomic_uint32_t* mpNum = nullptr;
public:
_QueryWaitEmpty(std::atomic_uint32_t* pNum) {
mpNum = pNum;
}
#else
std::atomic_uint* mpNum = nullptr;
public:
_QueryWaitEmpty(std::atomic_uint* pNum) {
mpNum = pNum;
}
#endif
virtual bool run(Curl* pCurl) {
if(mpNum)
(*mpNum)--; //请求停止
return true;
}
};
#endif
//设置代理
class _QuerySetProxy :public _QueryBase {
protected:
std::string strProxy ;
public:
_QuerySetProxy(const char* pProxy) {
if(pProxy)
strProxy = pProxy;
}
virtual bool run(Curl* pCurl) {
if (strProxy.length() > 0) {
pCurl->setProxyString(strProxy.c_str());
}
return true;
}
};
void JCDownloadMgr::defCompleteFunc(JCBuffer&,const std::string&, const std::string&,
int,int,const std::string&){};
int JCDownloadMgr::defProgressFunc(unsigned int, unsigned int,float){return 0;}
JCDownloadMgr* JCDownloadMgr::getInstance(){
if( gDownloadMgr == NULL){
gDownloadMgr = new JCDownloadMgr();
//gDownloadMgr->init(2); //线程数目
}
return gDownloadMgr;
}
void JCDownloadMgr::delInstance(){
if(gDownloadMgr){
delete gDownloadMgr;
gDownloadMgr = NULL;
}
}
JCDownloadMgr::JCDownloadMgr(){
//_l_PoolQuery = new JCThreadPoolSingle<_Query_t *>();
m_nTryNumOpt = TIMEOUTTRYNUM;
m_nTryNumConn = CONNECTERRTRYNUM;
m_nStopNum = 0;
m_nDownloadTailType = 0; //缺省为不加随机数，防止瞬间回流造成服务器的负载太大。
m_nThreadNum = 0;
m_pCurDownloadingUrl=NULL;
m_nTimeout = 0;
}
JCDownloadMgr::~JCDownloadMgr(){
_l_ServerStop = 1;
clearAllAsyncTask();
//因为可能已经关掉了，没必要再跑一次
if(!m_bCancelTask)
stopCurTask();
//if(m_pCurDownloadingUrl)
// delete [] m_pCurDownloadingUrl;
/*
_l_PoolQuery->Stop();
if( _l_PoolQuery ){
delete _l_PoolQuery ;
_l_PoolQuery = NULL;
}
*/
}
void JCDownloadMgr::setOpt_optTimeout( int p_nTimeout){
m_nTimeout = p_nTimeout;
}
void JCDownloadMgr::setOpt_progCB(onProgressFunc p_ProgCb){
m_funcProgress = p_ProgCb;
}
void JCDownloadMgr::setOpt_completeCB(onEndFunc p_CompleteCb){
m_funcComplete = p_CompleteCb;
}
void JCDownloadMgr::setCookieFile( const char* p_pszCookieFile ){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
m_strCookieFile = p_pszCookieFile;
int num = m_ThreadPool.getThreadNum();
for( int i=0; i<num; i++){
_QuerySetCookieFile *pQuery = new _QuerySetCookieFile(p_pszCookieFile);
m_ThreadPool.sendToThread(pQuery,i);
}
#endif
}
void JCDownloadMgr::download(const char *p_pszURL, int p_nPriority,
onProgressFunc p_ProgCb, onEndFunc p_CompleteCb,
const char* p_pPostData, int p_nPostLen, //可以为0
bool p_bOnlyHeader, //一般是false
int p_nTimeout, int p_nConnTimeout, //为0则缺省
std::vector<std::string> p_vHeaders, //size()==0则忽略
const char* p_pLocalFile, //一边下载，一边保存，一般用在大文件下载，0则忽略
bool p_bChkRemoteChange //检查远端文件是否改变了，大文件用
) {
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
m_bCancelTask = false;
if (0 != p_pszURL) {
if (strlen(p_pszURL) <= 0) {
LOGE("Error! downloadMgr::download url len=0");
return;
}
static int nTh = 0;
int nThNum = m_ThreadPool.getThreadNum();// _l_PoolQuery->GetSize();
if (nThNum <= 0)
return;
_QueryDownload* pQuery = new _QueryDownload(p_pszURL);
pQuery->mOnEnd = p_CompleteCb;
pQuery->mOnProg = p_ProgCb;
pQuery->mnOptTimeout = p_nTimeout>0?p_nTimeout:s_nOptTimeout;
pQuery->mnConnTimeout = p_nConnTimeout > 0 ? p_nConnTimeout : s_nConnTimeout;
pQuery->mHeaders = p_vHeaders;
pQuery->mbOnlyHeader = p_bOnlyHeader;
if(p_pPostData)
pQuery->setPostData(p_pPostData, p_nPostLen);
if (p_pLocalFile) {
pQuery->mstrLocalFile = p_pLocalFile;
}
if (p_nPriority == 1 || nThNum == 1)
m_ThreadPool.sendToThread(pQuery, 0);
else {
nTh %= (nThNum - 1);
m_ThreadPool.sendToThread(pQuery, nTh + 1);
}
nTh++;
}
#endif
}
void JCDownloadMgr::download(const char* p_pszURL){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
download(p_pszURL,P_PRIORITY_NORMAL,
defProgressFunc,defCompleteFunc,
P_NOPOSTDATA,0,
false,
P_NOTIMEOUT,P_NOCONNTIMEOUT,
Curl::NoHeader,
P_NOLOCALFILE, false);
#endif
}
void JCDownloadMgr::download(const char* p_pszURL, int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb , int p_nOptTimeout ){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
download(p_pszURL, p_nPriority,
p_ProgCb, p_CompleteCb,
P_NOPOSTDATA, 0,
false,
p_nOptTimeout, P_NOCONNTIMEOUT,
Curl::NoHeader,
P_NOLOCALFILE, false);
#endif
}
void JCDownloadMgr::download(const char* p_pszURL, int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb, int p_nOptTimeout, int p_nConnTimeout) {
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
download(p_pszURL, p_nPriority,
p_ProgCb, p_CompleteCb,
P_NOPOSTDATA, 0,
false,
p_nOptTimeout, p_nConnTimeout,
Curl::NoHeader,
P_NOLOCALFILE, false);
#endif
}
void JCDownloadMgr::download(const char* p_pszURL, int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb , int p_nOptTimeout,
std::vector<std::string>& p_headers){
download(p_pszURL, p_nPriority,
p_ProgCb, p_CompleteCb,
P_NOPOSTDATA, 0,
false,
p_nOptTimeout, P_NOCONNTIMEOUT,
p_headers,
P_NOLOCALFILE, false);
}
void JCDownloadMgr::downloadBigFile(const char* p_pszURL, const char* p_pszLocal,
onProgressFunc p_ProgCb, onEndFunc p_CompleteCb, int p_nTryNum, int p_nOptTimeout) {
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
download(p_pszURL, P_PRIORITY_NORMAL,
p_ProgCb, p_CompleteCb,
P_NOPOSTDATA, 0,
false,
p_nOptTimeout, P_NOCONNTIMEOUT,
Curl::NoHeader,
p_pszLocal, true);
#endif
}
void JCDownloadMgr::postToDownloadThread(std::function<int(void*, unsigned char **,size_t*)> task, int p_nPriority) {
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
_QueryFunction *pQuery = new _QueryFunction();
pQuery->mTask = task;
static int nTh = 0;
int nThNum= m_ThreadPool.getThreadNum();// _l_PoolQuery->GetSize();
if (nThNum <= 0)
return;
if( p_nPriority==1 || nThNum==1)
m_ThreadPool.sendToThread(pQuery, 0);
else {
nTh %= (nThNum - 1);
m_ThreadPool.sendToThread(pQuery, nTh+1);
}
nTh++;
#endif
}
void JCDownloadMgr::getHeader(const char* p_pszUrl, onEndFunc p_CompleteCb, int p_nTryNum, int p_nOptTimeout) {
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
download(p_pszUrl, P_PRIORITY_NORMAL,
defProgressFunc, p_CompleteCb,
P_NOPOSTDATA, 0,
true,
p_nOptTimeout, P_NOCONNTIMEOUT,
Curl::NoHeader,
P_NOLOCALFILE, false);
#endif
}
void JCDownloadMgr::postData(const char* p_pszURL,const char* p_Buffer, int p_nLength,
onEndFunc p_completeCb){
std::vector<std::string> headers;
postData(p_pszURL,p_Buffer,p_nLength,p_completeCb,headers);
}
void JCDownloadMgr::postData(const char* p_pszURL,const char* p_Buffer, int p_nLength, onEndFunc p_completeCb,
std::vector<std::string>& p_headers){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
m_bCancelTask = false;
_QueryDownload* pQuery = new _QueryDownload(p_pszURL);
pQuery->mOnEnd = p_completeCb;
pQuery->setPostData(p_Buffer, p_nLength);
pQuery->mHeaders = p_headers;
m_ThreadPool.sendToThread(pQuery,0);
#endif
}
void JCDownloadMgr::clearAllAsyncTask(){
int num = m_ThreadPool.getThreadNum();
for( int i=0; i<num; i++){
//_l_PoolQuery->ClearDataOfThread(i);
m_ThreadPool.ClearDataOfThread(i);
}
}
void JCDownloadMgr::stopCurTask(){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
m_bCancelTask = true;
//等待所有的线程完成
int num = m_ThreadPool.getThreadNum();
for( int i=0; i<num; i++){
_QueryWaitEmpty *pQuery = new _QueryWaitEmpty(&m_nStopNum);
if (m_ThreadPool.sendToThread(pQuery, i)) {
m_nStopNum++;
}
}
int left = m_nStopNum;
//TODO 临时加一个超时，否则可能有线程问题导致退出卡死。
auto st = tmGetCurms();
while(left>0 && tmGetCurms()-st<3000){
left = m_nStopNum;
}
LOGI("stopCurTask end stopnum=%d",(int)m_nStopNum);
#endif
}
void JCDownloadMgr::setProxyString(const char* pProxy) {
int nThNum = m_ThreadPool.getThreadNum();// _l_PoolQuery->GetSize();
if (nThNum <= 0)
return;
for (int i = 0; i < nThNum; i++) {
_QuerySetProxy *pQuery = new _QuerySetProxy(pProxy);
m_ThreadPool.sendToThread(pQuery, i);
}
}
void JCDownloadMgr::init( int p_nWorkThreadNum ){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
Curl::global_init();
#endif
//等10ms是为了防止上一个线程还没有起来（假如存在的话）。
//否则：这里设置了_l_ServerStop=true会被线程起来后_l_ServerStop=false冲掉。
std::this_thread::sleep_for(std::chrono::milliseconds(10));
_l_ServerStop = true;//这个是为了能让之前的线程正确退出。
m_nThreadNum = p_nWorkThreadNum;
//m_pCurDownloadingUrl = new char* [p_nWorkThreadNum];
//memset(m_pCurDownloadingUrl,0,p_nWorkThreadNum*sizeof(char*));
m_ThreadPool.setThreadName("download thread");
m_ThreadPool.init(p_nWorkThreadNum, std::bind(&JCDownloadMgr::__WorkThread, this));
/*
_l_PoolQuery->Stop(); //TODO 如果加上stop会导致死锁。
for(int i=0;i<p_nWorkThreadNum;++i)
{
_l_PoolQuery->CreateThread( std::bind(&downloadMgr::__WorkThread, this) );
}
*/
}
inline void _addsplitchar(std::string& str){
if( str.at(0)=='/'){}
else if(str.at(0)=='\\') str.at(0)='/';
else
str=std::string("/")+str;
if( str.at(str.length()-1)=='/'){}
else if(str.at(str.length()-1)=='\\') str.at(str.length()-1)='/';
else
str+="/";
}
void JCDownloadMgr::setFinalReplacePath(const char* p_pszPath, const char* p_pszReplace){
m_strStubPath = p_pszPath?p_pszPath:"";
m_strStubReplace = p_pszReplace?p_pszReplace:"";
if(m_strStubPath.length()>0){
_addsplitchar(m_strStubPath);
_addsplitchar(m_strStubReplace);
}
LOGI("setFinalReplacePath:%s,%s", m_strStubPath.c_str(), m_strStubReplace.c_str());
}
void JCDownloadMgr::setDownloadTail(int type, const char* p_strTail){
if(type<0||type>2)
type=1;
m_nDownloadTailType = type;
if( m_nDownloadTailType==2){
if(p_strTail)
m_strDownloadTail = p_strTail;
else
m_nDownloadTailType=1;
}
}
void JCDownloadMgr::__WorkThread(){
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
_l_ServerStop = false;
_QueryBase *pQuery;
#ifdef WIN32
srand(timeGetTime());
#else
//TODO 随机数种子 在线程中访问全局随机数可能有问题
#endif
Curl _lCurl;
if( !_lCurl.Init() ){
printf("Curl init failed, thread exit\n");
return;
}
for(;0==_l_ServerStop;){
pQuery = 0;
//现在的waitdata返回false不再表示要退出。
//if( !m_ThreadPool.waitData(&pQuery ) )
// break;
//if( !_l_PoolQuery->WaitData( &pQuery ) )
// break;
void* pThread = m_ThreadPool.getCurThread();
int nID = m_ThreadPool.getCurThreadNo(pThread);
if( !m_ThreadPool.waitData(pThread,&pQuery ) ){
continue;
}
if( 0 != pQuery ){
while (! pQuery->run(&_lCurl)) {
LOGI("processQuery 再次尝试");
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
delete pQuery;
}
}
_lCurl.Release();
#endif
}
char* JCDownloadMgr::getFinalUrl(const char* p_pszUrl){
size_t iSize = strlen(p_pszUrl);
char *pszUrlBuff = new char[iSize+512];//这里大小会有问题么
const char* pQuery = strchr(p_pszUrl,'?');
bool bQuery = pQuery!=0;
//扩展名
std::string pathNoExt; //没有扩展名，没有query的部分
std::string ext;
const char* pExtDot = NULL;
const char* pExtEnd = NULL;
int extreplaceSz = m_vExtReplace.size();
if(extreplaceSz==0 && m_strStubPath.length()==0 ){
//简单情况
strcpy(pszUrlBuff,p_pszUrl);
}else{
//if(extreplaceSz)
{
int urllen = strlen(p_pszUrl);
pExtEnd = pQuery?(pQuery-1):(p_pszUrl+urllen-1);
char* pCurDt=(char*)pExtEnd;
while(pCurDt>p_pszUrl){
if(*pCurDt=='/'||*pCurDt==':'||*pCurDt=='\\')
break; //没有扩展名
if(*pCurDt=='.'){
pExtDot=pCurDt;
}
pCurDt--;
}
if( pExtDot==pExtEnd)
pExtDot=NULL;
if( pExtDot ){
pathNoExt.append(p_pszUrl,(pExtDot-p_pszUrl));
ext.append(pExtDot+1,(pExtEnd-pExtDot));
for( int i=0; i<extreplaceSz/2; i++){
if( m_vExtReplace[i*2]==ext){
ext = m_vExtReplace[i*2+1];
break;
}
}
}
else {
if (bQuery)
pathNoExt.append(p_pszUrl, (pQuery - p_pszUrl));
else
pathNoExt = p_pszUrl;
}
}
const char* pPathNoExt = pathNoExt.c_str();
if(m_strStubPath.length()>=3){
const char* pPos = strstr(pPathNoExt,m_strStubPath.c_str());
if(pPos){//&&(!bQuery||pPos<pQuery)){//不允许替换？后面的。
char* pCur = pszUrlBuff;
int l1 = pPos-pPathNoExt;
memcpy(pCur,pPathNoExt,l1); //拷贝替换路径之前的内容
pCur+=l1;
int l2 = m_strStubReplace.length();
memcpy(pCur,m_strStubReplace.c_str(),l2);//加上替换路径
pCur+=l2;
strcpy(pCur,pPos+m_strStubPath.length());//剩下的其他内容
}else{
strcpy(pszUrlBuff,pPathNoExt);
}
}else
strcpy(pszUrlBuff,pPathNoExt);
//加上扩展名
if(ext.length()){
strcat(pszUrlBuff,".");
strcat(pszUrlBuff,ext.c_str());
}
//加上其他的
if(pQuery)
strcat(pszUrlBuff,pQuery);
}
char strTail[512];
switch(m_nDownloadTailType){
case 1://随机数
sprintf( strTail, "%crnd=%d", bQuery?'&':'?', rand());
break;
case 2://指定
sprintf( strTail, "%c%s", bQuery?'&':'?', m_strDownloadTail.c_str());
break;
default://无
strTail[0]=0;
break;
}
strcat(pszUrlBuff,strTail);
return pszUrlBuff;
}
int JCDownloadMgr::__Download( void *p_pCurl, const char *p_pszUrl, unsigned char **p_ppBuff,
size_t *p_piSize, bool p_bHaveQuery){
/*
Curl::ErrorCode ret=Curl::EC_Unknown;
__Buffer *pResBuffer=NULL;
Curl * pCurl = (Curl*)p_pCurl;
*p_ppBuff = 0;
*p_piSize = 0;
char *pszUrlBuff = getFinalUrl(p_pszUrl);
LOGI("process:DownloadFile:%s",pszUrlBuff);
int i=0;
Curl::ErrorCode ecode=Curl::EC_Unknown;
int trynum = m_nTryNumOpt;
for(;i<trynum&&!m_bCancelTask;i++) //这里会进行多次尝试
{
ecode = pCurl->query( pszUrlBuff, &pResBuffer, Curl::__Type_GET );
if( ecode == Curl::EC_DownloadTimeout||ecode==Curl::EC_PartialFile){
LOGW("download timeout. i=%d",i);
int UNK=0;
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}else if(ecode==Curl::EC_Success){
*p_piSize = pResBuffer->GetDataSize();
*p_ppBuff = (unsigned char *)pResBuffer->SwapBuff( 0, 0 );
if( *p_piSize>0 && (void*)*p_ppBuff ){
int bufsz = (int)*p_piSize;
unsigned char* pBuff = *p_ppBuff;
postDownload(pszUrlBuff, pBuff, bufsz);
*p_piSize = bufsz;
if( *p_ppBuff != pBuff){
delete [] *p_ppBuff;
*p_ppBuff = pBuff;
}
}
break;
}else if(ecode==Curl::EC_ConnectError){
if( i==0){trynum=m_nTryNumConn; }
int UNK=0;
LOGW("connect timeout ! i=%d,trynum=%d,m_bCanceltask=%d", i, trynum, m_bCancelTask);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}else if(ecode==Curl::EC_404){
LOGE("download err:404");
*p_ppBuff=0;
*p_piSize=0;
break;
}else if(ecode==Curl::EC_Cancel){
*p_ppBuff=0;
*p_piSize=0;
break;
}
else{
//其他错误
LOGE("download unk error!");
break;
}
}
delete [] pszUrlBuff;
return ecode;
*/
return 0;
}
int JCDownloadMgr::__DownloadHeader(void *p_pCurl, const char *p_pszUrl, unsigned char **p_ppBuff, size_t *p_piSize,bool p_bHaveQuery, int p_nOptTimeout, int p_nTryNum) {
/* TODO 打开
Curl * pCurl = (Curl*)p_pCurl;
*p_ppBuff = 0;
*p_piSize = 0;
__Buffer *pResBuffer = NULL;
char *pszUrlBuff = getFinalUrl(p_pszUrl);
LOGI("流程：DownloadHeader:%s", pszUrlBuff);
//JCErrorHandleInterface* pEH = getErrorHandler();
int i = 0;
Curl::ErrorCode ecode = Curl::EC_Unknown;
int trynum = p_nTryNum;
for (; i<trynum&&!m_bCancelTask; i++) { //这里会进行多次尝试
ecode = pCurl->downloadHeader(pszUrlBuff, &pResBuffer, p_nOptTimeout);
if (ecode == Curl::EC_DownloadTimeout || ecode == Curl::EC_PartialFile) {
LOGW("download timeout. i=%d", i);
int UNK = 0;
//if (pEH) pEH->networkSlowly((float)UNK,pCurl->m_nResponseCode);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
else if (ecode == Curl::EC_Success) {
*p_piSize = pResBuffer->GetDataSize();
*p_ppBuff = (unsigned char *)pResBuffer->SwapBuff(0, 0);
break;
}
else if (ecode == Curl::EC_ConnectError) {
if (i == 0) { trynum = m_nTryNumConn; }
int UNK = 0;
LOGW("connect timeout ! i=%d,trynum=%d,m_bCanceltask=%d", i, trynum, m_bCancelTask);
//if (pEH) pEH->networkSlowly((float)UNK,pCurl->m_nResponseCode);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
else if (ecode == Curl::EC_404) {
LOGE("download err:404");
break;
}
else if (ecode == Curl::EC_Cancel) {
break;
}
else {
//其他错误
LOGE("download unk error!");
break;
}
}
delete[] pszUrlBuff;
return ecode;
*/
return 0;
}
int JCDownloadMgr::__DownloadBigFile(void *p_pCurl, const char *p_pszUrl,
const char* p_pszLocal, bool p_bHaveQuery, int p_nOptTimeout, int p_nTryNum){
/*
Curl * pCurl = (Curl*)p_pCurl;
char *pszUrlBuff = getFinalUrl(p_pszUrl);
LOGI("流程：DownloadFile:%s",pszUrlBuff);
int i=0;
Curl::ErrorCode ecode=Curl::EC_Unknown;
int trynum = p_nTryNum;
for(;i<trynum&&!m_bCancelTask;i++){ //这里会进行多次尝试
ecode = pCurl->downloadBigFile(pszUrlBuff, p_pszLocal, p_nOptTimeout );
if( ecode == Curl::EC_DownloadTimeout||ecode==Curl::EC_PartialFile){
LOGW("download timeout. i=%d",i);
int UNK=0;
//JCErrorHandleInterface* pEH = getErrorHandler();
//if (pEH) pEH->networkSlowly((float)UNK,pCurl->m_nResponseCode);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}else if(ecode==Curl::EC_Success){
break;
}else if(ecode==Curl::EC_ConnectError){
if( i==0){trynum=m_nTryNumConn; }
int UNK=0;
LOGW("connect timeout ! i=%d,trynum=%d,m_bCanceltask=%d", i, trynum, m_bCancelTask);
//JCErrorHandleInterface* pEH = getErrorHandler();
//if (pEH)
// pEH->networkSlowly((float)UNK, pCurl->m_nResponseCode);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}else if(ecode==Curl::EC_404){
LOGE("download err:404");
break;
}else if(ecode==Curl::EC_Cancel){
break;
}
else{
//其他错误
LOGE("download unk error!");
break;
}
}
delete [] pszUrlBuff;
return ecode;
*/
return 0;
}
bool JCDownloadMgr::postDownload(const char* p_pszUrl, unsigned char*& p_pBuff, int& p_nLen ){
if( !p_pszUrl)
return true;
std::string ext = getLowercaseExtOfUrl(p_pszUrl);
maskinfo mi = getMaskInfo(ext.c_str());
if(mi.key)
maskBuffer(mi, (char*)p_pBuff, p_nLen );
return true;
}
void JCDownloadMgr::setDownloadReplaceExt(const char* p_pszOrigin, const char* p_pszNew ){
if(p_pszOrigin==NULL || p_pszNew==NULL)
return;
for(int i=0,sz=m_vExtReplace.size(); i<sz/2; i++){
if(m_vExtReplace[i*2]==p_pszOrigin){
return;
}
}
m_vExtReplace.push_back(p_pszOrigin);
m_vExtReplace.push_back(p_pszNew);
}
void JCDownloadMgr::resetDownloadReplaceExt(){
m_vExtReplace.clear();
}
void JCDownloadMgr::setDownloadUnmask(const char* p_pszExt, unsigned char p_nKey, int p_nLen ){
if(!p_pszExt )
return;
std::string sExt = p_pszExt;
std::transform(sExt.begin(), sExt.end(),sExt.begin(),::tolower);
maskMap::iterator it = m_maskInfo.find(sExt);
if(it==m_maskInfo.end()){
if(p_nKey!=0){
maskinfo mi={(unsigned int)p_nKey, p_nLen};
m_maskInfo[sExt]=mi;
}
}else{
if(p_nKey==0){
m_maskInfo.erase(it);
}else{
maskinfo& info = (*it).second;
info.key = p_nKey;
info.len = p_nLen;
}
}
}
void JCDownloadMgr::resetDownloadUnmask(){
m_maskInfo.clear();
}
JCDownloadMgr::maskinfo JCDownloadMgr::getMaskInfo(const char* p_pszExt ){
maskinfo mi={0,0};
if(!p_pszExt )
return mi;
std::string sExt = p_pszExt;
std::transform(sExt.begin(), sExt.end(),sExt.begin(),::tolower);
maskMap::iterator it = m_maskInfo.find(sExt);
if(it==m_maskInfo.end()){
return mi;
}else{
return (*it).second;
}
}
void JCDownloadMgr::maskBuffer(JCDownloadMgr::maskinfo& p_mask, char* p_pBuff, int p_nLen ){
if(p_mask.key==0)
return;
int l = p_mask.len>p_nLen?p_nLen:p_mask.len;
unsigned char key = p_mask.key&0xff;
for( int i=0; i<l; i++){
p_pBuff[i]^=key;
}
}
std::string encodeURI(const char* value) {
if (!value)
return "";
std::ostringstream escaped;
escaped.fill('0');
escaped << std::hex;
unsigned char* pc = (unsigned char*)value;
unsigned char c = *pc;
bool afterQuery = false; //query中的'也要处理. query中的 |^ 不处理
while (c=*pc) {
c = *pc;
//标准是没有%的，但是chrome并不处理%，所以这里也不处理
if (isalnum(c) || c ==';' ||c=='/' || c =='?' || c == ':' || c == '@' || c == '&' || c == '=' || c == '+' ||
c == '$' || c == ',' || c == '-' || c == '_' || c == '.' || c == '!' || c == '~' || c == '*' ||
(!afterQuery && c == '\'' )|| c == '(' || c == ')' || c == '[' || c == ']' ||c=='%' ||
(afterQuery && c=='|') ||
(afterQuery && c=='^') ) {
if (!afterQuery&&c == '?')
afterQuery = true;
escaped << c;
pc++;
}
else {
escaped << std::uppercase;
escaped << '%' << std::setw(2) << int((unsigned char)c);
escaped << std::nouppercase;
pc++;
}
}
return escaped.str();
}
}
Conch/source/common/downloadMgr/JCDownloadMgr.h
+206
View File
@@ -0,0 +1,206 @@
/**
@file JCDownloadMgr.h
@brief
@author hugao
@version 1.0
@date 2016_5_11
*/
#ifndef __JCDownloadMgr_H__
#define __JCDownloadMgr_H__
#include "../buffer/JCBuffer.h"
#include <functional>
#include <thread>
#include <mutex>
#include <map>
#include <atomic>
#include "../misc/JCLayaThreadPool.h"
namespace laya{
#ifndef THIN_COMMON_WITHOUT_DOWNLOAD
class Curl;
#endif
struct _QueryBase;
class __Buffer;
/**
* @brief 只负责从网络下载资源。post数据。不管缓存。更新。
* 同时做转换层的工作：替换扩展名解密
*/
class JCDownloadMgr{
public:
struct maskinfo {
unsigned int key;
int len;
};
//回调
typedef std::function<int(unsigned int, unsigned int, float)> onProgressFunc; //返回1表示终止下载
//数据，localip，svip，curlret，httpret，header
typedef std::function<void(JCBuffer& buff, const std::string& localip,
const std::string& svip, int curlret, int httpret,
const std::string& httpresheader)> onEndFunc;
public:
static void defCompleteFunc(JCBuffer&, const std::string&, const std::string&,
int, int, const std::string&);
static int defProgressFunc(unsigned int, unsigned int,float);
~JCDownloadMgr();
static JCDownloadMgr* getInstance();
static void delInstance();
//初始化。p_nWorkThreadNum 表示开启几个下载线程。
void init( int p_nWorkThreadNum );
//设置操作超时
void setOpt_optTimeout( int p_nTimeout);
//设置超时后的重试次数。第一个是操作超时后的重试次数，第二个是连接超时后的重试次数。
void setOpt_tryNumOnTimeout(int p_nOptTry, int p_nConnTry){ m_nTryNumConn = p_nConnTry; m_nTryNumOpt=p_nOptTry;};
void getOpt_tryNumOnTimeout(int& p_nOptTry, int& p_nConnTry){ p_nConnTry=m_nTryNumConn ; p_nOptTry= m_nTryNumOpt;};
void setOpt_progCB(onProgressFunc p_ProgCb);
void setOpt_completeCB(onEndFunc p_CompleteCb);
/*
* @brief 完整参数的下载请求
* @prame p_nOptTimeout 下载超时。单位是秒， 要求>2, 如果为0，则使用缺省的。
*/
void download(const char *p_pszUrl,int p_nPriority,
onProgressFunc p_ProgCb, onEndFunc p_CompleteCb,
const char* p_pPostData, int p_nPostLen, //可以为0
bool p_bOnlyHeader, //一般是false
int p_nTimeout, int p_nConnTimeout, //为0则缺省
std::vector<std::string> p_vHeaders, //size()==0则忽略
const char* p_pLocalFile, //一边下载，一边保存，一般用在大文件下载，0则忽略
bool p_bChkRemoteChange=false //检查远端文件是否改变了，大文件用
);
void download(const char* p_pszURL);
virtual void download(const char* p_pszURL,int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb ,int p_nOptTimeout);
virtual void download(const char* p_pszURL, int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb, int p_nOptTimeout,int p_nConnTimeout);
virtual void download(const char* p_pszURL,int p_nPriority, onProgressFunc p_ProgCb,
onEndFunc p_CompleteCb , int p_nOptTimeout,
std::vector<std::string>& p_headers);
/*
* @brief
* 下载大文件，如果失败，回自动断点续传。
* 中间过程会直接写文件，而普通的下载是下载到内存中。
* @pram p_pszLocal 本地文件位置，包括文件名，要求目录必须存在，且有写的权限。
*/
void downloadBigFile(const char* p_pszURL, const char* p_pszLocal, onProgressFunc p_ProgCb, onEndFunc p_CompleteCb, int p_nTryNum, int p_nOptTimeout);
void getHeader(const char* p_pszUrl, onEndFunc p_CompleteCb, int p_nTryNum, int p_nOptTimeout);
void postData(const char* p_pszURL,const char* p_Buffer, int p_nLength, onEndFunc p_completeCb);
void postData(const char* p_pszURL,const char* p_Buffer, int p_nLength, onEndFunc p_completeCb,
std::vector<std::string>& p_headers);
//删除所有挂起的任务
void clearAllAsyncTask();
//结束当前的任务
void stopCurTask();
//设置代理
void setProxyString(const char* pProxy);
/*
* @brief 替换某个路径。
* 这个是为了解决有些cdn实在不好用而做的。如果路径中存在p_pstrPath（必须完全匹配）, 则用p_pszReplace替换。
* 这样可以导致一个动态目录，从而规避cdn问题。
* 注意替换后url的长度不要增加太多。与下面的tail结合不许比原始大小大512
*/
void setFinalReplacePath(const char* p_pstrPath, const char* p_pszReplace);
/*
* @brief 设置下载的时候附加的东西。
* 注意不要让url的长度增加太多。与上面的替换结合不许比原始大小大512
* @param type
* 0表示不附加
* 1随机数，形式为rnd=xxxx
* 2外面设置的字符串
* @param p_strTail
*/
void setDownloadTail(int type, const char* p_strTail);
//根据替换和添加规则，得到一个新的url，内部会重新分配空间，所以需要外部delete
char* getFinalUrl(const char* p_pszUrl);
void resetFinalReplacePath(){
m_strStubPath=""; m_strStubReplace="";
}
void resetDownloadTail(){ m_nDownloadTailType=0; m_strDownloadTail="";};
//下载的时候修改文件的扩展名，以防止中间路途被修改。
//由于没有进行多线程保护。这个必须要保证在程序刚开始的时候设置。
void setDownloadReplaceExt(const char* p_pszOrigin, const char* p_pszNew );
void resetDownloadReplaceExt();
//凡是扩展名为p_pszExt的文件,用key来进行异或操作，操作长度为p_nLen
void setDownloadUnmask(const char* p_pszExt, unsigned char p_nKey, int p_nLen );
void resetDownloadUnmask();
//如果不存在，则key为0
maskinfo getMaskInfo(const char* p_pszExt );
//如果buffer长度太小，则mask整个buffer
void maskBuffer( maskinfo& p_mask, char* p_pBuff, int p_nLen );
int getThreadNum(){ return m_nThreadNum; }
void setCookieFile( const char* p_pszCookieFile );
//优先级最低的给一个独立的线程
void postToDownloadThread(std::function<int(void*, unsigned char **,size_t*)> task, int p_nPriority);
protected:
//这是一个单件，不允许直接构造
JCDownloadMgr();
int __DownloadBigFile(void *p_pCurl, const char *p_pszUrl, const char* p_pszLocal, bool p_bHaveQuery, int p_nOptTimeout, int p_nTryNum);
int __DownloadHeader(void *p_pCurl, const char *p_pszUrl, unsigned char **p_ppBuff, size_t *p_piSize, bool p_bHaveQuery, int p_nOptTimeout, int p_nTryNum);
public://不对外的
void __WorkThread();
//下载完数据的处理。
bool postDownload(const char* p_pszUrl, unsigned char*& p_pBuff, int& p_nLen);
//返回错误代码。实际使用的是 Curl::ErrorCode
//p_bHaveQuery 是表示是否已经有Query了，因为需要决定添加?rnd=xx还是&rnd=xx
int __Download(void *p_pCurl, const char *p_pszUrl, unsigned char **p_ppBuff, size_t *p_piSize, bool p_bHaveQuery);
static bool m_bCancelTask;
public:
char** m_pCurDownloadingUrl; //正在下载的文件。调试用。如果某一个为0则表示正在空闲
static int s_nNoResponseTimeout; //如果下载中途没有数据了，则超过这么长时间之后就重来。
static std::string s_curlProxyString;
static int s_nConnTimeout;//超时的全局设置
static int s_nOptTimeout;
static bool s_bEncodeURI;
protected:
short m_nTryNumOpt;
short m_nTryNumConn;
short m_nThreadNum;
JCThreadPool<_QueryBase *> m_ThreadPool;
std::recursive_mutex m_CancelMutex;
#ifdef WIN32
std::atomic_uint32_t m_nStopNum; //完成个数
#else
std::atomic_uint m_nStopNum; //完成个数
#endif
std::string m_strStubPath;
std::string m_strStubReplace;
int m_nDownloadTailType;
std::string m_strDownloadTail;
std::vector<std::string> m_vExtReplace; //所有需要替换的扩展名。两个一对，第一个是原始的，第二个是新的
int m_nTimeout;
onProgressFunc m_funcProgress;
onEndFunc m_funcComplete;
std::string m_strCookieFile;
typedef std::map<std::string, maskinfo> maskMap;
maskMap m_maskInfo;
};
std::string encodeURI(const char* url);
}
//------------------------------------------------------------------------------
#endif //__JCDownloadMgr_H__
Conch/source/common/downloadMgr/JCHttpHeader.cpp
+244
View File
@@ -0,0 +1,244 @@
/**
@file JCHttpHeader.cpp
@brief
@author guo
@version 1.0
@date 2016_09_03
*/
#include "JCHttpHeader.h"
#include "../util/Log.h"
#include "../util/JCCommonMethod.h"
#include "../util/JCCrypto.h"
#include <stdlib.h>
#define peek(buf, c1) *(buf+1) != '\0' && *(buf+1) == c1
#define peek2(buf, c1, c2) \
*(buf+1) != '\0' && *(buf+1) == c1 \
*(buf+2) != '\0' && *(buf+2) == c2
#define iscrlf(p) (*p == '\r' && *(p + 1) == '\n')
#define notcrlf(p) (*p != '\r' && *(p + 1) != '\n')
#define notend(p) (*p != '\0')
#define end(p) (*p == '\0')
#define H3_DEFAULT_HTTP_VERSION "HTTP/1.0"
namespace laya{
/**
* This function returns a char pointer, which is the end of the request line.
*
* Return NULL if parse failed.
*/
const char * JCHttpHeader::request_line_parse(RequestHeader *header, const char *body, int bodyLength) {
// Parse the request-line
// http://tools.ietf.org/html/rfc2616#section-5.1
// Request-Line = Method SP Request-URI SP HTTP-Version CRLF
const char * p = body;
header->RequestLineStart = body;
while (notend(p) && !isspace(*p)) p++;
if (end(p) || iscrlf(p)) {
// set error
return NULL;
}
header->RequestMethod = body;
header->RequestMethodLen = p - body;
// Skip space
// parse RequestURI
while (isspace(*p) && notcrlf(p) && notend(p)) p++;
header->RequestURI = p;
while (!isspace(*p) && notcrlf(p) && notend(p)) p++;
header->RequestURILen = p - header->RequestURI;
// Skip space and parse HTTP-Version
if (iscrlf(p) || end(p)) {
header->HTTPVersion = H3_DEFAULT_HTTP_VERSION;
}
else {
while (isspace(*p) && notcrlf(p)) p++;
header->HTTPVersion = p;
while (!isspace(*p) && notcrlf(p)) p++;
header->HTTPVersionLen = p - header->HTTPVersion;
}
return p;
}
/**
* Parse header body
*/
int JCHttpHeader::request_header_parse(RequestHeader *header, const char *body, int bodyLength) {
const char *p = request_line_parse(header, body, bodyLength);
if (p == NULL) {
return ERR_REQUEST_LINE_PARSE_FAIL;
}
// should be ended with CR-LF
if (end(p)) return -1;
// skip CR-LF
iscrlf(p); p += 2;
if (end(p)) return 0;
header->HeaderSize = 0;
// Parse Header Fields Here
do {
HeaderField *field = &header->Fields[header->HeaderSize++];
// HeaderField *field = h3_header_field_new();
field->FieldName = p; // start of a header field name
while (notend(p) && *p != ':') p++;
field->FieldNameLen = p - field->FieldName;
p++; // skip ':'
// CRLF is not allowed here
if (end(p) || iscrlf(p)) return -1;
while (notend(p) && isspace(*p)) p++; // skip space
// CRLF is not allowed here
if (end(p) || iscrlf(p)) return -1;
field->Value = p;
while (notend(p) && notcrlf(p)) p++;
field->ValueLen = p - field->Value + 1;
iscrlf(p); p += 2;
#ifdef _DEBUG
printf("==> %.*s ==> %.*s\n", field->FieldNameLen, field->FieldName, field->ValueLen, field->Value);
#endif
// end of header
if (iscrlf(p)) return 0;
} while (notend(p) && header->HeaderSize < MAX_FILED_NUM);
return 0;
}
constexpr int _First_Len(char v1, int len) {
return (((int)v1) << 8) | len;
}
JCHttpHeader::JCHttpHeader(const char* pHeader) {
m_bCacheContrl = false;
request_header_parse(this, pHeader, strlen(pHeader));
for (int i = 0; i < (int)HeaderSize; i++) {
HeaderField& cfield = Fields[i];
int id = (cfield.FieldName[0] << 8)|(cfield.FieldNameLen & 0xff);
switch (id) {
case _First_Len('C',12)://Content-Type
if (memcmp(cfield.FieldName + 1, "ontent-Type",11) == 0) {
int a = 0;
}
break;
case _First_Len('C',13)://Cache-Control
if (memcmp(cfield.FieldName + 1, "ache-Control",12) == 0) {
parseCacheControl((char*)cfield.Value, cfield.ValueLen);
}
break;
case _First_Len('C',14)://Content-Length
if (memcmp(cfield.FieldName + 1, "ontent-Length", 13) == 0) {
int a = 0;
}
break;
case _First_Len('E',4):
break;
case _First_Len('E',7):
break;
}
}
}
int _getTokeHash(const char*& pDt) {
const char* pst = pDt;
while (*pDt != ',' && *pDt != ' '&& *pDt!='=' && *pDt != 0) {
*pDt++;
}
return JCBKDRHash::hashMem((const unsigned char*)pst, pDt-pst);
}
void _getSplit(const char*& pDt) {
while (*pDt == ',' || *pDt == ' ' ||*pDt=='=')
pDt++;
}
int _getNumber(const char*& pDt) {
const char* pSt = pDt;
char number[16] = { 0 };
while (*pDt >= '0' && *pDt <= '9') {
pDt++;
}
if (pDt - pSt > sizeof(number)) {
LOGE("parseCacheControl error");
*(int*)0 = 1;//报错
}
memcpy(number, pSt, pDt - pSt);
return atoi(number);
}
bool JCHttpHeader::parseCacheControl(char* p_pStr, int p_len) {
char* cp = new char[p_len + 1];
memcpy(cp, p_pStr, p_len);
cp[p_len] = 0; //最后用0比较容易处理。
const char* pDt = cp;
while (*pDt != 0) {//
int hash = _getTokeHash(pDt);
switch (hash){
case 0x7dbba7b2://max-age
pDt++;
m_CacheControl.maxage = _getNumber(pDt);
break;
case 0x4fbcb749://no-store
m_CacheControl.nostore = true;
_getSplit(pDt);
break;
case 0x3452662e://no-cache
m_CacheControl.nocache = true;
_getSplit(pDt);
break;
case 0x19fb0881://must-revalidate
_getSplit(pDt);
break;
default:
_getSplit(pDt);
break;
}
}
delete[] cp;
return true;
}
const char* JCHttpHeader::getLastModifyed() {//"Last-Modified"
return "";
}
const char* JCHttpHeader::getPragma() {//"Pragma"
return "";
}
const char* JCHttpHeader::getContentType() {//"Content-Type"
return "";
}
const char* JCHttpHeader::getETag() {//"ETag"
return "";
}
int64_t JCHttpHeader::getExpires() {
static const char* strExpires = "Expires";
if (m_tmExpires != 0)
return m_tmExpires;
for (int i = 0; i < (int)HeaderSize; i++) {
HeaderField& cfield = Fields[i];
if (memcmp(cfield.FieldName, strExpires, strlen(strExpires)) == 0) {
//return
}
}
return 0;
}
}
Conch/source/common/downloadMgr/JCHttpHeader.h
+92
View File
@@ -0,0 +1,92 @@
/**
@file JCHttpHeader.h
@brief
@author guo
@version 1.0
@date 2016_09_03
*/
#ifndef __JCHttpHeader_H__
#define __JCHttpHeader_H__
#include <stdint.h>
namespace laya {
/*
Host: github.com
^ ^
| |
| Value (ValueLen = 10)
|
| FieldName, FieldNameLen = 4
*/
typedef struct _HeaderField {
const char *FieldName = nullptr;
int FieldNameLen = 0;;
const char *Value = nullptr;
int ValueLen = 0;
}HeaderField;
typedef struct {
/**
* Pointer to start of the request line.
*/
const char * RequestLineStart = nullptr;
/**
* Pointer to the end of the request line
*/
const char * RequestLineEnd = nullptr;
/**
* Pointer to the start of the request method string
*/
const char * RequestMethod = nullptr;
int RequestMethodLen = 0;
const char * RequestURI = nullptr;
int RequestURILen = 0;
const char * HTTPVersion = nullptr;
int HTTPVersionLen = 0;
unsigned int HeaderSize = 0;
enum { MAX_FILED_NUM = 26 };
HeaderField Fields[MAX_FILED_NUM];
} RequestHeader;
class JCHttpHeader :public RequestHeader {
public:
enum _ERROR {
ERR_REQUEST_LINE_PARSE_FAIL = -1,
ERR_INCOMPLETE_HEADER,
ERR_UNEXPECTED_CHAR,
};
static const char * request_line_parse(RequestHeader *header, const char *body, int bodyLength);
static int request_header_parse(RequestHeader *header, const char *body, int bodyLength);
struct CacheContrl {
bool nocache = false;
bool nostore = false;
bool mustrevalidate = false;
int maxage = 0;
};
public:
JCHttpHeader(const char* pHeader);
const char* getLastModifyed();//"Last-Modified"
CacheContrl* getCacheContrl() { return &m_CacheControl; };//"Cache-Control"
const char* getPragma();//"Pragma"
const char* getContentType();//"Content-Type"
const char* getETag();//"ETag"
int64_t getExpires();//"Expires" 这个不用了
bool parseCacheControl(char* p_pStr, int p_nlen);
protected:
int64_t m_tmExpires = 0;
CacheContrl m_CacheControl;
bool m_bCacheContrl = false;
};
}
#endif