|
|
|
@ -0,0 +1,559 @@
|
|
|
|
|
package com.ann.utils.downLoader;
|
|
|
|
|
|
|
|
|
|
import com.alibaba.fastjson.JSON;
|
|
|
|
|
import com.ann.utils.downLoader.download.CallBackPara;
|
|
|
|
|
import com.ann.utils.downLoader.download.FileCheckPoints;
|
|
|
|
|
import com.ann.utils.downLoader.download.IDownCallBack;
|
|
|
|
|
import com.ann.utils.downLoader.download.IDownloadInfo;
|
|
|
|
|
import com.ann.utils.downLoader.download.MultiDownFile;
|
|
|
|
|
import com.ann.utils.downLoader.download.SaveFileItem;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.apache.commons.httpclient.util.URIUtil;
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
|
|
import javax.net.ssl.HostnameVerifier;
|
|
|
|
|
import javax.net.ssl.HttpsURLConnection;
|
|
|
|
|
import javax.net.ssl.SSLContext;
|
|
|
|
|
import javax.net.ssl.SSLSession;
|
|
|
|
|
import javax.net.ssl.SSLSocketFactory;
|
|
|
|
|
import javax.net.ssl.TrustManager;
|
|
|
|
|
import javax.net.ssl.X509TrustManager;
|
|
|
|
|
import java.io.ByteArrayOutputStream;
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.FileOutputStream;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.io.InputStream;
|
|
|
|
|
import java.io.UnsupportedEncodingException;
|
|
|
|
|
import java.net.HttpURLConnection;
|
|
|
|
|
import java.net.MalformedURLException;
|
|
|
|
|
import java.net.URL;
|
|
|
|
|
import java.net.URLConnection;
|
|
|
|
|
import java.net.URLEncoder;
|
|
|
|
|
import java.security.cert.CertificateException;
|
|
|
|
|
import java.sql.Timestamp;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Arrays;
|
|
|
|
|
import java.util.Date;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.regex.Matcher;
|
|
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
@Slf4j
|
|
|
|
|
public class HttpDownloader extends Thread {
|
|
|
|
|
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(HttpDownloader.class);
|
|
|
|
|
private IDownloadInfo info;
|
|
|
|
|
private int maxRetry = 5;
|
|
|
|
|
private IDownCallBack downCallBack;
|
|
|
|
|
private String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36";
|
|
|
|
|
|
|
|
|
|
public HttpDownloader(IDownloadInfo info, int maxRetry) {
|
|
|
|
|
this.info = info;
|
|
|
|
|
this.maxRetry = maxRetry;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 初始化回调方法
|
|
|
|
|
*
|
|
|
|
|
* @param downCallBack
|
|
|
|
|
*/
|
|
|
|
|
public void intiCallBack(IDownCallBack downCallBack) {
|
|
|
|
|
this.downCallBack = downCallBack;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public HttpDownloader(IDownloadInfo info) {
|
|
|
|
|
this.info = info;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public String[] removeArraysEmpty(String[] arr) {
|
|
|
|
|
return Arrays.stream(arr).filter(s -> !"".equals(s)).toArray(String[]::new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void run() {
|
|
|
|
|
CallBackPara para = new CallBackPara();
|
|
|
|
|
para.setId(info.getPair().id);
|
|
|
|
|
para.setFilename(info.getPair().localName);
|
|
|
|
|
para.setLocalpath(info.getPair().localPath);
|
|
|
|
|
para.setStarttime(getNewtime());
|
|
|
|
|
para.setFileStorageFormat(info.getPair().fileStorageFormat);
|
|
|
|
|
para.setDatatype(info.getPair().datatype);
|
|
|
|
|
String url = info.getPair().remoteUrl;
|
|
|
|
|
List<MultiDownFile> files = new ArrayList<>();
|
|
|
|
|
|
|
|
|
|
String[] urls = removeArraysEmpty(url.split("http://|https://"));
|
|
|
|
|
if (urls.length > 1) {
|
|
|
|
|
for (int i = 0; i < urls.length; i++) {
|
|
|
|
|
String urltemp = "";
|
|
|
|
|
if (url.indexOf("http://" + urls[i]) >= 0) {
|
|
|
|
|
urltemp = "http://" + urls[i];
|
|
|
|
|
}
|
|
|
|
|
if (url.indexOf("https://" + urls[i]) >= 0) {
|
|
|
|
|
urltemp = "https://" + urls[i];
|
|
|
|
|
}
|
|
|
|
|
for (String o : this.info.getPair().separators) {
|
|
|
|
|
urltemp = urltemp.replaceAll(o + "$", "");
|
|
|
|
|
}
|
|
|
|
|
files.add(new MultiDownFile(urltemp, info.getPair().localPath, "docustemp_" + i + "_" + info.getPair().localName));
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
files.add(new MultiDownFile(url, info.getPair().localPath, info.getPair().localName));
|
|
|
|
|
}
|
|
|
|
|
para.setFiles(files);
|
|
|
|
|
if (info.getPair().proxyurls != null) {
|
|
|
|
|
Pattern p = Pattern.compile(String.join("|", info.getPair().proxyurls));
|
|
|
|
|
for (MultiDownFile o : files) {
|
|
|
|
|
Matcher matcher = p.matcher(o.getRemoteUrl());
|
|
|
|
|
if (matcher.find()) {
|
|
|
|
|
o.setRemoteUrl(String.format(info.getPair().failurl, o.getRemoteUrl()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// URLHttpDownBootstrapBuilder builder=null;
|
|
|
|
|
// HttpDownBootstrap bootstrap;
|
|
|
|
|
try {
|
|
|
|
|
for (MultiDownFile file : files) {
|
|
|
|
|
// try {
|
|
|
|
|
// url = EncoderUrl(file.getRemoteUrl());
|
|
|
|
|
// } catch (Exception e) {
|
|
|
|
|
//
|
|
|
|
|
// }
|
|
|
|
|
downLoadFromUrl(url, file.getLocalName(), file.getLocalPath());
|
|
|
|
|
if (downCallBack != null) {
|
|
|
|
|
downCallBack.success(para);
|
|
|
|
|
}
|
|
|
|
|
//防止过快,第三链接无法支持
|
|
|
|
|
try {
|
|
|
|
|
Thread.sleep(100);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
// builder = HttpDownBootstrap.builder(url);
|
|
|
|
|
// builder.downConfig(new HttpDownConfigInfo()
|
|
|
|
|
// .setFilePath(file.getLocalPath())
|
|
|
|
|
// ).callBackPara(para);
|
|
|
|
|
// builder.response(new HttpResponseInfo(file.getLocalName()));
|
|
|
|
|
// bootstrap = builder.callback(new ConsoleHttpDownCallback()).build();
|
|
|
|
|
// bootstrap.start();
|
|
|
|
|
// bootstrap = null;
|
|
|
|
|
// builder = null;
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
// e.printStackTrace();
|
|
|
|
|
log.error("nio下载失败," + JSON.toJSONString(para) + ";失败信息:" + e.getMessage());
|
|
|
|
|
if (downCallBack != null) {
|
|
|
|
|
downCallBack.fail(para);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// finally {
|
|
|
|
|
// bootstrap = null;
|
|
|
|
|
// builder = null;
|
|
|
|
|
// }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public String EncoderUrl(String url) throws UnsupportedEncodingException {
|
|
|
|
|
String resultURL = "";
|
|
|
|
|
for (int i = 0; i < url.length(); i++) {
|
|
|
|
|
char charAt = url.charAt(i);
|
|
|
|
|
//只对汉字处理
|
|
|
|
|
if (isChineseChar(charAt)) {
|
|
|
|
|
String encode = URLEncoder.encode(charAt + "", "UTF-8");
|
|
|
|
|
resultURL += encode;
|
|
|
|
|
} else {
|
|
|
|
|
resultURL += charAt;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return resultURL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public boolean isChineseChar(char c) {
|
|
|
|
|
return String.valueOf(c).matches("[\u4e00-\u9fa5]");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理多级302等跳转
|
|
|
|
|
*
|
|
|
|
|
* @param uc
|
|
|
|
|
* @return
|
|
|
|
|
* @throws Exception
|
|
|
|
|
*/
|
|
|
|
|
private HttpURLConnection reload(HttpURLConnection uc) throws Exception {
|
|
|
|
|
|
|
|
|
|
HttpURLConnection huc = uc;
|
|
|
|
|
|
|
|
|
|
if (huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP
|
|
|
|
|
|| huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) {// 302, 301
|
|
|
|
|
String url = huc.getHeaderField("Location");
|
|
|
|
|
url = url.replace("\\", "/");
|
|
|
|
|
return reload((HttpURLConnection) new URL(url).openConnection());
|
|
|
|
|
}
|
|
|
|
|
return uc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception {
|
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
|
urlStr = urlStr.replace("\\", "/");
|
|
|
|
|
urlStr = URIUtil.encodePathQuery(urlStr);
|
|
|
|
|
URL url = new URL(urlStr);
|
|
|
|
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
|
|
|
|
try {
|
|
|
|
|
boolean useHttps = urlStr.toLowerCase().startsWith("https");
|
|
|
|
|
if (useHttps) {
|
|
|
|
|
HttpsURLConnection https = (HttpsURLConnection) conn;
|
|
|
|
|
trustAllHosts(https);
|
|
|
|
|
https.setHostnameVerifier(DO_NOT_VERIFY);
|
|
|
|
|
}
|
|
|
|
|
//设置超时间为3秒
|
|
|
|
|
//防止屏蔽程序抓取而返回403错误
|
|
|
|
|
conn.setRequestProperty("User-Agent", userAgent);
|
|
|
|
|
conn.setRequestProperty("Accept-Encoding", "identity");
|
|
|
|
|
conn.setConnectTimeout(8 * 1000);
|
|
|
|
|
conn.setReadTimeout(8 * 1000);
|
|
|
|
|
conn = reload(conn);
|
|
|
|
|
long length = conn.getContentLength();
|
|
|
|
|
if (length < 0) {
|
|
|
|
|
String values = conn.getHeaderField("Content-Length");
|
|
|
|
|
if (values != null && !values.isEmpty()) {
|
|
|
|
|
length = Long.parseLong(values);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// log.info(urlStr+" 文件大小:"+length);
|
|
|
|
|
InputStream inputStream = null;
|
|
|
|
|
if (conn.getResponseCode() >= 400) {
|
|
|
|
|
throw new Exception("文件不存在");
|
|
|
|
|
// inputStream = conn.getErrorStream();
|
|
|
|
|
} else {
|
|
|
|
|
inputStream = conn.getInputStream();
|
|
|
|
|
}
|
|
|
|
|
//得到输入流
|
|
|
|
|
//InputStream inputStream = conn.getInputStream();
|
|
|
|
|
//获取自己数组
|
|
|
|
|
byte[] getData = readInputStream(inputStream);
|
|
|
|
|
|
|
|
|
|
//文件保存位置
|
|
|
|
|
File saveDir = new File(savePath);
|
|
|
|
|
if (!saveDir.exists()) {
|
|
|
|
|
saveDir.mkdirs();
|
|
|
|
|
}
|
|
|
|
|
File file = new File(saveDir + File.separator + fileName);
|
|
|
|
|
|
|
|
|
|
FileOutputStream fos = new FileOutputStream(file);
|
|
|
|
|
fos.write(getData);
|
|
|
|
|
if (fos != null) {
|
|
|
|
|
fos.close();
|
|
|
|
|
}
|
|
|
|
|
if (inputStream != null) {
|
|
|
|
|
inputStream.close();
|
|
|
|
|
}
|
|
|
|
|
long end = System.currentTimeMillis();
|
|
|
|
|
logger.info("info:" + url + " download success;用时:" + (end - start) + "ms");
|
|
|
|
|
} catch (Exception ex) {
|
|
|
|
|
throw ex;
|
|
|
|
|
} finally {
|
|
|
|
|
// 断开连接,释放资源
|
|
|
|
|
conn.disconnect();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 从输入流中获取字节数组
|
|
|
|
|
*
|
|
|
|
|
* @param inputStream
|
|
|
|
|
* @return
|
|
|
|
|
* @throws IOException
|
|
|
|
|
*/
|
|
|
|
|
public byte[] readInputStream(InputStream inputStream) throws IOException {
|
|
|
|
|
byte[] buffer = new byte[1024];
|
|
|
|
|
int len = 0;
|
|
|
|
|
ByteArrayOutputStream bos = new ByteArrayOutputStream();
|
|
|
|
|
while ((len = inputStream.read(buffer)) != -1) {
|
|
|
|
|
bos.write(buffer, 0, len);
|
|
|
|
|
}
|
|
|
|
|
bos.close();
|
|
|
|
|
return bos.toByteArray();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public static FileCheckPoints initCheckPoint(int splitNum, long totalSize, long timeStamp) {
|
|
|
|
|
long[] startPos = new long[splitNum];
|
|
|
|
|
long[] endPos = new long[splitNum];
|
|
|
|
|
for (int i = 0, len = startPos.length; i < len; i++) {
|
|
|
|
|
long size = i * (totalSize / len);
|
|
|
|
|
startPos[i] = size;
|
|
|
|
|
// 设置最后一个结束点的位置
|
|
|
|
|
if (i == len - 1) {
|
|
|
|
|
endPos[i] = totalSize;
|
|
|
|
|
} else {
|
|
|
|
|
size = (i + 1) * (totalSize / len);
|
|
|
|
|
endPos[i] = size;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
FileCheckPoints chp = new FileCheckPoints();
|
|
|
|
|
chp.setEndPos(endPos);
|
|
|
|
|
chp.setStartPos(startPos);
|
|
|
|
|
chp.totalSize = totalSize;
|
|
|
|
|
chp.timestamp = timeStamp;
|
|
|
|
|
return chp;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private FileCheckPoints getInitedCheckPoint() {
|
|
|
|
|
long fileLength = -1;
|
|
|
|
|
long timeStamp = -1;
|
|
|
|
|
HttpURLConnection conn = null;
|
|
|
|
|
int stateCode = 0;
|
|
|
|
|
try {
|
|
|
|
|
URL url = new URL(this.info.getPair().remoteUrl);
|
|
|
|
|
conn = (HttpURLConnection) url.openConnection();
|
|
|
|
|
conn.setRequestProperty("Accept-Encoding", "identity");
|
|
|
|
|
HttpDownloader.RetriveSingleStream.setHeader(conn);
|
|
|
|
|
stateCode = conn.getResponseCode();
|
|
|
|
|
// 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK
|
|
|
|
|
if (stateCode != HttpURLConnection.HTTP_OK
|
|
|
|
|
&& stateCode != HttpURLConnection.HTTP_PARTIAL) {
|
|
|
|
|
logger.warn(info.getPair().remoteUrl + " #Error Code:# "
|
|
|
|
|
+ stateCode);
|
|
|
|
|
fileLength = -2;
|
|
|
|
|
} else if (stateCode >= 400) {
|
|
|
|
|
logger.warn(info.getPair().remoteUrl + " #Error Code:# "
|
|
|
|
|
+ stateCode);
|
|
|
|
|
fileLength = -2;
|
|
|
|
|
} else {
|
|
|
|
|
// 获取长度
|
|
|
|
|
fileLength = conn.getContentLengthLong();
|
|
|
|
|
timeStamp = conn.getLastModified();
|
|
|
|
|
logger.info(info.getPair().remoteUrl + " #FileLength:# "
|
|
|
|
|
+ fileLength);
|
|
|
|
|
}
|
|
|
|
|
} catch (MalformedURLException e) {
|
|
|
|
|
// e.printStackTrace();
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
// e.printStackTrace();
|
|
|
|
|
} finally {
|
|
|
|
|
if (conn != null) {
|
|
|
|
|
conn.disconnect();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
FileCheckPoints chp;
|
|
|
|
|
if (fileLength > 0) {
|
|
|
|
|
chp = initCheckPoint(info.getSplitNum(), fileLength, timeStamp);
|
|
|
|
|
chp.timestamp = timeStamp;
|
|
|
|
|
} else {
|
|
|
|
|
chp = new FileCheckPoints();
|
|
|
|
|
chp.statecode = stateCode;
|
|
|
|
|
}
|
|
|
|
|
return chp;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* bug fixed change the RandomAccessFile size
|
|
|
|
|
*
|
|
|
|
|
* @author burkun
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
protected static class RetriveSingleStream implements Runnable {
|
|
|
|
|
private boolean isDone = false;
|
|
|
|
|
private FileCheckPoints chp;
|
|
|
|
|
private int curIndex;
|
|
|
|
|
private SaveFileItem file;
|
|
|
|
|
private long startPos;
|
|
|
|
|
private long endPos;
|
|
|
|
|
byte[] buffer = new byte[1024 * 12];
|
|
|
|
|
private IDownloadInfo __info;
|
|
|
|
|
private int maxRetry;
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(RetriveSingleStream.class);
|
|
|
|
|
|
|
|
|
|
public boolean isDone() {
|
|
|
|
|
return isDone;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public RetriveSingleStream(IDownloadInfo info, FileCheckPoints chp,
|
|
|
|
|
int curIndex, int maxRetry) {
|
|
|
|
|
this.__info = info;
|
|
|
|
|
this.chp = chp;
|
|
|
|
|
this.curIndex = curIndex;
|
|
|
|
|
this.startPos = chp.getStartPos()[curIndex];
|
|
|
|
|
this.endPos = chp.getEndPos()[curIndex];
|
|
|
|
|
this.maxRetry = maxRetry;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void run() {
|
|
|
|
|
InputStream in = null;
|
|
|
|
|
HttpURLConnection conn = null;
|
|
|
|
|
int curRetry = 0;
|
|
|
|
|
|
|
|
|
|
while (curRetry < maxRetry && !isDone) {
|
|
|
|
|
try {
|
|
|
|
|
URL url = new URL(__info.getPair().remoteUrl);
|
|
|
|
|
conn = (HttpURLConnection) url.openConnection();
|
|
|
|
|
conn.setConnectTimeout(10000);
|
|
|
|
|
conn.setReadTimeout(30000);
|
|
|
|
|
setHeader(conn);
|
|
|
|
|
String property = "bytes=" + startPos + "-";
|
|
|
|
|
conn.setRequestProperty("RANGE", property);
|
|
|
|
|
logger.info(__info.getPair().localName + " #Block"
|
|
|
|
|
+ (curIndex + 1) + "# begin downloading...");
|
|
|
|
|
int length;
|
|
|
|
|
long counter = 0;
|
|
|
|
|
InputStream is = conn.getInputStream();
|
|
|
|
|
file = new SaveFileItem(__info.getPair().getLocalFullPath(), startPos);
|
|
|
|
|
//--bug fixed
|
|
|
|
|
file.setLength(__info.getCurCheckPoints().totalSize);
|
|
|
|
|
//--bug fixed
|
|
|
|
|
while (!isDone && startPos < endPos && (length = is.read(buffer)) > 0) {
|
|
|
|
|
startPos += file.write(buffer, 0, length);
|
|
|
|
|
counter += 1;
|
|
|
|
|
chp.getStartPos()[curIndex] = Math.min(startPos, endPos);
|
|
|
|
|
if (counter % 20 == 0) {
|
|
|
|
|
__info.writeInfo(chp);
|
|
|
|
|
logger.info(__info.getPair().remoteUrl + " #Block"
|
|
|
|
|
+ (curIndex + 1) + "# download "
|
|
|
|
|
+ getPercentage() + "%...");
|
|
|
|
|
Thread.yield();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
__info.writeInfo(chp);
|
|
|
|
|
isDone = true;
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
isDone = false;
|
|
|
|
|
logger.debug(__info.getPair().remoteUrl, e);
|
|
|
|
|
} finally {
|
|
|
|
|
if (!isDone) {
|
|
|
|
|
curRetry++;
|
|
|
|
|
logger.debug(__info.getPair().remoteUrl + " download failed, retry again!");
|
|
|
|
|
if (curRetry >= maxRetry) {
|
|
|
|
|
//保证循环跳出
|
|
|
|
|
isDone = true;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
curRetry = maxRetry;
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
if (in != null) {
|
|
|
|
|
in.close();
|
|
|
|
|
}
|
|
|
|
|
if (file != null) {
|
|
|
|
|
file.close();
|
|
|
|
|
}
|
|
|
|
|
if (conn != null) {
|
|
|
|
|
conn.disconnect();
|
|
|
|
|
}
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
logger.debug(__info.getPair().remoteUrl, e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public static void setHeader(URLConnection conn) {
|
|
|
|
|
conn.setRequestProperty(
|
|
|
|
|
"User-Agent",
|
|
|
|
|
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 BIDUBrowser/7.0 Safari/537.36");
|
|
|
|
|
conn.setRequestProperty("Accept-Language",
|
|
|
|
|
"en-us,en;q=0.7,zh-cn;q=0.3");
|
|
|
|
|
conn.setRequestProperty("Accept-Encoding", "utf-8");
|
|
|
|
|
conn.setRequestProperty("Accept-Charset",
|
|
|
|
|
"ISO-8859-1,utf-8;q=0.7,*;q=0.7");
|
|
|
|
|
conn.setRequestProperty("Keep-Alive", "300");
|
|
|
|
|
conn.setRequestProperty("connnection", "keep-alive");
|
|
|
|
|
// conn.setRequestProperty("If-Modified-Since",
|
|
|
|
|
// "Fri, 02 Jan 2009 17:00:05 GMT");
|
|
|
|
|
// conn.setRequestProperty("If-None-Match",
|
|
|
|
|
// "\"1261d8-4290-df64d224\"");
|
|
|
|
|
conn.setRequestProperty("Cache-conntrol", "max-age=0");
|
|
|
|
|
conn.setRequestProperty("Referer", "http://www.baidu.com");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private int getPercentage() {
|
|
|
|
|
long total = 0;
|
|
|
|
|
for (int i = 0; i < chp.getSplit(); i++) {
|
|
|
|
|
total += chp.getEndPos()[i] - chp.getStartPos()[i];
|
|
|
|
|
}
|
|
|
|
|
return (int) ((chp.totalSize - total) * 100 / chp.totalSize);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private Timestamp getNewtime() {
|
|
|
|
|
Date now = new Date();
|
|
|
|
|
Timestamp timestamp = new Timestamp(now.getTime());
|
|
|
|
|
return timestamp;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// public static void main(String[] args) {
|
|
|
|
|
// String url="http://ss,ss,https://bbbbb;http://ccc";
|
|
|
|
|
// List<String> separators = new ArrayList<>();
|
|
|
|
|
// separators.add(",");
|
|
|
|
|
// separators.add(";");
|
|
|
|
|
// String[] urls = url.split("http://|https://");
|
|
|
|
|
// for (int i = 0; i < urls.length; i++) {
|
|
|
|
|
// String urltemp = "";
|
|
|
|
|
// if (url.indexOf("http://" + urls[i]) >= 0) {
|
|
|
|
|
// urltemp="http://" + urls[i];
|
|
|
|
|
// }
|
|
|
|
|
// if (url.indexOf("https://" + urls[i]) >= 0) {
|
|
|
|
|
// urltemp="https://" + urls[i];
|
|
|
|
|
// }
|
|
|
|
|
// for(String o:separators){
|
|
|
|
|
// urltemp=urltemp.replaceAll(o+"$", "");
|
|
|
|
|
// }
|
|
|
|
|
// System.out.println(urltemp);
|
|
|
|
|
// }
|
|
|
|
|
// }
|
|
|
|
|
/**
|
|
|
|
|
* 覆盖java默认的证书验证
|
|
|
|
|
*/
|
|
|
|
|
private final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
|
|
|
|
|
@Override
|
|
|
|
|
public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
|
|
|
|
|
return new java.security.cert.X509Certificate[]{};
|
|
|
|
|
}
|
|
|
|
|
}};
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 设置不验证主机
|
|
|
|
|
*/
|
|
|
|
|
private final HostnameVerifier DO_NOT_VERIFY = new HostnameVerifier() {
|
|
|
|
|
public boolean verify(String hostname, SSLSession session) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 信任所有
|
|
|
|
|
*
|
|
|
|
|
* @param connection
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
private SSLSocketFactory trustAllHosts(HttpsURLConnection connection) {
|
|
|
|
|
SSLSocketFactory oldFactory = connection.getSSLSocketFactory();
|
|
|
|
|
try {
|
|
|
|
|
SSLContext sc = SSLContext.getInstance("TLS");
|
|
|
|
|
sc.init(null, trustAllCerts, new java.security.SecureRandom());
|
|
|
|
|
SSLSocketFactory newFactory = sc.getSocketFactory();
|
|
|
|
|
connection.setSSLSocketFactory(newFactory);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
return oldFactory;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|