diff --git a/collector-scheduling-management/src/main/java/com/docus/server/DefaultWebConfig.java b/collector-scheduling-management/src/main/java/com/docus/server/DefaultWebConfig.java index d9ec30b..412d59d 100644 --- a/collector-scheduling-management/src/main/java/com/docus/server/DefaultWebConfig.java +++ b/collector-scheduling-management/src/main/java/com/docus/server/DefaultWebConfig.java @@ -3,7 +3,19 @@ package com.docus.server; import com.docus.infrastructure.WebConfig; import com.docus.infrastructure.web.json.JsonSerializerModule; import com.docus.server.common.serializer.DefJsonSerializerModule; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.StringHttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.Charset; +import java.util.List; @Configuration public class DefaultWebConfig extends WebConfig { @@ -14,4 +26,25 @@ public class DefaultWebConfig extends WebConfig { return new DefJsonSerializerModule(); } + //http请求工具 restTemplate + @ConditionalOnMissingBean + @Bean + public RestTemplate restTemplate() { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setReadTimeout(3 * 60 * 1000); + requestFactory.setConnectTimeout(3 * 60 * 1000); + RestTemplate restTemplate = new RestTemplate(requestFactory); + ObjectMapper objectMapper = new ObjectMapper(); + //]忽略未定义的属性 + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + List> messageConverters = restTemplate.getMessageConverters(); + messageConverters.removeIf(converter -> converter instanceof StringHttpMessageConverter); + messageConverters.add(1, new StringHttpMessageConverter(Charset.forName("UTF-8"))); + HttpMessageConverter jsonConverter = messageConverters.stream() + .filter(p -> p instanceof MappingJackson2HttpMessageConverter).findFirst().orElse(null); + if (jsonConverter != null) { + ((MappingJackson2HttpMessageConverter) jsonConverter).setObjectMapper(objectMapper); + } + return restTemplate; + } } diff --git a/collector-scheduling-management/src/main/java/com/docus/server/controller/FileController.java b/collector-scheduling-management/src/main/java/com/docus/server/controller/FileController.java index dc42eab..6d15adf 100644 --- a/collector-scheduling-management/src/main/java/com/docus/server/controller/FileController.java +++ b/collector-scheduling-management/src/main/java/com/docus/server/controller/FileController.java @@ -2,11 +2,20 @@ package com.docus.server.controller; import com.docus.server.api.scheduling.management.FileApi; import com.docus.server.service.IFileUploadService; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; /** * 文件上传下载 API @@ -18,6 +27,8 @@ import javax.servlet.http.HttpServletResponse; public class FileController implements FileApi { @Resource private IFileUploadService iFileUploadService; + @Resource + private RestTemplate restTemplate; @Override public void downloadFile(String filePath, HttpServletResponse response) throws Exception { @@ -28,4 +39,21 @@ public class FileController implements FileApi { public void uploadFile(MultipartFile[] multipartFiles, String pathKey) throws Exception { iFileUploadService.uploadFile(multipartFiles, pathKey); } + + @Override + public void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception { + iFileUploadService.downLoadFromUrl(urlStr, fileName, savePath); + } + + @Override + public ResponseEntity downloadFile(@RequestParam String url) throws IOException { + HttpEntity request = new HttpEntity<>(new HttpHeaders()); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, request, byte[].class); + ByteArrayResource resource = new ByteArrayResource(response.getBody()); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + "your_file_name") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(resource.contentLength()) + .body(resource); + } } diff --git a/collector-scheduling-management/src/main/java/com/docus/server/service/IFileUploadService.java b/collector-scheduling-management/src/main/java/com/docus/server/service/IFileUploadService.java index 00e46ec..e46ed31 100644 --- a/collector-scheduling-management/src/main/java/com/docus/server/service/IFileUploadService.java +++ b/collector-scheduling-management/src/main/java/com/docus/server/service/IFileUploadService.java @@ -11,4 +11,6 @@ public interface IFileUploadService { List uploadFile(MultipartFile[] multipartFiles, String pathKey) throws Exception; void downloadFile(String filePath, HttpServletResponse response); + + void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception; } diff --git a/collector-scheduling-management/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java b/collector-scheduling-management/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java index 647a2bb..eb92f4e 100644 --- a/collector-scheduling-management/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java +++ b/collector-scheduling-management/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java @@ -7,17 +7,32 @@ import com.docus.infrastructure.web.exception.ExceptionCode; import com.docus.server.service.IFileUploadService; import com.docus.server.vo.scheduling.management.schcollectorversionfile.UploadFileVO; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.httpclient.util.URIUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.net.URLEncoder; +import java.security.cert.CertificateException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -35,6 +50,8 @@ public class FileUploadServiceImpl implements IFileUploadService { private static DateTimeFormatter ymdDtf = DateTimeFormatter.ofPattern("yyyyMMdd"); + private String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"; + @Override public List uploadFile(MultipartFile[] multipartFiles, String pathKey) throws Exception { @@ -120,6 +137,155 @@ public class FileUploadServiceImpl implements IFileUploadService { } } + /** + * 设置不验证主机 + */ + private final HostnameVerifier DO_NOT_VERIFY = new HostnameVerifier() { + @Override + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + @Override + public void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception { + long start = System.currentTimeMillis(); + urlStr = urlStr.replace("\\", "/"); + urlStr = URIUtil.encodePathQuery(urlStr); + URL url = new URL(urlStr); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + try { + boolean useHttps = urlStr.toLowerCase().startsWith("https"); + if (useHttps) { + HttpsURLConnection https = (HttpsURLConnection) conn; + trustAllHosts(https); + https.setHostnameVerifier(DO_NOT_VERIFY); + } + //设置超时间为3秒 + //防止屏蔽程序抓取而返回403错误 + conn.setRequestProperty("User-Agent", userAgent); + conn.setRequestProperty("Accept-Encoding", "identity"); + conn.setConnectTimeout(8 * 1000); + conn.setReadTimeout(8 * 1000); + conn = reload(conn); + long length = conn.getContentLength(); + if (length < 0) { + String values = conn.getHeaderField("Content-Length"); + if (values != null && !values.isEmpty()) { + length = Long.parseLong(values); + } + } + InputStream inputStream = null; + if (conn.getResponseCode() >= 400) { + throw new Exception("文件不存在"); + } else { + inputStream = conn.getInputStream(); + } + //获取自己数组 + byte[] getData = readInputStream(inputStream); + + //文件保存位置 + File saveDir = new File(savePath); + if (!saveDir.exists()) { + saveDir.mkdir(); + } + File file = new File(saveDir + File.separator + fileName); + FileOutputStream fos = new FileOutputStream(file); + fos.write(getData); + if (fos != null) { + fos.close(); + } + if (inputStream != null) { + inputStream.close(); + } + long end = System.currentTimeMillis(); + log.info("info:" + url + " download success;用时:" + (end - start) + "ms"); + } catch (Exception ex) { + throw ex; + } finally { + // 断开连接，释放资源 + conn.disconnect(); + } + } + + /** + * 处理多级302等跳转 + * + * @param uc + * @return + * @throws Exception + */ + private HttpURLConnection reload(HttpURLConnection uc) throws Exception { + + HttpURLConnection huc = uc; + + if (huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP + || huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) {// 302, 301 + String url = huc.getHeaderField("Location"); + url = url.replace("\\", "/"); + return reload((HttpURLConnection) new URL(url).openConnection()); + } + return uc; + } + + /** + * 覆盖java默认的证书验证 + */ + private final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[]{}; + } + }}; + + /** + * 信任所有 + * + * @param connection + * @return + */ + private SSLSocketFactory trustAllHosts(HttpsURLConnection connection) { + SSLSocketFactory oldFactory = connection.getSSLSocketFactory(); + try { + SSLContext sc = SSLContext.getInstance("TLS"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + SSLSocketFactory newFactory = sc.getSocketFactory(); + connection.setSSLSocketFactory(newFactory); + } catch (Exception e) { + e.printStackTrace(); + } + return oldFactory; + } + + /** + * 从输入流中获取字节数组 + * + * @param inputStream + * @return + * @throws IOException + */ + public byte[] readInputStream(InputStream inputStream) throws IOException { + byte[] buffer = new byte[1024]; + int len = 0; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + while ((len = inputStream.read(buffer)) != -1) { + bos.write(buffer, 0, len); + } + bos.close(); + return bos.toByteArray(); + } + + private void valid(MultipartFile multipartFile) { if (multipartFile.isEmpty()) { throw new ApiException(ExceptionCode.ParamIllegal.getCode(), "上传失败，请选择文件！"); diff --git a/collector-terminal-management/pom.xml b/collector-terminal-management/pom.xml index d2224c7..fa4f759 100644 --- a/collector-terminal-management/pom.xml +++ b/collector-terminal-management/pom.xml @@ -6,8 +6,8 @@ 1.0-SNAPSHOT 4.0.0 - collector-terminal-management[0] - Archetype - collector-scheduling-management + collector-terminal-management + Archetype - collector-terminal-management http://maven.apache.org diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/Base64Utils.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/Base64Utils.java new file mode 100644 index 0000000..3159ce3 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/Base64Utils.java @@ -0,0 +1,154 @@ +package com.docus.server.common.download; + +import lombok.extern.slf4j.Slf4j; +import sun.misc.BASE64Decoder; +import sun.misc.BASE64Encoder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +@Slf4j +public class Base64Utils { + private static Base64Utils utils =null; + private Base64Utils(){ + + } + public static Base64Utils getInstance(){ + if(utils==null){ + synchronized (Base64Utils.class){ + if(utils == null){ + utils = new Base64Utils(); + } + } + } + return utils; + } + + public String imageToBase64Str(String imgFile) { + InputStream inputStream = null; + byte[] data = null; + try { + inputStream = new FileInputStream(imgFile); + data = new byte[inputStream.available()]; + inputStream.read(data); + inputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + // 加密 + BASE64Encoder encoder = new BASE64Encoder(); + return encoder.encode(data); + } + + + public byte[] base64StrToBytes(String imgStr) { + if (imgStr == null) { + return null; + } + String[] s = imgStr.split(","); + if(s.length>0){ + imgStr=s[1]; + } + BASE64Decoder decoder = new BASE64Decoder(); + try { + // 解密 + byte[] b = decoder.decodeBuffer(imgStr); + // 处理数据 + for (int i = 0; i < b.length; ++i) { + if (b[i] < 0) { + b[i] += 256; + } + } + return b; + } catch (Exception e) { + log.error("Base64 转 byte[] 出错啦！",e); + return null; + } + } + + + public boolean base64StrToImage(byte[] bytes, String path) { + if (bytes == null) { + return false; + } + try { + //文件夹不存在则自动创建 + File tempFile = new File(path); + if (!tempFile.getParentFile().exists()) { + tempFile.getParentFile().mkdirs(); + } + OutputStream out = new FileOutputStream(tempFile); + out.write(bytes); + out.flush(); + out.close(); + return true; + } catch (Exception e) { + log.error("图片文件下载失败："+e.getMessage()); + return false; + } + } + + public boolean base64StrToImage(String imgStr, String path) { + if (imgStr == null) { + return false; + } + String[] s = imgStr.split(","); + if(s.length>0){ + imgStr=s[1]; + } + BASE64Decoder decoder = new BASE64Decoder(); + try { + // 解密 + byte[] b = decoder.decodeBuffer(imgStr); + // 处理数据 + for (int i = 0; i < b.length; ++i) { + if (b[i] < 0) { + b[i] += 256; + } + } + //文件夹不存在则自动创建 + File tempFile = new File(path); + if (!tempFile.getParentFile().exists()) { + tempFile.getParentFile().mkdirs(); + } + OutputStream out = new FileOutputStream(tempFile); + out.write(b); + out.flush(); + out.close(); + return true; + } catch (Exception e) { + log.error("图片文件下载失败："+e.getMessage()); + return false; + } + } + + public int getFileSize(File inFile){ + InputStream in =null; + try{ + in= new FileInputStream(inFile); + int len =in.available(); + return len; + } + catch (Exception e){ + + } + finally { + try{ + in.close(); + } + catch (IOException e){ + e.printStackTrace(); + } + } + return -1; + } + + + public static void main(String[] args) { + System.out.println(Base64Utils.getInstance().imageToBase64Str("e:\\1.jpg")); + } +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/CallBackPara.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/CallBackPara.java new file mode 100644 index 0000000..cd87c6e --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/CallBackPara.java @@ -0,0 +1,50 @@ +package com.docus.server.common.download; + +import lombok.Data; + +import java.sql.Timestamp; +import java.util.List; + +@Data +public class CallBackPara { + /** + * 主键 + */ + private String id; + /** + * 其他信息 + */ + private String mess; + + /** + * 本地存放路径 + */ + private String localpath; + + /** + * 文件名 + */ + private String filename; + + /** + * 下载开始时间 + */ + private Timestamp starttime; + + /** + * 下载完成时间 + */ + private Timestamp endtime; + + private List files; + + /** + * 下载存储文件的方式 0 传过来的格式，1转为 pdf 默认 ，2 转为图片jpg存储 + */ + private int fileStorageFormat=1; + + /** + * 0:病案，1：门急诊 2：省中医封存 + */ + private int datatype=0; +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/DownloadManager.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/DownloadManager.java new file mode 100644 index 0000000..1788611 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/DownloadManager.java @@ -0,0 +1,67 @@ +package com.docus.server.common.download; + +import com.docus.core.util.SpringUtils; +import com.docus.server.common.download.downLoader.HttpDownloader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + + +public class DownloadManager { + + private static Logger logger = LoggerFactory.getLogger(DownloadManager.class); + + enum ProtocolType {HTTP, HTTPS, FTP, BASE64, SMB, OTHER} + + ; + + public static void feedLinks(Collection list, int maxHttpRetry) { + for (RemoteLocalPair pair : list) { + ProtocolType deal = getLinkProtocol(pair.remoteUrl); + if (deal == ProtocolType.HTTP || deal == ProtocolType.HTTPS) { + HttpDownload(pair, maxHttpRetry); + } else if (deal == ProtocolType.FTP) { + } else if (deal == ProtocolType.BASE64) { + } else if (deal == ProtocolType.SMB) { + } else { + logger.error(pair.remoteUrl + " Url is not http or ftp!"); + } + } + } + + + public static void HttpDownload(RemoteLocalPair pair, int maxHttpRetry) { + //GeneralDownloadInfo info = new GeneralDownloadInfo(pair); + NoCheckPointInfo info = new NoCheckPointInfo(pair); + HttpDownloader downloader = new HttpDownloader(info, maxHttpRetry); + downloader.intiCallBack(SpringUtils.getBean(IDownCallBack.class)); + //可以用thread.start()不过不建议 + downloader.run(); + } + + + protected static ProtocolType getLinkProtocol(String link) { + int idx = link.indexOf("://"); + if (idx == -1) { + if (link.contains("base64")) { + return ProtocolType.BASE64; + } + return ProtocolType.OTHER; + } + String name = link.substring(0, idx); + if (name.equals("http")) { + return ProtocolType.HTTP; + } + if (name.equals("https")) { + return ProtocolType.HTTPS; + } + if (name.equals("ftp")) { + return ProtocolType.FTP; + } + if (name.equals("smb")) { + return ProtocolType.SMB; + } + return ProtocolType.OTHER; + } +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/FileCheckPoints.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/FileCheckPoints.java new file mode 100644 index 0000000..59ba410 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/FileCheckPoints.java @@ -0,0 +1,40 @@ +package com.docus.server.common.download; + +public class FileCheckPoints { + public int statecode; + public long timestamp = -99; + public long totalSize = -99; + private int split = -1; + private long[] startPos; + private long[] endPos; + + public long[] getStartPos() { + return startPos; + } + + public void setStartPos(long[] startPos) { + split = startPos.length; + this.startPos = startPos; + } + + public long[] getEndPos() { + return endPos; + } + + public void setEndPos(long[] endPos) { + split = endPos.length; + this.endPos = endPos; + } + + public int getSplit() { + return split; + } + + public void copy(FileCheckPoints _chp) { + this.setEndPos(_chp.endPos); + this.setStartPos(_chp.startPos); + this.totalSize = _chp.totalSize; + this.timestamp = _chp.timestamp; + } + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/FtpInfo.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/FtpInfo.java new file mode 100644 index 0000000..0101ee1 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/FtpInfo.java @@ -0,0 +1,11 @@ +package com.docus.server.common.download; + +import lombok.Data; + +@Data +public class FtpInfo { + private String ip; + private Integer port; + private String user; + private String pwd; +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java new file mode 100644 index 0000000..32b0d8f --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java @@ -0,0 +1,200 @@ +package com.docus.server.common.download; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +public class GeneralDownloadInfo implements IDownloadInfo { + + /** + * 0:ftp地址，1:文件，2：用户名：3：密码，4：端口 + * @param fullUrl + * @return + */ + public static String[] getHostNameAndFilePath(String fullUrl) { + fullUrl = fullUrl.trim(); + String[] res = new String[5]; + int beginIndex = "ftp://".length(); + int endIndex = fullUrl.indexOf('/', beginIndex); + if (endIndex != -1) { + res[0] = fullUrl.substring(beginIndex, endIndex); + res[1] = fullUrl.substring(endIndex + 1); + } else { + res[0] = fullUrl.substring(beginIndex); + res[1] = Tools.getRandomUUID().toString() + ".none"; + } + String[] temp = res[0].split("@"); + if(temp.length>1){ + res[0] = temp[1]; + res[2] =temp[0]; + } + else + { + res[2]=""; + res[3]=""; + } + temp = res[0].split(":"); + if(temp.length>1){ + res[0] = temp[0]; + res[4] =temp[1]; + } + else + { + res[4]=""; + } + if(!StringUtils.isEmpty(res[2])){ + temp = res[2].split(":"); + if(temp.length>1){ + res[2] = temp[0]; + res[3] =temp[1]; + } + } + + return res; + } + + public GeneralDownloadInfo(RemoteLocalPair pair) { + this.pair = pair; + curName = pair.localName; + curFlagFile = new File(pair.getLocalFullPath()+".flags"); + if (!curFlagFile.getParentFile().exists()) { + curFlagFile.getParentFile().mkdirs(); + } + } + + @Override + public FileCheckPoints getCurCheckPoints() { + return chp; + } + + @Override + public RemoteLocalPair getPair() { + return this.pair; + } + + @Override + public void initDownload() { + + } + + + + @Override + public int getSplitNum(){ + return this.pair.splitNum; + } + @Override + public boolean isNeedDownload(FileCheckPoints serverInitChp) { + chp = readInfo(); + if (chp.timestamp == serverInitChp.timestamp && chp.totalSize == serverInitChp.totalSize && chp.getSplit() == serverInitChp.getSplit()) { + // 不需要下载 + boolean isneed = false; + for (int i = 0; i < chp.getStartPos().length; i++) { + if (chp.getStartPos()[i] < chp.getEndPos()[i]) { + isneed = true; + logger.info("restore from checkpoint"); + break; + } + } + return isneed; + } else { + chp.copy(serverInitChp); + return true; + } + } + + @Override + public synchronized boolean writeInfo(FileCheckPoints chkp) { + DataOutputStream dos = null; + try { + dos = new DataOutputStream(new FileOutputStream(curFlagFile)); + dos.writeLong(chkp.timestamp); + dos.writeLong(chkp.totalSize); + int split = chp.getSplit(); + dos.writeInt(split); + for(int i=0; i < chp.getSplit(); i++){ + dos.writeLong(chp.getStartPos()[i]); + dos.writeLong(chp.getEndPos()[i]); + } + } catch (FileNotFoundException e) { + logger.debug(curName, e); + } catch (IOException e) { + logger.debug(curName, e); + } finally { + if (dos != null) { + try { + dos.close(); + } catch (IOException e) { + logger.debug(curName, e); + } + } + } + return true; + } + + /** + * function:读取写入点的位置信息 如果不存在，那么 + */ + @Override + public FileCheckPoints readInfo() { + if (curFlagFile.exists()) { + DataInputStream dis = null; + try { + dis = new DataInputStream(new FileInputStream(curFlagFile)); + long curTimeStamp = dis.readLong(); + long curTotalSize = dis.readLong(); + int curSplit = dis.readInt(); + FileCheckPoints chkP = new FileCheckPoints(); + chkP.timestamp = curTimeStamp; + chkP.totalSize = curTotalSize; + long[] sp = new long[curSplit]; + long[] ep = new long[curSplit]; + for (int i=0; i< curSplit; i++){ + sp[i] = dis.readLong(); + ep[i] = dis.readLong(); + } + chkP.setStartPos(sp); + chkP.setEndPos(ep); + return chkP; + } catch (FileNotFoundException e) { + logger.debug(curName, e); + } catch (IOException e) { + logger.debug(curName, e); + } finally { + if (dis != null) { + try { + dis.close(); + } catch (IOException e) { + logger.debug(curName, e); + } + } + } + } else { + return new FileCheckPoints(); + } + return new FileCheckPoints(); + } + + @Override + public void downloadDone(FileCheckPoints chkp) { + writeInfo(chkp); + } + + + private Logger logger = LoggerFactory.getLogger(GeneralDownloadInfo.class); + + + private RemoteLocalPair pair; + private String curName; + private File curFlagFile; + private FileCheckPoints chp = null; + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/GlobalData.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/GlobalData.java new file mode 100644 index 0000000..fbfe4e6 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/GlobalData.java @@ -0,0 +1,143 @@ +package com.docus.server.common.download; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class GlobalData { + + /** + * @param args + */ + public static void main(String[] args) { + + } + + private Properties properties; + private String filePath; + + private GlobalData(String filePath) { + this.filePath = filePath; + properties = loadProperties(filePath); + } + + //=================================== + private Properties loadProperties(String filePath) { + InputStream is; + try { + is = new BufferedInputStream(new FileInputStream(new File(filePath))); + Properties properties = new Properties(); + properties.load(is); + return properties; + } catch (FileNotFoundException e) { +// e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); + } + return null; + } + + public Properties getProperties() { + return properties; + } + + public void setPropertity(String key, String value) { + FileOutputStream oFile; + try { + properties.setProperty(key, value); + oFile = new FileOutputStream(filePath); + properties.store(oFile, "add counter!"); + oFile.close(); + } catch (IOException e) { +// e.printStackTrace(); + } + } + + public String getTempDir() { + String path = properties.getProperty("TEMP_PATH"); + File dir = new File(path); + if (!dir.exists()) { + try { + dir.mkdir(); + } catch (SecurityException e) { +// e.printStackTrace(); + } + } + return path; + } + + public String getParserDir() { + String path = properties.getProperty("PARSER_DIR"); + File dir = new File(path); + if (!dir.exists()) { + try { + dir.mkdir(); + } catch (SecurityException e) { +// e.printStackTrace(); + } + } + return path; + } + + public String getHomoGoaUrl() { + return properties.getProperty("GO_HOMO_GOA_GZ_LINK"); + } + + public String getGoBasicOboUrl() { + return properties.getProperty("GO_OBO_BASIC_LINK"); + } + + public String getGoOboUrl() { + return properties.getProperty("GO_OBO_LINK"); + } + + public String getGoOwlUrl() { + return properties.getProperty("GO_OWL_LINK"); + } + + public String getGoOwlPlusUrl() { + return properties.getProperty("GO_OWL_PLUS_LINK"); + } + + public String getDoOboUrl() { + return properties.getProperty("DO_OBO_LINK"); + } + + public String getProOboUrl() { + return properties.getProperty("PRO_OBO_LINK"); + } + + public String getChebiOboUrl() { + return properties.getProperty("CHEBI_OBO_LINK"); + } + + public String getLocalGoBasicPath() { + String goUrl = getGoBasicOboUrl(); + return getTempDir() + File.separator + GlobalData.getFileName(goUrl); + } + + //==================================== + private static GlobalData single; + private static Object lock = new Object(); + + //==================================== + public static String getFileName(String url) { + return url.substring(url.lastIndexOf("/") + 1, url.length()); + } + + public static GlobalData getInstance() { + synchronized (lock) { + if (single == null) { + single = new GlobalData("configure/settings.properties"); + } + return single; + } + } + + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownCallBack.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownCallBack.java new file mode 100644 index 0000000..7a96913 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownCallBack.java @@ -0,0 +1,7 @@ +package com.docus.server.common.download; + +public interface IDownCallBack { + void success(CallBackPara para); + + void fail(CallBackPara para); +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownloadInfo.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownloadInfo.java new file mode 100644 index 0000000..7b8f03c --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/IDownloadInfo.java @@ -0,0 +1,60 @@ +package com.docus.server.common.download; + + +public interface IDownloadInfo { + /** + * 初始化download这个阶段可以做连接数据库的操作 + */ + void initDownload(); + + /** + * 判断是否需要下载，从数据库获取断点数据 + * + * @param serverInitChp checkpoint + * @return + */ + boolean isNeedDownload(FileCheckPoints serverInitChp); + + /** + * 写入断点，可以进行数据库写入操作 + * + * @param chkp checkpoint + * @return + */ + boolean writeInfo(FileCheckPoints chkp); + + /** + * 获取当前的checkPoint + * + * @return + */ + FileCheckPoints getCurCheckPoints(); + + /** + * 读取断点信息，可以进行数据库操作 + * + * @return + */ + FileCheckPoints readInfo(); + + /** + * 最后下载完成之后的操作 + * + * @param chkp + */ + void downloadDone(FileCheckPoints chkp); + + /** + * 获取当前链接和本地保存的路径信息 + * + * @return + */ + RemoteLocalPair getPair(); + + /** + * 文件分块，默认为1 + * + * @return + */ + int getSplitNum(); +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/MultiDownFile.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/MultiDownFile.java new file mode 100644 index 0000000..e4a4e6d --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/MultiDownFile.java @@ -0,0 +1,38 @@ +package com.docus.server.common.download; + +public class MultiDownFile { + public MultiDownFile(String remoteUrl, String localPath, String localName) { + this.remoteUrl = remoteUrl; + this.localPath = localPath; + this.localName = localName; + } + + private String remoteUrl; + private String localPath; + private String localName; + + public String getRemoteUrl() { + return remoteUrl; + } + + public void setRemoteUrl(String remoteUrl) { + this.remoteUrl = remoteUrl; + } + + public String getLocalPath() { + return localPath; + } + + public void setLocalPath(String localPath) { + this.localPath = localPath; + } + + public String getLocalName() { + return localName; + } + + public void setLocalName(String localName) { + this.localName = localName; + } + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java new file mode 100644 index 0000000..05f77c7 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java @@ -0,0 +1,73 @@ +package com.docus.server.common.download; + + +import java.io.File; + +/** + * 使用这个info可以实现无断线续传功能的下载 + * + * @author burkun + */ +public class NoCheckPointInfo implements IDownloadInfo { + + + private boolean isDownloding; + + public NoCheckPointInfo(RemoteLocalPair pair) { + this.pair = pair; + File file = new File(pair.localPath); + if (!file.exists()) { + file.mkdirs(); + } + } + + @Override + public void initDownload() { + isDownloding = true; + } + + @Override + public boolean isNeedDownload(FileCheckPoints serverInitChp) { + chp = serverInitChp; + return true; + } + + @Override + public boolean writeInfo(FileCheckPoints chkp) { + chp = chkp; + return true; + } + + @Override + public FileCheckPoints getCurCheckPoints() { + return chp; + } + + @Override + public FileCheckPoints readInfo() { + return null; + } + + @Override + public void downloadDone(FileCheckPoints chkp) { + isDownloding = false; + } + + @Override + public RemoteLocalPair getPair() { + return pair; + } + + @Override + public int getSplitNum() { + return 1; + } + + public boolean IsDownloading() { + return isDownloding; + } + + private RemoteLocalPair pair; + private FileCheckPoints chp = null; + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/RemoteLocalPair.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/RemoteLocalPair.java new file mode 100644 index 0000000..bf687c1 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/RemoteLocalPair.java @@ -0,0 +1,72 @@ +package com.docus.server.common.download; + +import java.io.File; +import java.util.List; + + +/** + * 需要的下载和保存的文件对 + * + * @author burkun + */ +public class RemoteLocalPair { + public String id; + public int splitNum = 1; + public String remoteUrl; + public String localPath; + public String localName; + + public String getLocalFullPath() { + return localPath + File.separator + localName; + } + + public String user; + public String pwd; + public Integer port; + public List separators; + public String failurl; + public List proxyurls; + public List smbips; + /** + * 下载存储文件的方式 0 传过来的格式，1转为 pdf 默认 ，2 转为图片jpg存储 + */ + public int fileStorageFormat = 1; + /** + * 0:病案，1：门急诊 2:省中医封存 + */ + public int datatype = 0; + + /** + * @param remoteUrl 远程url + * @param localPath 本地路径 + * @param localName 本地文件名 + */ + public RemoteLocalPair(String remoteUrl, String localPath, String localName) { + this.remoteUrl = remoteUrl; + if (localName.length() == 0) { + this.localName = getFileName(remoteUrl); + ; + } else { + this.localName = localName; + } + if (localPath.length() == 0) { + this.localPath = GlobalData.getInstance().getTempDir(); + } else { + this.localPath = localPath; + } + } + + /** + * @param remoteUrl 远程url + * @param localPath 本地路径 默认文件名从url中提取 + */ + public RemoteLocalPair(String remoteUrl, String localPath) { + this.remoteUrl = remoteUrl; + this.localName = getFileName(remoteUrl); + this.localPath = localPath; + } + + private String getFileName(String url) { + return url.substring(url.lastIndexOf("/") + 1, url.length()); + } +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/SaveFileItem.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/SaveFileItem.java new file mode 100644 index 0000000..485eee5 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/SaveFileItem.java @@ -0,0 +1,73 @@ +package com.docus.server.common.download; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.RandomAccessFile; + + +/** + * function: 写入文件、保存文件 + */ +public class SaveFileItem { + //存储文件 + private RandomAccessFile itemFile; + private String name; + private Logger logger = LoggerFactory.getLogger(SaveFileItem.class); + + /** + * @param name 文件路径、名称 + * @param pos 写入点位置position + * @throws IOException + */ + public SaveFileItem(String name, long pos) throws IOException { + this.name = name; + itemFile = new RandomAccessFile(name, "rwd"); + //在指定的pos位置写入数据 + itemFile.seek(pos); + } + + /** + * function: 同步方法写入文件 + * @author hoojo + * @createDate 2011-9-26 下午12:21:22 + * @param buff 缓冲数组 + * @param start 起始位置 + * @param length 长度 + * @return + */ + public synchronized int write(byte[] buff, int start, int length) { + int i = -1; + try { + itemFile.write(buff, start, length); + i = length; + } catch (IOException e) { + logger.debug(name, e); + } + return i; + } + + public void close() throws IOException { + if (itemFile != null) { + itemFile.close(); + } + } + + public String getFileName(){ + return this.name; + } + /** + * 设置文件大小，在old 文件大于新文件时特别有用 + * @param newLength 新的文件长度 + */ + public void setLength(long newLength){ + try { + if(newLength != itemFile.length()){ + itemFile.setLength(newLength); + } + } catch (IOException e) { + logger.debug(name, e); + } + } +} \ No newline at end of file diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/Tools.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/Tools.java new file mode 100644 index 0000000..a50caaf --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/Tools.java @@ -0,0 +1,30 @@ +package com.docus.server.common.download; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.UUID; + +public class Tools { + public static UUID getRandomUUID() { + return UUID.randomUUID(); + } + + /** + * @param filename propertiy file path + * @return + * @author burkun + */ + public static Properties readPropertiesFile(String filename) { + Properties properties = new Properties(); + try { + InputStream inputs = new FileInputStream(filename); + properties.load(inputs); + inputs.close(); + } catch (IOException e) { +// e.printStackTrace(); + } + return properties; + } +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java b/collector-terminal-management/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java new file mode 100644 index 0000000..5f67416 --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java @@ -0,0 +1,558 @@ +package com.docus.server.common.download.downLoader; + +import com.alibaba.fastjson.JSON; +import com.docus.server.common.download.CallBackPara; +import com.docus.server.common.download.FileCheckPoints; +import com.docus.server.common.download.IDownCallBack; +import com.docus.server.common.download.IDownloadInfo; +import com.docus.server.common.download.MultiDownFile; +import com.docus.server.common.download.SaveFileItem; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.httpclient.util.URIUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * + */ +@Slf4j +public class HttpDownloader extends Thread { + + private Logger logger = LoggerFactory.getLogger(HttpDownloader.class); + private IDownloadInfo info; + private int maxRetry = 5; + private IDownCallBack downCallBack; + private String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"; + + public HttpDownloader(IDownloadInfo info, int maxRetry) { + this.info = info; + this.maxRetry = maxRetry; + } + + /** + * 初始化回调方法 + * + * @param downCallBack + */ + public void intiCallBack(IDownCallBack downCallBack) { + this.downCallBack = downCallBack; + } + + + public HttpDownloader(IDownloadInfo info) { + this.info = info; + } + + public String[] removeArraysEmpty(String[] arr) { + return Arrays.stream(arr).filter(s -> !"".equals(s)).toArray(String[]::new); + } + + @Override + public void run() { + CallBackPara para = new CallBackPara(); + para.setId(info.getPair().id); + para.setFilename(info.getPair().localName); + para.setLocalpath(info.getPair().localPath); + para.setStarttime(getNewtime()); + para.setFileStorageFormat(info.getPair().fileStorageFormat); + para.setDatatype(info.getPair().datatype); + String url = info.getPair().remoteUrl; + List files = new ArrayList<>(); + + String[] urls = removeArraysEmpty(url.split("http://|https://")); + if (urls.length > 1) { + for (int i = 0; i < urls.length; i++) { + String urltemp = ""; + if (url.indexOf("http://" + urls[i]) >= 0) { + urltemp = "http://" + urls[i]; + } + if (url.indexOf("https://" + urls[i]) >= 0) { + urltemp = "https://" + urls[i]; + } + for (String o : this.info.getPair().separators) { + urltemp = urltemp.replaceAll(o + "$", ""); + } + files.add(new MultiDownFile(urltemp, info.getPair().localPath, "docustemp_" + i + "_" + info.getPair().localName)); + } + } else { + files.add(new MultiDownFile(url, info.getPair().localPath, info.getPair().localName)); + } + para.setFiles(files); + if (info.getPair().proxyurls != null) { + Pattern p = Pattern.compile(String.join("|", info.getPair().proxyurls)); + for (MultiDownFile o : files) { + Matcher matcher = p.matcher(o.getRemoteUrl()); + if (matcher.find()) { + o.setRemoteUrl(String.format(info.getPair().failurl, o.getRemoteUrl())); + } + } + } +// URLHttpDownBootstrapBuilder builder=null; +// HttpDownBootstrap bootstrap; + try { + for (MultiDownFile file : files) { +// try { +// url = EncoderUrl(file.getRemoteUrl()); +// } catch (Exception e) { +// +// } + downLoadFromUrl(url, file.getLocalName(), file.getLocalPath()); + if (downCallBack != null) { + downCallBack.success(para); + } + //防止过快，第三链接无法支持 + try { + Thread.sleep(100); + } catch (Exception e) { + + } +// builder = HttpDownBootstrap.builder(url); +// builder.downConfig(new HttpDownConfigInfo() +// .setFilePath(file.getLocalPath()) +// ).callBackPara(para); +// builder.response(new HttpResponseInfo(file.getLocalName())); +// bootstrap = builder.callback(new ConsoleHttpDownCallback()).build(); +// bootstrap.start(); +// bootstrap = null; +// builder = null; + } + } catch (Exception e) { +// e.printStackTrace(); + log.error("nio下载失败，" + JSON.toJSONString(para) + ";失败信息：" + e.getMessage()); + if (downCallBack != null) { + downCallBack.fail(para); + } + } +// finally { +// bootstrap = null; +// builder = null; +// } + } + + public String EncoderUrl(String url) throws UnsupportedEncodingException { + String resultURL = ""; + for (int i = 0; i < url.length(); i++) { + char charAt = url.charAt(i); + //只对汉字处理 + if (isChineseChar(charAt)) { + String encode = URLEncoder.encode(charAt + "", "UTF-8"); + resultURL += encode; + } else { + resultURL += charAt; + } + } + return resultURL; + } + + public boolean isChineseChar(char c) { + return String.valueOf(c).matches("[\u4e00-\u9fa5]"); + } + + /** + * 处理多级302等跳转 + * + * @param uc + * @return + * @throws Exception + */ + private HttpURLConnection reload(HttpURLConnection uc) throws Exception { + + HttpURLConnection huc = uc; + + if (huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP + || huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) {// 302, 301 + String url = huc.getHeaderField("Location"); + url = url.replace("\\", "/"); + return reload((HttpURLConnection) new URL(url).openConnection()); + } + return uc; + } + + + public void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception { + long start = System.currentTimeMillis(); + urlStr = urlStr.replace("\\", "/"); + urlStr = URIUtil.encodePathQuery(urlStr); + URL url = new URL(urlStr); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + try { + boolean useHttps = urlStr.toLowerCase().startsWith("https"); + if (useHttps) { + HttpsURLConnection https = (HttpsURLConnection) conn; + trustAllHosts(https); + https.setHostnameVerifier(DO_NOT_VERIFY); + } + //设置超时间为3秒 + //防止屏蔽程序抓取而返回403错误 + conn.setRequestProperty("User-Agent", userAgent); + conn.setRequestProperty("Accept-Encoding", "identity"); + conn.setConnectTimeout(8 * 1000); + conn.setReadTimeout(8 * 1000); + conn = reload(conn); + long length = conn.getContentLength(); + if (length < 0) { + String values = conn.getHeaderField("Content-Length"); + if (values != null && !values.isEmpty()) { + length = Long.parseLong(values); + } + } +// log.info(urlStr+" 文件大小:"+length); + InputStream inputStream = null; + if (conn.getResponseCode() >= 400) { + throw new Exception("文件不存在"); +// inputStream = conn.getErrorStream(); + } else { + inputStream = conn.getInputStream(); + } + //得到输入流 + //InputStream inputStream = conn.getInputStream(); + //获取自己数组 + byte[] getData = readInputStream(inputStream); + + //文件保存位置 + File saveDir = new File(savePath); + if (!saveDir.exists()) { + saveDir.mkdir(); + } + File file = new File(saveDir + File.separator + fileName); + FileOutputStream fos = new FileOutputStream(file); + fos.write(getData); + if (fos != null) { + fos.close(); + } + if (inputStream != null) { + inputStream.close(); + } + long end = System.currentTimeMillis(); + logger.info("info:" + url + " download success;用时:" + (end - start) + "ms"); + } catch (Exception ex) { + throw ex; + } finally { + // 断开连接，释放资源 + conn.disconnect(); + } + } + + + /** + * 从输入流中获取字节数组 + * + * @param inputStream + * @return + * @throws IOException + */ + public byte[] readInputStream(InputStream inputStream) throws IOException { + byte[] buffer = new byte[1024]; + int len = 0; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + while ((len = inputStream.read(buffer)) != -1) { + bos.write(buffer, 0, len); + } + bos.close(); + return bos.toByteArray(); + } + + + public static FileCheckPoints initCheckPoint(int splitNum, long totalSize, long timeStamp) { + long[] startPos = new long[splitNum]; + long[] endPos = new long[splitNum]; + for (int i = 0, len = startPos.length; i < len; i++) { + long size = i * (totalSize / len); + startPos[i] = size; + // 设置最后一个结束点的位置 + if (i == len - 1) { + endPos[i] = totalSize; + } else { + size = (i + 1) * (totalSize / len); + endPos[i] = size; + } + } + FileCheckPoints chp = new FileCheckPoints(); + chp.setEndPos(endPos); + chp.setStartPos(startPos); + chp.totalSize = totalSize; + chp.timestamp = timeStamp; + return chp; + } + + private FileCheckPoints getInitedCheckPoint() { + long fileLength = -1; + long timeStamp = -1; + HttpURLConnection conn = null; + int stateCode = 0; + try { + URL url = new URL(this.info.getPair().remoteUrl); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("Accept-Encoding", "identity"); + HttpDownloader.RetriveSingleStream.setHeader(conn); + stateCode = conn.getResponseCode(); + // 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK + if (stateCode != HttpURLConnection.HTTP_OK + && stateCode != HttpURLConnection.HTTP_PARTIAL) { + logger.warn(info.getPair().remoteUrl + " #Error Code:# " + + stateCode); + fileLength = -2; + } else if (stateCode >= 400) { + logger.warn(info.getPair().remoteUrl + " #Error Code:# " + + stateCode); + fileLength = -2; + } else { + // 获取长度 + fileLength = conn.getContentLengthLong(); + timeStamp = conn.getLastModified(); + logger.info(info.getPair().remoteUrl + " #FileLength:# " + + fileLength); + } + } catch (MalformedURLException e) { +// e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + FileCheckPoints chp; + if (fileLength > 0) { + chp = initCheckPoint(info.getSplitNum(), fileLength, timeStamp); + chp.timestamp = timeStamp; + } else { + chp = new FileCheckPoints(); + chp.statecode = stateCode; + } + return chp; + } + + /** + * bug fixed change the RandomAccessFile size + * + * @author burkun + */ + + + protected static class RetriveSingleStream implements Runnable { + private boolean isDone = false; + private FileCheckPoints chp; + private int curIndex; + private SaveFileItem file; + private long startPos; + private long endPos; + byte[] buffer = new byte[1024 * 12]; + private IDownloadInfo __info; + private int maxRetry; + private Logger logger = LoggerFactory.getLogger(RetriveSingleStream.class); + + public boolean isDone() { + return isDone; + } + + public RetriveSingleStream(IDownloadInfo info, FileCheckPoints chp, + int curIndex, int maxRetry) { + this.__info = info; + this.chp = chp; + this.curIndex = curIndex; + this.startPos = chp.getStartPos()[curIndex]; + this.endPos = chp.getEndPos()[curIndex]; + this.maxRetry = maxRetry; + } + + @Override + public void run() { + InputStream in = null; + HttpURLConnection conn = null; + int curRetry = 0; + + while (curRetry < maxRetry && !isDone) { + try { + URL url = new URL(__info.getPair().remoteUrl); + conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(10000); + conn.setReadTimeout(30000); + setHeader(conn); + String property = "bytes=" + startPos + "-"; + conn.setRequestProperty("RANGE", property); + logger.info(__info.getPair().localName + " #Block" + + (curIndex + 1) + "# begin downloading..."); + int length; + long counter = 0; + InputStream is = conn.getInputStream(); + file = new SaveFileItem(__info.getPair().getLocalFullPath(), startPos); + //--bug fixed + file.setLength(__info.getCurCheckPoints().totalSize); + //--bug fixed + while (!isDone && startPos < endPos && (length = is.read(buffer)) > 0) { + startPos += file.write(buffer, 0, length); + counter += 1; + chp.getStartPos()[curIndex] = Math.min(startPos, endPos); + if (counter % 20 == 0) { + __info.writeInfo(chp); + logger.info(__info.getPair().remoteUrl + " #Block" + + (curIndex + 1) + "# download " + + getPercentage() + "%..."); + Thread.yield(); + } + } + __info.writeInfo(chp); + isDone = true; + } catch (IOException e) { + isDone = false; + logger.debug(__info.getPair().remoteUrl, e); + } finally { + if (!isDone) { + curRetry++; + logger.debug(__info.getPair().remoteUrl + " download failed, retry again!"); + if (curRetry >= maxRetry) { + //保证循环跳出 + isDone = true; + } + } else { + curRetry = maxRetry; + } + try { + if (in != null) { + in.close(); + } + if (file != null) { + file.close(); + } + if (conn != null) { + conn.disconnect(); + } + } catch (IOException e) { + logger.debug(__info.getPair().remoteUrl, e); + } + } + } + } + + + public static void setHeader(URLConnection conn) { + conn.setRequestProperty( + "User-Agent", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 BIDUBrowser/7.0 Safari/537.36"); + conn.setRequestProperty("Accept-Language", + "en-us,en;q=0.7,zh-cn;q=0.3"); + conn.setRequestProperty("Accept-Encoding", "utf-8"); + conn.setRequestProperty("Accept-Charset", + "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); + conn.setRequestProperty("Keep-Alive", "300"); + conn.setRequestProperty("connnection", "keep-alive"); + // conn.setRequestProperty("If-Modified-Since", + // "Fri, 02 Jan 2009 17:00:05 GMT"); + // conn.setRequestProperty("If-None-Match", + // "\"1261d8-4290-df64d224\""); + conn.setRequestProperty("Cache-conntrol", "max-age=0"); + conn.setRequestProperty("Referer", "http://www.baidu.com"); + } + + private int getPercentage() { + long total = 0; + for (int i = 0; i < chp.getSplit(); i++) { + total += chp.getEndPos()[i] - chp.getStartPos()[i]; + } + return (int) ((chp.totalSize - total) * 100 / chp.totalSize); + } + + } + + private Timestamp getNewtime() { + Date now = new Date(); + Timestamp timestamp = new Timestamp(now.getTime()); + return timestamp; + } + +// public static void main(String[] args) { +// String url="http://ss,ss,https://bbbbb;http://ccc"; +// List separators = new ArrayList<>(); +// separators.add(","); +// separators.add(";"); +// String[] urls = url.split("http://|https://"); +// for (int i = 0; i < urls.length; i++) { +// String urltemp = ""; +// if (url.indexOf("http://" + urls[i]) >= 0) { +// urltemp="http://" + urls[i]; +// } +// if (url.indexOf("https://" + urls[i]) >= 0) { +// urltemp="https://" + urls[i]; +// } +// for(String o:separators){ +// urltemp=urltemp.replaceAll(o+"$", ""); +// } +// System.out.println(urltemp); +// } +// } + /** + * 覆盖java默认的证书验证 + */ + private final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[]{}; + } + }}; + + /** + * 设置不验证主机 + */ + private final HostnameVerifier DO_NOT_VERIFY = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + /** + * 信任所有 + * + * @param connection + * @return + */ + private SSLSocketFactory trustAllHosts(HttpsURLConnection connection) { + SSLSocketFactory oldFactory = connection.getSSLSocketFactory(); + try { + SSLContext sc = SSLContext.getInstance("TLS"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + SSLSocketFactory newFactory = sc.getSocketFactory(); + connection.setSSLSocketFactory(newFactory); + } catch (Exception e) { + e.printStackTrace(); + } + return oldFactory; + } + +} diff --git a/collector-terminal-management/src/main/java/com/docus/server/common/netty/server/handler/NettyBusinessHandler.java b/collector-terminal-management/src/main/java/com/docus/server/common/netty/server/handler/NettyBusinessHandler.java index 35777fb..13e25fb 100644 --- a/collector-terminal-management/src/main/java/com/docus/server/common/netty/server/handler/NettyBusinessHandler.java +++ b/collector-terminal-management/src/main/java/com/docus/server/common/netty/server/handler/NettyBusinessHandler.java @@ -38,14 +38,14 @@ public class NettyBusinessHandler extends SimpleChannelInboundHandler { private static final ChannelGroup DEFAULT_CHANNEL_GROUP = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); - public static final ChannelAttribute session = new ChannelAttribute<>("state"); + public static final ChannelAttribute STATE_MACHINE_SESSION = new ChannelAttribute<>("state"); @Resource private ChannelRepository repository; @Override protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception { - DeviceStateContext deviceStateContext = session.getAttributeValue(ctx); + DeviceStateContext deviceStateContext = STATE_MACHINE_SESSION.getAttributeValue(ctx); ByteBuf buf = (ByteBuf) msg; //创建目标大小的数组 @@ -132,7 +132,7 @@ public class NettyBusinessHandler extends SimpleChannelInboundHandler { public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { System.out.println(getClass().getSimpleName() + "." + "userEventTriggered" + ctx.channel().remoteAddress()); if (evt instanceof IdleStateEvent) { - DeviceStateContext deviceStateContext = session.getAttributeValue(ctx); + DeviceStateContext deviceStateContext = STATE_MACHINE_SESSION.getAttributeValue(ctx); long lastUpdateTime = deviceStateContext.getLastUpdateTime(); long currentTimeMillis = System.currentTimeMillis(); long intervalTime = currentTimeMillis - lastUpdateTime; @@ -160,7 +160,7 @@ public class NettyBusinessHandler extends SimpleChannelInboundHandler { //===========设置设备状态为 未登录================= deviceStateContext.onConnect(System.currentTimeMillis(), "设备 active"); //更新添加 state 属性 - session.setAttribute(ctx, deviceStateContext); + STATE_MACHINE_SESSION.setAttribute(ctx, deviceStateContext); System.out.println("channelActive:" + deviceStateContext.toString()); //===========设置设备状态为 未登录================= @@ -193,7 +193,7 @@ public class NettyBusinessHandler extends SimpleChannelInboundHandler { } //================设置为断开================ - DeviceStateContext deviceStateContext = session.getAttributeValue(ctx); + DeviceStateContext deviceStateContext = STATE_MACHINE_SESSION.getAttributeValue(ctx); deviceStateContext.onDisconnect("设备 inactive"); System.out.println("channelInactive:" + deviceStateContext.toString()); } @@ -205,7 +205,7 @@ public class NettyBusinessHandler extends SimpleChannelInboundHandler { public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { //==============发生异常切换到断开模式=============== System.out.println("exceptionCaught:" + cause.getMessage()); - DeviceStateContext deviceStateContext = session.getAttributeValue(ctx); + DeviceStateContext deviceStateContext = STATE_MACHINE_SESSION.getAttributeValue(ctx); deviceStateContext.onDisconnect(cause.getMessage()); System.out.println("exceptionCaught:" + deviceStateContext.toString()); diff --git a/collector-terminal-management/src/main/java/com/docus/server/controller/FileController.java b/collector-terminal-management/src/main/java/com/docus/server/controller/FileController.java new file mode 100644 index 0000000..91f59aa --- /dev/null +++ b/collector-terminal-management/src/main/java/com/docus/server/controller/FileController.java @@ -0,0 +1,16 @@ +package com.docus.server.controller; + +import org.springframework.web.bind.annotation.RestController; + +/** + * 文件上传下载 API + * + * @author AutoGenerator + * @since 2023-07-15 + */ +@RestController +public class FileController { + + + +} diff --git a/collector-terminal-management/src/test/java/com/docus/server/FileController.java b/collector-terminal-management/src/test/java/com/docus/server/FileController.java new file mode 100644 index 0000000..b7ca272 --- /dev/null +++ b/collector-terminal-management/src/test/java/com/docus/server/FileController.java @@ -0,0 +1,25 @@ +package com.docus.server; + +import com.docus.server.api.scheduling.management.FileApi; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +/** + * 文件上传下载 API + * + * @author AutoGenerator + * @since 2023-07-15 + */ +@SpringBootTest +public class FileController { + + @Resource + private FileApi fileApi; + + @Test + void test() { + } + +} diff --git a/docus-client-interface/src/main/java/com/docus/server/api/scheduling.management/FileApi.java b/docus-client-interface/src/main/java/com/docus/server/api/scheduling.management/FileApi.java index e4b53b6..6a4d03b 100644 --- a/docus-client-interface/src/main/java/com/docus/server/api/scheduling.management/FileApi.java +++ b/docus-client-interface/src/main/java/com/docus/server/api/scheduling.management/FileApi.java @@ -5,6 +5,8 @@ import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -25,6 +27,16 @@ import javax.servlet.http.HttpServletResponse; @RequestMapping("/sch/file") public interface FileApi { + @ApiOperation("文件下载") + @GetMapping("/download") + void downLoadFromUrl(@RequestParam(value = "urlStr") String urlStr, + @RequestParam(value = "fileName") String fileName, + @RequestParam(value = "savePath") String savePath) throws Exception; + + @ApiOperation("文件下载") + @GetMapping("/download") + ResponseEntity downloadFile(@RequestParam(value = "filePath") String filePath) throws Exception; + @ApiOperation("文件下载") @GetMapping("/download") void downloadFile(@RequestParam(value = "filePath") String filePath, HttpServletResponse response) throws Exception; diff --git a/pom.xml b/pom.xml index ba3e05e..8a5f9b2 100644 --- a/pom.xml +++ b/pom.xml @@ -250,6 +250,12 @@ 5.7.4 + + commons-httpclient + commons-httpclient + 3.1 + +