diff --git a/docus-api-common/src/main/java/com/docus/server/common/config/DefaultWebConfig.java b/docus-api-common/src/main/java/com/docus/server/common/config/DefaultWebConfig.java new file mode 100644 index 0000000..5f14bc4 --- /dev/null +++ b/docus-api-common/src/main/java/com/docus/server/common/config/DefaultWebConfig.java @@ -0,0 +1,17 @@ +package com.docus.server.common.config; + +import com.docus.infrastructure.WebConfig; +import com.docus.infrastructure.web.json.JsonSerializerModule; +import com.docus.server.common.serializer.DefJsonSerializerModule; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class DefaultWebConfig extends WebConfig { + + @Override + public JsonSerializerModule jsonSerializerModu1e() { + super.jsonSerializerModu1e(); + return new DefJsonSerializerModule(); + } + +} diff --git a/docus-api-common/src/main/java/com/docus/server/common/serializer/DefJsonSerializerModule.java b/docus-api-common/src/main/java/com/docus/server/common/serializer/DefJsonSerializerModule.java new file mode 100644 index 0000000..b5033ed --- /dev/null +++ b/docus-api-common/src/main/java/com/docus/server/common/serializer/DefJsonSerializerModule.java @@ -0,0 +1,25 @@ +package com.docus.server.common.serializer; + +import com.docus.core.util.Convert; +import com.docus.infrastructure.web.json.JsonSerializerModule; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.Date; +import java.util.TimeZone; + +public class DefJsonSerializerModule extends JsonSerializerModule { + public DefJsonSerializerModule() { + + addSerializer(Date.class, new JsonSerializer() { + @Override + public void serialize(Date date, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeString(date == null ? null : Convert.toString(date, Convert.DATA_FORMAT_DATETIME_SLASH, TimeZone.getTimeZone(ZoneId.systemDefault()))); + } + }); + } + +} diff --git a/docus-api-common/src/main/java/com/docus/server/common/serializer/StringToDateConverter.java b/docus-api-common/src/main/java/com/docus/server/common/serializer/StringToDateConverter.java new file mode 100644 index 0000000..4663077 --- /dev/null +++ b/docus-api-common/src/main/java/com/docus/server/common/serializer/StringToDateConverter.java @@ -0,0 +1,64 @@ +package com.docus.server.common.serializer; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Component; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * 全局handler前日期统一处理 + */ +@Component +public class StringToDateConverter implements Converter { + + private static final List DATE_FORMARTS = new ArrayList<>(4); + + static { + DATE_FORMARTS.add("yyyy-MM"); + DATE_FORMARTS.add("yyyy-MM-dd"); + DATE_FORMARTS.add("yyyy-MM-dd hh:mm"); + DATE_FORMARTS.add("yyyy-MM-dd hh:mm:ss"); + } + + @Override + public Date convert(String source) { + String value = source.trim(); + if ("".equals(value)) { + return null; + } + if (source.matches("^\\d{4}-\\d{1,2}$")) { + return parseDate(source, DATE_FORMARTS.get(0)); + } else if (source.matches("^\\d{4}-\\d{1,2}-\\d{1,2}$")) { + return parseDate(source, DATE_FORMARTS.get(1)); + } else if (source.matches("^\\d{4}-\\d{1,2}-\\d{1,2} {1}\\d{1,2}:\\d{1,2}$")) { + return parseDate(source, DATE_FORMARTS.get(2)); + } else if (source.matches("^\\d{4}-\\d{1,2}-\\d{1,2} {1}\\d{1,2}:\\d{1,2}:\\d{1,2}$")) { + return parseDate(source, DATE_FORMARTS.get(3)); + } else { + throw new IllegalArgumentException("Invalid boolean value '" + source + "'"); + } + } + + /** + * 格式化日期 + * + * @param dateStr String 字符型日期 + * @param format String 格式 + * @return Date 日期 + */ + public Date parseDate(String dateStr, String format) { + Date date = null; + try { + DateFormat dateFormat = new SimpleDateFormat(format); + date = dateFormat.parse(dateStr); + } catch (Exception e) { + System.out.println(String.format("日期%s转换%s错误", dateStr, format)); + } + return date; + } + +} diff --git a/docus-segmentation/WinSW.exe b/docus-segmentation/WinSW.exe new file mode 100644 index 0000000..6806bb4 Binary files /dev/null and b/docus-segmentation/WinSW.exe differ diff --git a/docus-segmentation/assembly.xml b/docus-segmentation/assembly.xml new file mode 100644 index 0000000..cbd39ff --- /dev/null +++ b/docus-segmentation/assembly.xml @@ -0,0 +1,73 @@ + + + exe + + dir + + false + + + + + /lib + ${basedir}/target/lib + + + + /config + ${basedir}/target/resources + 0755 + + *.xml + *.yml + *.properties + + + + + /dataConfig + ${basedir}/target/dataConfig + 0755 + + *.json + + + + + / + ${basedir}/target/resources/bin + 0755 + + *.bat + + + + + / + ${basedir}/target/resources/bin + 0755 + + *.xml + + + + + / + ${basedir} + 0755 + + *.exe + + + + + ${basedir}/target + / + 0755 + + ${project.build.finalName}.jar + + + + \ No newline at end of file diff --git a/docus-segmentation/pom.xml b/docus-segmentation/pom.xml new file mode 100644 index 0000000..ea960d3 --- /dev/null +++ b/docus-segmentation/pom.xml @@ -0,0 +1,185 @@ + + + docus-collector-server + com.docus + 1.0-SNAPSHOT + + 4.0.0 + docus-segmentation + Archetype - docus-segmentation + http://maven.apache.org + + + + com.docus + docus-api-common + 1.0-SNAPSHOT + compile + + + + org.freemarker + freemarker + 2.3.30 + + + + cn.smallbun.screw + screw-core + 1.0.3 + + + + + + + src/main/resources + true + + + + + + org.springframework.boot + spring-boot-maven-plugin + 2.4.4 + + ZIP + + + non-exists + non-exists + + + + + + + repackage + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + + target/lib + false + false + runtime + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.2.0 + + + copy-resources + package + + copy-resources + + + + + src/main/resources + + **/*.* + + + + ${project.build.directory}/resources + + + + copy-bin + package + + copy-resources + + + + + src/main/resources + true + + bin/*.xml + bin/*.bat + *.yml + *.ftl + + + + ${project.build.directory}/resources + + + + copy-data-config + package + + copy-resources + + + + + ../../dataConfig + true + + + ${project.build.directory}/dataConfig + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + **/*.yml + + + + + + maven-assembly-plugin + + + + ${project.artifactId} + false + + assembly.xml + + + make-assembly + package + + single + + + + + + + + + + diff --git a/docus-segmentation/src/main/java/com/docus/server/CodeGenerator.java b/docus-segmentation/src/main/java/com/docus/server/CodeGenerator.java new file mode 100644 index 0000000..bbe9875 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/CodeGenerator.java @@ -0,0 +1,33 @@ +package com.docus.server; + +import com.baomidou.mybatisplus.generator.AutoGenerator; +import com.docus.infrastructure.generator.BaseCodeGenerator; + +import java.util.HashMap; +import java.util.Map; + +public class CodeGenerator { + public static void main(String[] args) { + //字段名和枚举名的映射，不区分表名 + Map enumColumnMap = new HashMap<>(); + enumColumnMap.put("PROFILE_ITЕM", "AdminProfileItemEnum"); + + enumColumnMap.put("ORGANIZATION_TYPE", "OrganizationTypeEnum "); + enumColumnMap.put("OPERATE_STATUS", "OperateStatusEnum"); + enumColumnMap.put("STATE", "StateEnum"); + enumColumnMap.put("PRIVILEGE_LEVEL", "PrivilegeLevelEnum"); + enumColumnMap.put("FLAG", "FlagEnum"); + enumColumnMap.put("collect_type", "CollectTypeEnum"); + BaseCodeGenerator.setEnumColumnMap(enumColumnMap); + BaseCodeGenerator.setModuleName("collector-scheduling-management");//多个module，需要指定modulename + //指定entity生成到独立module里，并生成 api interface + BaseCodeGenerator.setClientInterfaceModuleName("docus-client-interface"); + BaseCodeGenerator.setClientInterfaceSubFolder("scheduling.management");//文件夹 + BaseCodeGenerator.setClientInterfaceBasePackage("com.docus.server"); + AutoGenerator defaultConfig = BaseCodeGenerator.getDefaultConfig(); + defaultConfig.getGlobalConfig().setSwagger2(true); + defaultConfig.getStrategy().setEntityLombokModel(true); + defaultConfig.getStrategy().setInclude("sch_collector_copy1");//需要生成的表，可指定多个，留空为全部生成 + BaseCodeGenerator.generate(defaultConfig); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/SegmentationBootstrap.java b/docus-segmentation/src/main/java/com/docus/server/SegmentationBootstrap.java new file mode 100644 index 0000000..607f354 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/SegmentationBootstrap.java @@ -0,0 +1,16 @@ +package com.docus.server; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableAsync; + +@EnableAsync +@EnableFeignClients(basePackages = {"com.docus.core.excel.feign", "com.docus.server.api"}) +@SpringBootApplication(scanBasePackages = {"com.docus"}) +public class SegmentationBootstrap { + public static void main(String[] args) { + System.setProperty("javax.xml.parsers.DocumentBuilderFactory", "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"); + SpringApplication.run(SegmentationBootstrap.class, args); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/MsgConstants.java b/docus-segmentation/src/main/java/com/docus/server/common/MsgConstants.java new file mode 100644 index 0000000..93cfeed --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/MsgConstants.java @@ -0,0 +1,68 @@ +package com.docus.server.common; + +public class MsgConstants { + + /** + * 客户端与采集调度器心跳 + */ + public static final String HEARTBEAT_REQUEST = "0"; + + /** + * 客户端上线注册 + */ + public static final String ONLINE_REGISTER = "1"; + + /** + * 客户端下线移除 + */ + public static final String OFFLINE_REMOVE = "2"; + + /** + * 客户端异常注册 + */ + public static final String EXCEPTION_REMOVE = "3"; + + /** + * 终端重启命令 + */ + public static final String TERMINATOR_RESTART = "4"; + + /** + * 采集器重启命令 + */ + public static final String COLLECTOR_RESTART = "5"; + + /** + * 虚拟机重启命令 + */ + public static final String VIRTUAL_RESTART = "6"; + + /** + * 更新采集器文件命令 + */ + public static final String UPDATE_COLLECTOR_FILE = "7"; + + /** + * 更新采集器配置命令 + */ + public static final String UPDATE_COLLECTOR_CONFIG = "8"; + + /** + * 采集调度器下发任务命令 + */ + public static final String SCH_DISTRIBUTE_TASKS = "9"; + /** + * 接收采集器上报的任务 + */ + public static final String REV_COLLECTOR_TASK = "10"; + + /** + * 是否有可用类型的采集器命令 + */ + public static final String HAS_VALID_COLLECTOR = "11"; + + /** + * 获取终端采集器数据量命令 + */ + public static final String HAS_COLLECTOR_COUNT = "12"; +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/SchVirtualLogTask.java b/docus-segmentation/src/main/java/com/docus/server/common/SchVirtualLogTask.java new file mode 100644 index 0000000..fccaf57 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/SchVirtualLogTask.java @@ -0,0 +1,52 @@ +package com.docus.server.common; + +import com.alibaba.fastjson.JSONObject; +import com.docus.server.api.scheduling.management.SchVirtualLogApi; +import com.docus.server.common.utils.SystemInfoUtils; +import com.docus.server.dto.scheduling.management.schvirtuallog.AddSchVirtualLogDTO; +import com.xxl.job.core.util.IpUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Date; + +@Component +@Slf4j +public class SchVirtualLogTask { + + @Resource + private SchVirtualLogApi schVirtualLogApi; + + //定时任务 + // 5 * * * * ? 在每分钟的5秒执行 + @Scheduled(cron = "${docus.vm-task-cron}") + public void runTask() { + try { + log.info("收集虚拟机信息定时任务： 开始执行"); + + JSONObject info = SystemInfoUtils.getInfo(); + + AddSchVirtualLogDTO addSchVirtualLogDTO = new AddSchVirtualLogDTO(); + addSchVirtualLogDTO.setCpuUtilization((String) ((JSONObject) info.get("cpuInfo")).get("cSys")); + addSchVirtualLogDTO.setMemoryTotal((String) ((JSONObject) info.get("memInfo")).get("total")); + addSchVirtualLogDTO.setMemoryAllowance((String) ((JSONObject) info.get("memInfo")).get("free")); + addSchVirtualLogDTO.setUplinkRate((String) ((JSONObject) info.get("networkInfo")).get("txPercent")); + addSchVirtualLogDTO.setDescendingRate((String) ((JSONObject) info.get("networkInfo")).get("rxPercent")); + addSchVirtualLogDTO.setIp(IpUtil.getIp()); + addSchVirtualLogDTO.setClientTime(new Date()); + addSchVirtualLogDTO.setDiskJson(info.get("sysFileInfo").toString()); + schVirtualLogApi.add(addSchVirtualLogDTO); + System.out.println(info); + + log.info("收集虚拟机信息定时任务： 执行完毕"); + } catch (Exception e) { + log.error("收集虚拟机信息定时任务执行出错", e); + } + + } + + +} + diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/Base64Utils.java b/docus-segmentation/src/main/java/com/docus/server/common/download/Base64Utils.java new file mode 100644 index 0000000..3159ce3 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/Base64Utils.java @@ -0,0 +1,154 @@ +package com.docus.server.common.download; + +import lombok.extern.slf4j.Slf4j; +import sun.misc.BASE64Decoder; +import sun.misc.BASE64Encoder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +@Slf4j +public class Base64Utils { + private static Base64Utils utils =null; + private Base64Utils(){ + + } + public static Base64Utils getInstance(){ + if(utils==null){ + synchronized (Base64Utils.class){ + if(utils == null){ + utils = new Base64Utils(); + } + } + } + return utils; + } + + public String imageToBase64Str(String imgFile) { + InputStream inputStream = null; + byte[] data = null; + try { + inputStream = new FileInputStream(imgFile); + data = new byte[inputStream.available()]; + inputStream.read(data); + inputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + // 加密 + BASE64Encoder encoder = new BASE64Encoder(); + return encoder.encode(data); + } + + + public byte[] base64StrToBytes(String imgStr) { + if (imgStr == null) { + return null; + } + String[] s = imgStr.split(","); + if(s.length>0){ + imgStr=s[1]; + } + BASE64Decoder decoder = new BASE64Decoder(); + try { + // 解密 + byte[] b = decoder.decodeBuffer(imgStr); + // 处理数据 + for (int i = 0; i < b.length; ++i) { + if (b[i] < 0) { + b[i] += 256; + } + } + return b; + } catch (Exception e) { + log.error("Base64 转 byte[] 出错啦！",e); + return null; + } + } + + + public boolean base64StrToImage(byte[] bytes, String path) { + if (bytes == null) { + return false; + } + try { + //文件夹不存在则自动创建 + File tempFile = new File(path); + if (!tempFile.getParentFile().exists()) { + tempFile.getParentFile().mkdirs(); + } + OutputStream out = new FileOutputStream(tempFile); + out.write(bytes); + out.flush(); + out.close(); + return true; + } catch (Exception e) { + log.error("图片文件下载失败："+e.getMessage()); + return false; + } + } + + public boolean base64StrToImage(String imgStr, String path) { + if (imgStr == null) { + return false; + } + String[] s = imgStr.split(","); + if(s.length>0){ + imgStr=s[1]; + } + BASE64Decoder decoder = new BASE64Decoder(); + try { + // 解密 + byte[] b = decoder.decodeBuffer(imgStr); + // 处理数据 + for (int i = 0; i < b.length; ++i) { + if (b[i] < 0) { + b[i] += 256; + } + } + //文件夹不存在则自动创建 + File tempFile = new File(path); + if (!tempFile.getParentFile().exists()) { + tempFile.getParentFile().mkdirs(); + } + OutputStream out = new FileOutputStream(tempFile); + out.write(b); + out.flush(); + out.close(); + return true; + } catch (Exception e) { + log.error("图片文件下载失败："+e.getMessage()); + return false; + } + } + + public int getFileSize(File inFile){ + InputStream in =null; + try{ + in= new FileInputStream(inFile); + int len =in.available(); + return len; + } + catch (Exception e){ + + } + finally { + try{ + in.close(); + } + catch (IOException e){ + e.printStackTrace(); + } + } + return -1; + } + + + public static void main(String[] args) { + System.out.println(Base64Utils.getInstance().imageToBase64Str("e:\\1.jpg")); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/CallBackPara.java b/docus-segmentation/src/main/java/com/docus/server/common/download/CallBackPara.java new file mode 100644 index 0000000..cd87c6e --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/CallBackPara.java @@ -0,0 +1,50 @@ +package com.docus.server.common.download; + +import lombok.Data; + +import java.sql.Timestamp; +import java.util.List; + +@Data +public class CallBackPara { + /** + * 主键 + */ + private String id; + /** + * 其他信息 + */ + private String mess; + + /** + * 本地存放路径 + */ + private String localpath; + + /** + * 文件名 + */ + private String filename; + + /** + * 下载开始时间 + */ + private Timestamp starttime; + + /** + * 下载完成时间 + */ + private Timestamp endtime; + + private List files; + + /** + * 下载存储文件的方式 0 传过来的格式，1转为 pdf 默认 ，2 转为图片jpg存储 + */ + private int fileStorageFormat=1; + + /** + * 0:病案，1：门急诊 2：省中医封存 + */ + private int datatype=0; +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/DownloadManager.java b/docus-segmentation/src/main/java/com/docus/server/common/download/DownloadManager.java new file mode 100644 index 0000000..1788611 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/DownloadManager.java @@ -0,0 +1,67 @@ +package com.docus.server.common.download; + +import com.docus.core.util.SpringUtils; +import com.docus.server.common.download.downLoader.HttpDownloader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + + +public class DownloadManager { + + private static Logger logger = LoggerFactory.getLogger(DownloadManager.class); + + enum ProtocolType {HTTP, HTTPS, FTP, BASE64, SMB, OTHER} + + ; + + public static void feedLinks(Collection list, int maxHttpRetry) { + for (RemoteLocalPair pair : list) { + ProtocolType deal = getLinkProtocol(pair.remoteUrl); + if (deal == ProtocolType.HTTP || deal == ProtocolType.HTTPS) { + HttpDownload(pair, maxHttpRetry); + } else if (deal == ProtocolType.FTP) { + } else if (deal == ProtocolType.BASE64) { + } else if (deal == ProtocolType.SMB) { + } else { + logger.error(pair.remoteUrl + " Url is not http or ftp!"); + } + } + } + + + public static void HttpDownload(RemoteLocalPair pair, int maxHttpRetry) { + //GeneralDownloadInfo info = new GeneralDownloadInfo(pair); + NoCheckPointInfo info = new NoCheckPointInfo(pair); + HttpDownloader downloader = new HttpDownloader(info, maxHttpRetry); + downloader.intiCallBack(SpringUtils.getBean(IDownCallBack.class)); + //可以用thread.start()不过不建议 + downloader.run(); + } + + + protected static ProtocolType getLinkProtocol(String link) { + int idx = link.indexOf("://"); + if (idx == -1) { + if (link.contains("base64")) { + return ProtocolType.BASE64; + } + return ProtocolType.OTHER; + } + String name = link.substring(0, idx); + if (name.equals("http")) { + return ProtocolType.HTTP; + } + if (name.equals("https")) { + return ProtocolType.HTTPS; + } + if (name.equals("ftp")) { + return ProtocolType.FTP; + } + if (name.equals("smb")) { + return ProtocolType.SMB; + } + return ProtocolType.OTHER; + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/FileCheckPoints.java b/docus-segmentation/src/main/java/com/docus/server/common/download/FileCheckPoints.java new file mode 100644 index 0000000..59ba410 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/FileCheckPoints.java @@ -0,0 +1,40 @@ +package com.docus.server.common.download; + +public class FileCheckPoints { + public int statecode; + public long timestamp = -99; + public long totalSize = -99; + private int split = -1; + private long[] startPos; + private long[] endPos; + + public long[] getStartPos() { + return startPos; + } + + public void setStartPos(long[] startPos) { + split = startPos.length; + this.startPos = startPos; + } + + public long[] getEndPos() { + return endPos; + } + + public void setEndPos(long[] endPos) { + split = endPos.length; + this.endPos = endPos; + } + + public int getSplit() { + return split; + } + + public void copy(FileCheckPoints _chp) { + this.setEndPos(_chp.endPos); + this.setStartPos(_chp.startPos); + this.totalSize = _chp.totalSize; + this.timestamp = _chp.timestamp; + } + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/FtpInfo.java b/docus-segmentation/src/main/java/com/docus/server/common/download/FtpInfo.java new file mode 100644 index 0000000..0101ee1 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/FtpInfo.java @@ -0,0 +1,11 @@ +package com.docus.server.common.download; + +import lombok.Data; + +@Data +public class FtpInfo { + private String ip; + private Integer port; + private String user; + private String pwd; +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java b/docus-segmentation/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java new file mode 100644 index 0000000..32b0d8f --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/GeneralDownloadInfo.java @@ -0,0 +1,200 @@ +package com.docus.server.common.download; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +public class GeneralDownloadInfo implements IDownloadInfo { + + /** + * 0:ftp地址，1:文件，2：用户名：3：密码，4：端口 + * @param fullUrl + * @return + */ + public static String[] getHostNameAndFilePath(String fullUrl) { + fullUrl = fullUrl.trim(); + String[] res = new String[5]; + int beginIndex = "ftp://".length(); + int endIndex = fullUrl.indexOf('/', beginIndex); + if (endIndex != -1) { + res[0] = fullUrl.substring(beginIndex, endIndex); + res[1] = fullUrl.substring(endIndex + 1); + } else { + res[0] = fullUrl.substring(beginIndex); + res[1] = Tools.getRandomUUID().toString() + ".none"; + } + String[] temp = res[0].split("@"); + if(temp.length>1){ + res[0] = temp[1]; + res[2] =temp[0]; + } + else + { + res[2]=""; + res[3]=""; + } + temp = res[0].split(":"); + if(temp.length>1){ + res[0] = temp[0]; + res[4] =temp[1]; + } + else + { + res[4]=""; + } + if(!StringUtils.isEmpty(res[2])){ + temp = res[2].split(":"); + if(temp.length>1){ + res[2] = temp[0]; + res[3] =temp[1]; + } + } + + return res; + } + + public GeneralDownloadInfo(RemoteLocalPair pair) { + this.pair = pair; + curName = pair.localName; + curFlagFile = new File(pair.getLocalFullPath()+".flags"); + if (!curFlagFile.getParentFile().exists()) { + curFlagFile.getParentFile().mkdirs(); + } + } + + @Override + public FileCheckPoints getCurCheckPoints() { + return chp; + } + + @Override + public RemoteLocalPair getPair() { + return this.pair; + } + + @Override + public void initDownload() { + + } + + + + @Override + public int getSplitNum(){ + return this.pair.splitNum; + } + @Override + public boolean isNeedDownload(FileCheckPoints serverInitChp) { + chp = readInfo(); + if (chp.timestamp == serverInitChp.timestamp && chp.totalSize == serverInitChp.totalSize && chp.getSplit() == serverInitChp.getSplit()) { + // 不需要下载 + boolean isneed = false; + for (int i = 0; i < chp.getStartPos().length; i++) { + if (chp.getStartPos()[i] < chp.getEndPos()[i]) { + isneed = true; + logger.info("restore from checkpoint"); + break; + } + } + return isneed; + } else { + chp.copy(serverInitChp); + return true; + } + } + + @Override + public synchronized boolean writeInfo(FileCheckPoints chkp) { + DataOutputStream dos = null; + try { + dos = new DataOutputStream(new FileOutputStream(curFlagFile)); + dos.writeLong(chkp.timestamp); + dos.writeLong(chkp.totalSize); + int split = chp.getSplit(); + dos.writeInt(split); + for(int i=0; i < chp.getSplit(); i++){ + dos.writeLong(chp.getStartPos()[i]); + dos.writeLong(chp.getEndPos()[i]); + } + } catch (FileNotFoundException e) { + logger.debug(curName, e); + } catch (IOException e) { + logger.debug(curName, e); + } finally { + if (dos != null) { + try { + dos.close(); + } catch (IOException e) { + logger.debug(curName, e); + } + } + } + return true; + } + + /** + * function:读取写入点的位置信息 如果不存在，那么 + */ + @Override + public FileCheckPoints readInfo() { + if (curFlagFile.exists()) { + DataInputStream dis = null; + try { + dis = new DataInputStream(new FileInputStream(curFlagFile)); + long curTimeStamp = dis.readLong(); + long curTotalSize = dis.readLong(); + int curSplit = dis.readInt(); + FileCheckPoints chkP = new FileCheckPoints(); + chkP.timestamp = curTimeStamp; + chkP.totalSize = curTotalSize; + long[] sp = new long[curSplit]; + long[] ep = new long[curSplit]; + for (int i=0; i< curSplit; i++){ + sp[i] = dis.readLong(); + ep[i] = dis.readLong(); + } + chkP.setStartPos(sp); + chkP.setEndPos(ep); + return chkP; + } catch (FileNotFoundException e) { + logger.debug(curName, e); + } catch (IOException e) { + logger.debug(curName, e); + } finally { + if (dis != null) { + try { + dis.close(); + } catch (IOException e) { + logger.debug(curName, e); + } + } + } + } else { + return new FileCheckPoints(); + } + return new FileCheckPoints(); + } + + @Override + public void downloadDone(FileCheckPoints chkp) { + writeInfo(chkp); + } + + + private Logger logger = LoggerFactory.getLogger(GeneralDownloadInfo.class); + + + private RemoteLocalPair pair; + private String curName; + private File curFlagFile; + private FileCheckPoints chp = null; + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/GlobalData.java b/docus-segmentation/src/main/java/com/docus/server/common/download/GlobalData.java new file mode 100644 index 0000000..fbfe4e6 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/GlobalData.java @@ -0,0 +1,143 @@ +package com.docus.server.common.download; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class GlobalData { + + /** + * @param args + */ + public static void main(String[] args) { + + } + + private Properties properties; + private String filePath; + + private GlobalData(String filePath) { + this.filePath = filePath; + properties = loadProperties(filePath); + } + + //=================================== + private Properties loadProperties(String filePath) { + InputStream is; + try { + is = new BufferedInputStream(new FileInputStream(new File(filePath))); + Properties properties = new Properties(); + properties.load(is); + return properties; + } catch (FileNotFoundException e) { +// e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); + } + return null; + } + + public Properties getProperties() { + return properties; + } + + public void setPropertity(String key, String value) { + FileOutputStream oFile; + try { + properties.setProperty(key, value); + oFile = new FileOutputStream(filePath); + properties.store(oFile, "add counter!"); + oFile.close(); + } catch (IOException e) { +// e.printStackTrace(); + } + } + + public String getTempDir() { + String path = properties.getProperty("TEMP_PATH"); + File dir = new File(path); + if (!dir.exists()) { + try { + dir.mkdir(); + } catch (SecurityException e) { +// e.printStackTrace(); + } + } + return path; + } + + public String getParserDir() { + String path = properties.getProperty("PARSER_DIR"); + File dir = new File(path); + if (!dir.exists()) { + try { + dir.mkdir(); + } catch (SecurityException e) { +// e.printStackTrace(); + } + } + return path; + } + + public String getHomoGoaUrl() { + return properties.getProperty("GO_HOMO_GOA_GZ_LINK"); + } + + public String getGoBasicOboUrl() { + return properties.getProperty("GO_OBO_BASIC_LINK"); + } + + public String getGoOboUrl() { + return properties.getProperty("GO_OBO_LINK"); + } + + public String getGoOwlUrl() { + return properties.getProperty("GO_OWL_LINK"); + } + + public String getGoOwlPlusUrl() { + return properties.getProperty("GO_OWL_PLUS_LINK"); + } + + public String getDoOboUrl() { + return properties.getProperty("DO_OBO_LINK"); + } + + public String getProOboUrl() { + return properties.getProperty("PRO_OBO_LINK"); + } + + public String getChebiOboUrl() { + return properties.getProperty("CHEBI_OBO_LINK"); + } + + public String getLocalGoBasicPath() { + String goUrl = getGoBasicOboUrl(); + return getTempDir() + File.separator + GlobalData.getFileName(goUrl); + } + + //==================================== + private static GlobalData single; + private static Object lock = new Object(); + + //==================================== + public static String getFileName(String url) { + return url.substring(url.lastIndexOf("/") + 1, url.length()); + } + + public static GlobalData getInstance() { + synchronized (lock) { + if (single == null) { + single = new GlobalData("configure/settings.properties"); + } + return single; + } + } + + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/IDownCallBack.java b/docus-segmentation/src/main/java/com/docus/server/common/download/IDownCallBack.java new file mode 100644 index 0000000..7a96913 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/IDownCallBack.java @@ -0,0 +1,7 @@ +package com.docus.server.common.download; + +public interface IDownCallBack { + void success(CallBackPara para); + + void fail(CallBackPara para); +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/IDownloadInfo.java b/docus-segmentation/src/main/java/com/docus/server/common/download/IDownloadInfo.java new file mode 100644 index 0000000..7b8f03c --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/IDownloadInfo.java @@ -0,0 +1,60 @@ +package com.docus.server.common.download; + + +public interface IDownloadInfo { + /** + * 初始化download这个阶段可以做连接数据库的操作 + */ + void initDownload(); + + /** + * 判断是否需要下载，从数据库获取断点数据 + * + * @param serverInitChp checkpoint + * @return + */ + boolean isNeedDownload(FileCheckPoints serverInitChp); + + /** + * 写入断点，可以进行数据库写入操作 + * + * @param chkp checkpoint + * @return + */ + boolean writeInfo(FileCheckPoints chkp); + + /** + * 获取当前的checkPoint + * + * @return + */ + FileCheckPoints getCurCheckPoints(); + + /** + * 读取断点信息，可以进行数据库操作 + * + * @return + */ + FileCheckPoints readInfo(); + + /** + * 最后下载完成之后的操作 + * + * @param chkp + */ + void downloadDone(FileCheckPoints chkp); + + /** + * 获取当前链接和本地保存的路径信息 + * + * @return + */ + RemoteLocalPair getPair(); + + /** + * 文件分块，默认为1 + * + * @return + */ + int getSplitNum(); +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/MultiDownFile.java b/docus-segmentation/src/main/java/com/docus/server/common/download/MultiDownFile.java new file mode 100644 index 0000000..e4a4e6d --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/MultiDownFile.java @@ -0,0 +1,38 @@ +package com.docus.server.common.download; + +public class MultiDownFile { + public MultiDownFile(String remoteUrl, String localPath, String localName) { + this.remoteUrl = remoteUrl; + this.localPath = localPath; + this.localName = localName; + } + + private String remoteUrl; + private String localPath; + private String localName; + + public String getRemoteUrl() { + return remoteUrl; + } + + public void setRemoteUrl(String remoteUrl) { + this.remoteUrl = remoteUrl; + } + + public String getLocalPath() { + return localPath; + } + + public void setLocalPath(String localPath) { + this.localPath = localPath; + } + + public String getLocalName() { + return localName; + } + + public void setLocalName(String localName) { + this.localName = localName; + } + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java b/docus-segmentation/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java new file mode 100644 index 0000000..05f77c7 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/NoCheckPointInfo.java @@ -0,0 +1,73 @@ +package com.docus.server.common.download; + + +import java.io.File; + +/** + * 使用这个info可以实现无断线续传功能的下载 + * + * @author burkun + */ +public class NoCheckPointInfo implements IDownloadInfo { + + + private boolean isDownloding; + + public NoCheckPointInfo(RemoteLocalPair pair) { + this.pair = pair; + File file = new File(pair.localPath); + if (!file.exists()) { + file.mkdirs(); + } + } + + @Override + public void initDownload() { + isDownloding = true; + } + + @Override + public boolean isNeedDownload(FileCheckPoints serverInitChp) { + chp = serverInitChp; + return true; + } + + @Override + public boolean writeInfo(FileCheckPoints chkp) { + chp = chkp; + return true; + } + + @Override + public FileCheckPoints getCurCheckPoints() { + return chp; + } + + @Override + public FileCheckPoints readInfo() { + return null; + } + + @Override + public void downloadDone(FileCheckPoints chkp) { + isDownloding = false; + } + + @Override + public RemoteLocalPair getPair() { + return pair; + } + + @Override + public int getSplitNum() { + return 1; + } + + public boolean IsDownloading() { + return isDownloding; + } + + private RemoteLocalPair pair; + private FileCheckPoints chp = null; + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/RemoteLocalPair.java b/docus-segmentation/src/main/java/com/docus/server/common/download/RemoteLocalPair.java new file mode 100644 index 0000000..bf687c1 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/RemoteLocalPair.java @@ -0,0 +1,72 @@ +package com.docus.server.common.download; + +import java.io.File; +import java.util.List; + + +/** + * 需要的下载和保存的文件对 + * + * @author burkun + */ +public class RemoteLocalPair { + public String id; + public int splitNum = 1; + public String remoteUrl; + public String localPath; + public String localName; + + public String getLocalFullPath() { + return localPath + File.separator + localName; + } + + public String user; + public String pwd; + public Integer port; + public List separators; + public String failurl; + public List proxyurls; + public List smbips; + /** + * 下载存储文件的方式 0 传过来的格式，1转为 pdf 默认 ，2 转为图片jpg存储 + */ + public int fileStorageFormat = 1; + /** + * 0:病案，1：门急诊 2:省中医封存 + */ + public int datatype = 0; + + /** + * @param remoteUrl 远程url + * @param localPath 本地路径 + * @param localName 本地文件名 + */ + public RemoteLocalPair(String remoteUrl, String localPath, String localName) { + this.remoteUrl = remoteUrl; + if (localName.length() == 0) { + this.localName = getFileName(remoteUrl); + ; + } else { + this.localName = localName; + } + if (localPath.length() == 0) { + this.localPath = GlobalData.getInstance().getTempDir(); + } else { + this.localPath = localPath; + } + } + + /** + * @param remoteUrl 远程url + * @param localPath 本地路径 默认文件名从url中提取 + */ + public RemoteLocalPair(String remoteUrl, String localPath) { + this.remoteUrl = remoteUrl; + this.localName = getFileName(remoteUrl); + this.localPath = localPath; + } + + private String getFileName(String url) { + return url.substring(url.lastIndexOf("/") + 1, url.length()); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/SaveFileItem.java b/docus-segmentation/src/main/java/com/docus/server/common/download/SaveFileItem.java new file mode 100644 index 0000000..485eee5 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/SaveFileItem.java @@ -0,0 +1,73 @@ +package com.docus.server.common.download; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.RandomAccessFile; + + +/** + * function: 写入文件、保存文件 + */ +public class SaveFileItem { + //存储文件 + private RandomAccessFile itemFile; + private String name; + private Logger logger = LoggerFactory.getLogger(SaveFileItem.class); + + /** + * @param name 文件路径、名称 + * @param pos 写入点位置position + * @throws IOException + */ + public SaveFileItem(String name, long pos) throws IOException { + this.name = name; + itemFile = new RandomAccessFile(name, "rwd"); + //在指定的pos位置写入数据 + itemFile.seek(pos); + } + + /** + * function: 同步方法写入文件 + * @author hoojo + * @createDate 2011-9-26 下午12:21:22 + * @param buff 缓冲数组 + * @param start 起始位置 + * @param length 长度 + * @return + */ + public synchronized int write(byte[] buff, int start, int length) { + int i = -1; + try { + itemFile.write(buff, start, length); + i = length; + } catch (IOException e) { + logger.debug(name, e); + } + return i; + } + + public void close() throws IOException { + if (itemFile != null) { + itemFile.close(); + } + } + + public String getFileName(){ + return this.name; + } + /** + * 设置文件大小，在old 文件大于新文件时特别有用 + * @param newLength 新的文件长度 + */ + public void setLength(long newLength){ + try { + if(newLength != itemFile.length()){ + itemFile.setLength(newLength); + } + } catch (IOException e) { + logger.debug(name, e); + } + } +} \ No newline at end of file diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/Tools.java b/docus-segmentation/src/main/java/com/docus/server/common/download/Tools.java new file mode 100644 index 0000000..a50caaf --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/Tools.java @@ -0,0 +1,30 @@ +package com.docus.server.common.download; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.UUID; + +public class Tools { + public static UUID getRandomUUID() { + return UUID.randomUUID(); + } + + /** + * @param filename propertiy file path + * @return + * @author burkun + */ + public static Properties readPropertiesFile(String filename) { + Properties properties = new Properties(); + try { + InputStream inputs = new FileInputStream(filename); + properties.load(inputs); + inputs.close(); + } catch (IOException e) { +// e.printStackTrace(); + } + return properties; + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java b/docus-segmentation/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java new file mode 100644 index 0000000..506b459 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/download/downLoader/HttpDownloader.java @@ -0,0 +1,559 @@ +package com.docus.server.common.download.downLoader; + +import com.alibaba.fastjson.JSON; +import com.docus.server.common.download.CallBackPara; +import com.docus.server.common.download.FileCheckPoints; +import com.docus.server.common.download.IDownCallBack; +import com.docus.server.common.download.IDownloadInfo; +import com.docus.server.common.download.MultiDownFile; +import com.docus.server.common.download.SaveFileItem; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.httpclient.util.URIUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * + */ +@Slf4j +public class HttpDownloader extends Thread { + + private Logger logger = LoggerFactory.getLogger(HttpDownloader.class); + private IDownloadInfo info; + private int maxRetry = 5; + private IDownCallBack downCallBack; + private String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"; + + public HttpDownloader(IDownloadInfo info, int maxRetry) { + this.info = info; + this.maxRetry = maxRetry; + } + + /** + * 初始化回调方法 + * + * @param downCallBack + */ + public void intiCallBack(IDownCallBack downCallBack) { + this.downCallBack = downCallBack; + } + + + public HttpDownloader(IDownloadInfo info) { + this.info = info; + } + + public String[] removeArraysEmpty(String[] arr) { + return Arrays.stream(arr).filter(s -> !"".equals(s)).toArray(String[]::new); + } + + @Override + public void run() { + CallBackPara para = new CallBackPara(); + para.setId(info.getPair().id); + para.setFilename(info.getPair().localName); + para.setLocalpath(info.getPair().localPath); + para.setStarttime(getNewtime()); + para.setFileStorageFormat(info.getPair().fileStorageFormat); + para.setDatatype(info.getPair().datatype); + String url = info.getPair().remoteUrl; + List files = new ArrayList<>(); + + String[] urls = removeArraysEmpty(url.split("http://|https://")); + if (urls.length > 1) { + for (int i = 0; i < urls.length; i++) { + String urltemp = ""; + if (url.indexOf("http://" + urls[i]) >= 0) { + urltemp = "http://" + urls[i]; + } + if (url.indexOf("https://" + urls[i]) >= 0) { + urltemp = "https://" + urls[i]; + } + for (String o : this.info.getPair().separators) { + urltemp = urltemp.replaceAll(o + "$", ""); + } + files.add(new MultiDownFile(urltemp, info.getPair().localPath, "docustemp_" + i + "_" + info.getPair().localName)); + } + } else { + files.add(new MultiDownFile(url, info.getPair().localPath, info.getPair().localName)); + } + para.setFiles(files); + if (info.getPair().proxyurls != null) { + Pattern p = Pattern.compile(String.join("|", info.getPair().proxyurls)); + for (MultiDownFile o : files) { + Matcher matcher = p.matcher(o.getRemoteUrl()); + if (matcher.find()) { + o.setRemoteUrl(String.format(info.getPair().failurl, o.getRemoteUrl())); + } + } + } +// URLHttpDownBootstrapBuilder builder=null; +// HttpDownBootstrap bootstrap; + try { + for (MultiDownFile file : files) { +// try { +// url = EncoderUrl(file.getRemoteUrl()); +// } catch (Exception e) { +// +// } + downLoadFromUrl(url, file.getLocalName(), file.getLocalPath()); + if (downCallBack != null) { + downCallBack.success(para); + } + //防止过快，第三链接无法支持 + try { + Thread.sleep(100); + } catch (Exception e) { + + } +// builder = HttpDownBootstrap.builder(url); +// builder.downConfig(new HttpDownConfigInfo() +// .setFilePath(file.getLocalPath()) +// ).callBackPara(para); +// builder.response(new HttpResponseInfo(file.getLocalName())); +// bootstrap = builder.callback(new ConsoleHttpDownCallback()).build(); +// bootstrap.start(); +// bootstrap = null; +// builder = null; + } + } catch (Exception e) { +// e.printStackTrace(); + log.error("nio下载失败，" + JSON.toJSONString(para) + ";失败信息：" + e.getMessage()); + if (downCallBack != null) { + downCallBack.fail(para); + } + } +// finally { +// bootstrap = null; +// builder = null; +// } + } + + public String EncoderUrl(String url) throws UnsupportedEncodingException { + String resultURL = ""; + for (int i = 0; i < url.length(); i++) { + char charAt = url.charAt(i); + //只对汉字处理 + if (isChineseChar(charAt)) { + String encode = URLEncoder.encode(charAt + "", "UTF-8"); + resultURL += encode; + } else { + resultURL += charAt; + } + } + return resultURL; + } + + public boolean isChineseChar(char c) { + return String.valueOf(c).matches("[\u4e00-\u9fa5]"); + } + + /** + * 处理多级302等跳转 + * + * @param uc + * @return + * @throws Exception + */ + private HttpURLConnection reload(HttpURLConnection uc) throws Exception { + + HttpURLConnection huc = uc; + + if (huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP + || huc.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) {// 302, 301 + String url = huc.getHeaderField("Location"); + url = url.replace("\\", "/"); + return reload((HttpURLConnection) new URL(url).openConnection()); + } + return uc; + } + + + public void downLoadFromUrl(String urlStr, String fileName, String savePath) throws Exception { + long start = System.currentTimeMillis(); + urlStr = urlStr.replace("\\", "/"); + urlStr = URIUtil.encodePathQuery(urlStr); + URL url = new URL(urlStr); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + try { + boolean useHttps = urlStr.toLowerCase().startsWith("https"); + if (useHttps) { + HttpsURLConnection https = (HttpsURLConnection) conn; + trustAllHosts(https); + https.setHostnameVerifier(DO_NOT_VERIFY); + } + //设置超时间为3秒 + //防止屏蔽程序抓取而返回403错误 + conn.setRequestProperty("User-Agent", userAgent); + conn.setRequestProperty("Accept-Encoding", "identity"); + conn.setConnectTimeout(8 * 1000); + conn.setReadTimeout(8 * 1000); + conn = reload(conn); + long length = conn.getContentLength(); + if (length < 0) { + String values = conn.getHeaderField("Content-Length"); + if (values != null && !values.isEmpty()) { + length = Long.parseLong(values); + } + } +// log.info(urlStr+" 文件大小:"+length); + InputStream inputStream = null; + if (conn.getResponseCode() >= 400) { + throw new Exception("文件不存在"); +// inputStream = conn.getErrorStream(); + } else { + inputStream = conn.getInputStream(); + } + //得到输入流 + //InputStream inputStream = conn.getInputStream(); + //获取自己数组 + byte[] getData = readInputStream(inputStream); + + //文件保存位置 + File saveDir = new File(savePath); + if (!saveDir.exists()) { + saveDir.mkdirs(); + } + File file = new File(saveDir + File.separator + fileName); + + FileOutputStream fos = new FileOutputStream(file); + fos.write(getData); + if (fos != null) { + fos.close(); + } + if (inputStream != null) { + inputStream.close(); + } + long end = System.currentTimeMillis(); + logger.info("info:" + url + " download success;用时:" + (end - start) + "ms"); + } catch (Exception ex) { + throw ex; + } finally { + // 断开连接，释放资源 + conn.disconnect(); + } + } + + + /** + * 从输入流中获取字节数组 + * + * @param inputStream + * @return + * @throws IOException + */ + public byte[] readInputStream(InputStream inputStream) throws IOException { + byte[] buffer = new byte[1024]; + int len = 0; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + while ((len = inputStream.read(buffer)) != -1) { + bos.write(buffer, 0, len); + } + bos.close(); + return bos.toByteArray(); + } + + + public static FileCheckPoints initCheckPoint(int splitNum, long totalSize, long timeStamp) { + long[] startPos = new long[splitNum]; + long[] endPos = new long[splitNum]; + for (int i = 0, len = startPos.length; i < len; i++) { + long size = i * (totalSize / len); + startPos[i] = size; + // 设置最后一个结束点的位置 + if (i == len - 1) { + endPos[i] = totalSize; + } else { + size = (i + 1) * (totalSize / len); + endPos[i] = size; + } + } + FileCheckPoints chp = new FileCheckPoints(); + chp.setEndPos(endPos); + chp.setStartPos(startPos); + chp.totalSize = totalSize; + chp.timestamp = timeStamp; + return chp; + } + + private FileCheckPoints getInitedCheckPoint() { + long fileLength = -1; + long timeStamp = -1; + HttpURLConnection conn = null; + int stateCode = 0; + try { + URL url = new URL(this.info.getPair().remoteUrl); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("Accept-Encoding", "identity"); + HttpDownloader.RetriveSingleStream.setHeader(conn); + stateCode = conn.getResponseCode(); + // 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK + if (stateCode != HttpURLConnection.HTTP_OK + && stateCode != HttpURLConnection.HTTP_PARTIAL) { + logger.warn(info.getPair().remoteUrl + " #Error Code:# " + + stateCode); + fileLength = -2; + } else if (stateCode >= 400) { + logger.warn(info.getPair().remoteUrl + " #Error Code:# " + + stateCode); + fileLength = -2; + } else { + // 获取长度 + fileLength = conn.getContentLengthLong(); + timeStamp = conn.getLastModified(); + logger.info(info.getPair().remoteUrl + " #FileLength:# " + + fileLength); + } + } catch (MalformedURLException e) { +// e.printStackTrace(); + } catch (IOException e) { +// e.printStackTrace(); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + FileCheckPoints chp; + if (fileLength > 0) { + chp = initCheckPoint(info.getSplitNum(), fileLength, timeStamp); + chp.timestamp = timeStamp; + } else { + chp = new FileCheckPoints(); + chp.statecode = stateCode; + } + return chp; + } + + /** + * bug fixed change the RandomAccessFile size + * + * @author burkun + */ + + + protected static class RetriveSingleStream implements Runnable { + private boolean isDone = false; + private FileCheckPoints chp; + private int curIndex; + private SaveFileItem file; + private long startPos; + private long endPos; + byte[] buffer = new byte[1024 * 12]; + private IDownloadInfo __info; + private int maxRetry; + private Logger logger = LoggerFactory.getLogger(RetriveSingleStream.class); + + public boolean isDone() { + return isDone; + } + + public RetriveSingleStream(IDownloadInfo info, FileCheckPoints chp, + int curIndex, int maxRetry) { + this.__info = info; + this.chp = chp; + this.curIndex = curIndex; + this.startPos = chp.getStartPos()[curIndex]; + this.endPos = chp.getEndPos()[curIndex]; + this.maxRetry = maxRetry; + } + + @Override + public void run() { + InputStream in = null; + HttpURLConnection conn = null; + int curRetry = 0; + + while (curRetry < maxRetry && !isDone) { + try { + URL url = new URL(__info.getPair().remoteUrl); + conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(10000); + conn.setReadTimeout(30000); + setHeader(conn); + String property = "bytes=" + startPos + "-"; + conn.setRequestProperty("RANGE", property); + logger.info(__info.getPair().localName + " #Block" + + (curIndex + 1) + "# begin downloading..."); + int length; + long counter = 0; + InputStream is = conn.getInputStream(); + file = new SaveFileItem(__info.getPair().getLocalFullPath(), startPos); + //--bug fixed + file.setLength(__info.getCurCheckPoints().totalSize); + //--bug fixed + while (!isDone && startPos < endPos && (length = is.read(buffer)) > 0) { + startPos += file.write(buffer, 0, length); + counter += 1; + chp.getStartPos()[curIndex] = Math.min(startPos, endPos); + if (counter % 20 == 0) { + __info.writeInfo(chp); + logger.info(__info.getPair().remoteUrl + " #Block" + + (curIndex + 1) + "# download " + + getPercentage() + "%..."); + Thread.yield(); + } + } + __info.writeInfo(chp); + isDone = true; + } catch (IOException e) { + isDone = false; + logger.debug(__info.getPair().remoteUrl, e); + } finally { + if (!isDone) { + curRetry++; + logger.debug(__info.getPair().remoteUrl + " download failed, retry again!"); + if (curRetry >= maxRetry) { + //保证循环跳出 + isDone = true; + } + } else { + curRetry = maxRetry; + } + try { + if (in != null) { + in.close(); + } + if (file != null) { + file.close(); + } + if (conn != null) { + conn.disconnect(); + } + } catch (IOException e) { + logger.debug(__info.getPair().remoteUrl, e); + } + } + } + } + + + public static void setHeader(URLConnection conn) { + conn.setRequestProperty( + "User-Agent", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 BIDUBrowser/7.0 Safari/537.36"); + conn.setRequestProperty("Accept-Language", + "en-us,en;q=0.7,zh-cn;q=0.3"); + conn.setRequestProperty("Accept-Encoding", "utf-8"); + conn.setRequestProperty("Accept-Charset", + "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); + conn.setRequestProperty("Keep-Alive", "300"); + conn.setRequestProperty("connnection", "keep-alive"); + // conn.setRequestProperty("If-Modified-Since", + // "Fri, 02 Jan 2009 17:00:05 GMT"); + // conn.setRequestProperty("If-None-Match", + // "\"1261d8-4290-df64d224\""); + conn.setRequestProperty("Cache-conntrol", "max-age=0"); + conn.setRequestProperty("Referer", "http://www.baidu.com"); + } + + private int getPercentage() { + long total = 0; + for (int i = 0; i < chp.getSplit(); i++) { + total += chp.getEndPos()[i] - chp.getStartPos()[i]; + } + return (int) ((chp.totalSize - total) * 100 / chp.totalSize); + } + + } + + private Timestamp getNewtime() { + Date now = new Date(); + Timestamp timestamp = new Timestamp(now.getTime()); + return timestamp; + } + +// public static void main(String[] args) { +// String url="http://ss,ss,https://bbbbb;http://ccc"; +// List separators = new ArrayList<>(); +// separators.add(","); +// separators.add(";"); +// String[] urls = url.split("http://|https://"); +// for (int i = 0; i < urls.length; i++) { +// String urltemp = ""; +// if (url.indexOf("http://" + urls[i]) >= 0) { +// urltemp="http://" + urls[i]; +// } +// if (url.indexOf("https://" + urls[i]) >= 0) { +// urltemp="https://" + urls[i]; +// } +// for(String o:separators){ +// urltemp=urltemp.replaceAll(o+"$", ""); +// } +// System.out.println(urltemp); +// } +// } + /** + * 覆盖java默认的证书验证 + */ + private final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[]{}; + } + }}; + + /** + * 设置不验证主机 + */ + private final HostnameVerifier DO_NOT_VERIFY = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + /** + * 信任所有 + * + * @param connection + * @return + */ + private SSLSocketFactory trustAllHosts(HttpsURLConnection connection) { + SSLSocketFactory oldFactory = connection.getSSLSocketFactory(); + try { + SSLContext sc = SSLContext.getInstance("TLS"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + SSLSocketFactory newFactory = sc.getSocketFactory(); + connection.setSSLSocketFactory(newFactory); + } catch (Exception e) { + e.printStackTrace(); + } + return oldFactory; + } + +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/utils/StartUpExeUtils.java b/docus-segmentation/src/main/java/com/docus/server/common/utils/StartUpExeUtils.java new file mode 100644 index 0000000..cdb387e --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/utils/StartUpExeUtils.java @@ -0,0 +1,198 @@ +package com.docus.server.common.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +public class StartUpExeUtils { + + public static void main(String[] args) throws Exception { +// startUpExeOnly("H:\\tools\\Navicat Premium 12免安装\\Navicat Premium 12\\navicat.exe"); +// killExeOnly("navicat.exe"); + +// startUpExe("H:\\tools\\Navicat Premium 12免安装\\Navicat Premium 12\\navicat.exe", "navicat.exe"); + restartTerminal(""); + } + + private static boolean startExeStatus = true;//exe启动状态 + + //重启终端 + public static String restartTerminal(String bat) throws IOException, InterruptedException { + Process process = Runtime.getRuntime().exec(bat); + + int exitCode = process.waitFor(); + + if (exitCode == 0) { + System.out.println("Command executed successfully."); + return "Command executed successfully."; + } else { + System.out.println("Command execution failed."); + return "Command executed failed."; + } + } + + //重启虚拟机 + public static String restartComputer() throws IOException, InterruptedException { + Process process = Runtime.getRuntime().exec("shutdown /r /t 0"); + + int exitCode = process.waitFor(); + + if (exitCode == 0) { + System.out.println("Command executed successfully."); + return "Command executed successfully."; + } else { + System.out.println("Command execution failed."); + return "Command executed failed."; + } + } + + //仅启动exe客户端，不检查进程 + public static void startUpExeOnly(String exePath) throws IOException { + if (exePath != "") { + Runtime.getRuntime().exec(exePath); + } + } + + /** + * 仅kill指定进程 + * + * @param procName + * @throws IOException + */ + public static void killExeOnly(String procName) throws IOException { + if (procName != "") { + String command = "taskkill /F /IM " + procName; + Runtime.getRuntime().exec("cmd /c " + command); + } + } + + //启动exe客户端 + public static boolean startUpExe(String exePath, String procName) { + if (exePath != "" && procName != "") { + String result = checkProcess(procName);//检查exe进程 + if (result.isEmpty()) { + try { + Runtime.getRuntime().exec(exePath); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("程序：" + exePath + "不存在！"); + } + } + } + return startExeStatus; + } + + //启动exe客户端,并传参 + public static boolean startUpExe(String exePath, String procName, int subId, String curModeId, String riskSet1, String riskSet2, String riskSet3) { + if (exePath != "" && procName != "") { + String result = checkProcess(procName);//检查exe进程 + if (result.isEmpty()) { + try { + //启动exe执行程序 + String[] cmd = {exePath, subId + "", curModeId, riskSet1, riskSet2, riskSet3}; + Runtime.getRuntime().exec(cmd); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("程序：" + exePath + "不存在！"); + } + } + } + return startExeStatus; + } + + /** + * 检查进程是否存在，存在则杀死进程 + * + * @param procName + */ + public static String checkProcess(String procName) { + String result = ""; + //判断是否存在进程 + Boolean existProc = false; + BufferedReader bufferedReader = null; + try { + Process proc = Runtime.getRuntime().exec("tasklist -fi " + '"' + "imagename eq " + procName + '"'); + bufferedReader = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + if (line.contains(procName)) { + existProc = true;//存在 + } + } + } catch (Exception ex) { + result = "查询程序进程异常：" + ex.getMessage(); + System.out.println("查询程序进程异常：" + ex.getMessage()); + return result; + } finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } catch (Exception ex) { + } + } + } + + // 存在，则先杀死该进程 + if (existProc) { + BufferedReader br = null; + try { + if (procName != "") { + //执行cmd命令 + String command = "taskkill /F /IM " + procName; + Runtime runtime = Runtime.getRuntime(); + Process process = runtime.exec("cmd /c " + command); + br = new BufferedReader(new InputStreamReader(process.getInputStream(), "UTF-8")); + String line = null; + StringBuilder build = new StringBuilder(); + while ((line = br.readLine()) != null) { + build.append(line); + } + } + } catch (Exception e) { + result = "关闭程序进程异常：" + e.getMessage(); + System.out.println("关闭程序进程异常：" + e.getMessage()); + return result; + } finally { + if (br != null) { + try { + br.close(); + } catch (Exception ex) { + } + } + } + } + return result; + } + + /** + * 仅检查进程是否存在 + * + * @param procName + */ + public static boolean checkProcessOnly(String procName) { + //判断是否存在进程 + Boolean existProc = false; + BufferedReader bufferedReader = null; + try { + Process proc = Runtime.getRuntime().exec("tasklist -fi " + '"' + "imagename eq " + procName + '"'); + bufferedReader = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + if (line.contains(procName)) { + existProc = true;//存在 + } + } + } catch (Exception ex) { + System.out.println("查询程序进程异常：" + ex.getMessage()); + return existProc; + } finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } catch (Exception ex) { + } + } + } + return existProc; + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/common/utils/SystemInfoUtils.java b/docus-segmentation/src/main/java/com/docus/server/common/utils/SystemInfoUtils.java new file mode 100644 index 0000000..9219cc6 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/common/utils/SystemInfoUtils.java @@ -0,0 +1,313 @@ +package com.docus.server.common.utils; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.hardware.GlobalMemory; +import oshi.hardware.HardwareAbstractionLayer; +import oshi.hardware.NetworkIF; +import oshi.software.os.FileSystem; +import oshi.software.os.OSFileStore; +import oshi.software.os.OperatingSystem; +import oshi.util.Util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.DecimalFormat; +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.StringTokenizer; + +/** + * 系统消息工具类 + **/ +public class SystemInfoUtils { + + private static final int OSHI_WAIT_SECOND = 1000; + private static final int SLEEP_TIME = 2 * 1000; + private static SystemInfo systemInfo = new SystemInfo(); + private static HardwareAbstractionLayer hardware = systemInfo.getHardware(); + private static OperatingSystem operatingSystem = systemInfo.getOperatingSystem(); + + public static void main(String[] args) throws Exception { + JSONObject info = getInfo(); + System.out.println(info); + } + + public static JSONObject getCpuInfo() { + JSONObject cpuInfo = new JSONObject(); + CentralProcessor processor = hardware.getProcessor(); + // CPU信息 + long[] prevTicks = processor.getSystemCpuLoadTicks(); + Util.sleep(OSHI_WAIT_SECOND); + long[] ticks = processor.getSystemCpuLoadTicks(); + long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()]; + long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()]; + long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()]; + long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()]; + long cSys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()]; + long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()]; + long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()]; + long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()]; + long totalCpu = user + nice + cSys + idle + iowait + irq + softirq + steal; + //cpu核数 + cpuInfo.put("cpuNum", processor.getLogicalProcessorCount()); + //cpu系统使用率 + cpuInfo.put("cSys", new DecimalFormat("#.##%").format(cSys * 1.0 / totalCpu)); + //cpu用户使用率 + cpuInfo.put("user", new DecimalFormat("#.##%").format(user * 1.0 / totalCpu)); + //cpu当前等待率 + cpuInfo.put("iowait", new DecimalFormat("#.##%").format(iowait * 1.0 / totalCpu)); + //cpu当前使用率 + cpuInfo.put("idle", new DecimalFormat("#.##%").format(1.0 - (idle * 1.0 / totalCpu))); + return cpuInfo; + } + + /** + * 系统jvm信息 + */ + public static JSONObject getJvmInfo() { + JSONObject cpuInfo = new JSONObject(); + Properties props = System.getProperties(); + Runtime runtime = Runtime.getRuntime(); + long jvmTotalMemoryByte = runtime.totalMemory(); + long freeMemoryByte = runtime.freeMemory(); + //jvm总内存 + cpuInfo.put("total", formatByte(runtime.totalMemory())); + //空闲空间 + cpuInfo.put("free", formatByte(runtime.freeMemory())); + //jvm最大可申请 + cpuInfo.put("max", formatByte(runtime.maxMemory())); + //vm已使用内存 + cpuInfo.put("user", formatByte(jvmTotalMemoryByte - freeMemoryByte)); + //jvm内存使用率 + cpuInfo.put("usageRate", new DecimalFormat("#.##%").format((jvmTotalMemoryByte - freeMemoryByte) * 1.0 / jvmTotalMemoryByte)); + //jdk版本 + cpuInfo.put("jdkVersion", props.getProperty("java.version")); + //jdk路径 + cpuInfo.put("jdkHome", props.getProperty("java.home")); + return cpuInfo; + } + + /** + * 系统内存信息 + */ + public static JSONObject getMemInfo() { + JSONObject cpuInfo = new JSONObject(); + GlobalMemory memory = systemInfo.getHardware().getMemory(); + //总内存 + long totalByte = memory.getTotal(); + //剩余 + long acaliableByte = memory.getAvailable(); + //总内存 + cpuInfo.put("total", formatByte(totalByte)); + //使用 + cpuInfo.put("used", formatByte(totalByte - acaliableByte)); + //剩余内存 + cpuInfo.put("free", formatByte(acaliableByte)); + //使用率 + cpuInfo.put("usageRate", new DecimalFormat("#.##%").format((totalByte - acaliableByte) * 1.0 / totalByte)); + return cpuInfo; + } + + /** + * 带宽 + */ + public static JSONArray networkIFs() throws Exception { + JSONObject cpuInfo; + JSONArray sysFiles = new JSONArray(); + + List networkIFs = systemInfo.getHardware().getNetworkIFs(); + for (NetworkIF networkIF : networkIFs) { + String name = networkIF.getName(); + long receiveBytes = networkIF.getBytesRecv(); + long transmitBytes = networkIF.getBytesSent(); + + cpuInfo = new JSONObject(); + //名称 + cpuInfo.put("name", name); + //网络接收 + cpuInfo.put("receiveBytes", formatNumber(receiveBytes / (1024.0 * (2 * 1000 / 1000)))); + //网络发送 + cpuInfo.put("transmitBytes", formatNumber(transmitBytes / (1024.0 * (2 * 1000 / 1000)))); + + sysFiles.add(cpuInfo); + } + return sysFiles; + } + + public static JSONObject getNetWork() { + JSONObject networkInfo = new JSONObject(); + Properties props = System.getProperties(); + String os = props.getProperty("os.name").toLowerCase(); + os = os.startsWith("win") ? "windows" : "linux"; + Map result = new HashMap<>(); + Process pro = null; + Runtime r = Runtime.getRuntime(); + BufferedReader input = null; + try { + String command = "windows".equals(os) ? "netstat -e" : "ifconfig"; + pro = r.exec(command); + input = new BufferedReader(new InputStreamReader(pro.getInputStream())); + long result1[] = readInLine(input, os); + Thread.sleep(SLEEP_TIME); + pro.destroy(); + input.close(); + pro = r.exec(command); + input = new BufferedReader(new InputStreamReader(pro.getInputStream())); + long result2[] = readInLine(input, os); + String rxPercent = formatNumber((result2[0] - result1[0]) / (1024.0 * (SLEEP_TIME / 1000)));// 下行速率(kB/s) + String txPercent = formatNumber((result2[1] - result1[1]) / (1024.0 * (SLEEP_TIME / 1000)));// 上行速率(kB/s) + networkInfo.put("rxPercent", rxPercent); + networkInfo.put("txPercent", txPercent); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (input != null) { + try { + input.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + Optional.ofNullable(pro).ifPresent(p -> p.destroy()); + } + return networkInfo; + } + + private static long[] readInLine(BufferedReader input, String osType) { + long arr[] = new long[2]; + StringTokenizer tokenStat = null; + try { + if (osType.equals("linux")) { // 获取linux环境下的网口上下行速率 + long rx = 0, tx = 0; + String line = null; + //RX packets:4171603 errors:0 dropped:0 overruns:0 frame:0 + //TX packets:4171603 errors:0 dropped:0 overruns:0 carrier:0 + while ((line = input.readLine()) != null) { + if (line.indexOf("RX packets") >= 0) { + rx += Long.parseLong(line.substring(line.indexOf("RX packets") + 11, line.indexOf(" ", line.indexOf("RX packets") + 11))); + } else if (line.indexOf("TX packets") >= 0) { + tx += Long.parseLong(line.substring(line.indexOf("TX packets") + 11, line.indexOf(" ", line.indexOf("TX packets") + 11))); + } + } + arr[0] = rx; + arr[1] = tx; + } else { // 获取windows环境下的网口上下行速率 + input.readLine(); + input.readLine(); + input.readLine(); + input.readLine(); + tokenStat = new StringTokenizer(input.readLine()); + tokenStat.nextToken(); + arr[0] = Long.parseLong(tokenStat.nextToken()); + arr[1] = Long.parseLong(tokenStat.nextToken()); + } + } catch (Exception e) { + e.printStackTrace(); + } + return arr; + } + + + private static String formatNumber(double f) { + return new Formatter().format("%.2f", f).toString(); + } + + /** + * 系统盘符信息 + */ + public static JSONArray getSysFileInfo() { + JSONObject cpuInfo; + JSONArray sysFiles = new JSONArray(); + FileSystem fileSystem = operatingSystem.getFileSystem(); + List fsArray = fileSystem.getFileStores(); + for (OSFileStore fs : fsArray) { + cpuInfo = new JSONObject(); + //盘符路径 + cpuInfo.put("dirName", fs.getMount()); + //盘符类型 + cpuInfo.put("sysTypeName", fs.getType()); + //文件类型 + cpuInfo.put("typeName", fs.getName()); + //总大小 + cpuInfo.put("total", formatByte(fs.getTotalSpace())); + //剩余大小 + cpuInfo.put("free", formatByte(fs.getUsableSpace())); + //已经使用量 + cpuInfo.put("used", formatByte(fs.getTotalSpace() - fs.getUsableSpace())); + if (fs.getTotalSpace() == 0) { + //资源的使用率 + cpuInfo.put("usage", 0); + } else { + cpuInfo.put("usage", new DecimalFormat("#.##%").format((fs.getTotalSpace() - fs.getUsableSpace()) * 1.0 / fs.getTotalSpace())); + } + sysFiles.add(cpuInfo); + } + return sysFiles; + } + + /** + * 系统信息 + */ + public static JSONObject getSysInfo() throws UnknownHostException { + JSONObject cpuInfo = new JSONObject(); + Properties props = System.getProperties(); + //操作系统名 + cpuInfo.put("osName", props.getProperty("os.name")); + //系统架构 + cpuInfo.put("osArch", props.getProperty("os.arch")); + //服务器名称 + cpuInfo.put("computerName", InetAddress.getLocalHost().getHostName()); + //服务器Ip + cpuInfo.put("computerIp", InetAddress.getLocalHost().getHostAddress()); + //项目路径 + cpuInfo.put("userDir", props.getProperty("user.dir")); + return cpuInfo; + } + + + /** + * 所有系统信息 + */ + public static JSONObject getInfo() throws Exception { + JSONObject info = new JSONObject(); + info.put("cpuInfo", getCpuInfo()); + info.put("jvmInfo", getJvmInfo()); + info.put("memInfo", getMemInfo()); + info.put("sysInfo", getSysInfo()); + info.put("sysFileInfo", getSysFileInfo()); + info.put("networkInfo", getNetWork()); + return info; + } + + /** + * 单位转换 + */ + private static String formatByte(long byteNumber) { + //换算单位 + double FORMAT = 1024.0; + double kbNumber = byteNumber / FORMAT; + if (kbNumber < FORMAT) { + return new DecimalFormat("#.##KB").format(kbNumber); + } + double mbNumber = kbNumber / FORMAT; + if (mbNumber < FORMAT) { + return new DecimalFormat("#.##MB").format(mbNumber); + } + double gbNumber = mbNumber / FORMAT; + if (gbNumber < FORMAT) { + return new DecimalFormat("#.##GB").format(gbNumber); + } + double tbNumber = gbNumber / FORMAT; + return new DecimalFormat("#.##TB").format(tbNumber); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/controller/FileController.java b/docus-segmentation/src/main/java/com/docus/server/controller/FileController.java new file mode 100644 index 0000000..2956091 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/controller/FileController.java @@ -0,0 +1,44 @@ +package com.docus.server.controller; + +import com.docus.server.service.IFileUploadService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletResponse; + +/** + * 文件上传下载 API + * + * @author AutoGenerator + * @since 2023-07-15 + */ +@Api(value = "通用文件上传下载接口", tags = "通用文件上传下载接口") +@RestController +@RequestMapping("/file") +public class FileController { + @Resource + private IFileUploadService iFileUploadService; + + @ApiOperation("文件下载") + @GetMapping("/download") + public void downloadFile(String filePath, HttpServletResponse response) throws Exception { + iFileUploadService.downloadFile(filePath, response); + } + + @ApiOperation("文件上传") + @PostMapping("/upload") + @ApiImplicitParams({ + @ApiImplicitParam(name = "files", value = "文件", required = true, dataTypeClass = MultipartFile.class) + }) + public void uploadFile(MultipartFile[] files, String pathKey) throws Exception { + iFileUploadService.uploadFile(files, pathKey); + } +} diff --git a/docus-segmentation/src/main/java/com/docus/server/infrastructure/cache/TaskCacheLayer.java b/docus-segmentation/src/main/java/com/docus/server/infrastructure/cache/TaskCacheLayer.java new file mode 100644 index 0000000..6fea6c6 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/infrastructure/cache/TaskCacheLayer.java @@ -0,0 +1,7 @@ +package com.docus.server.infrastructure.cache; + +import com.docus.server.annotation.CacheLayer; + +@CacheLayer("schetaskCacheLayer") +public class TaskCacheLayer { +} diff --git a/docus-segmentation/src/main/java/com/docus/server/infrastructure/client/DownLoadAPI.java b/docus-segmentation/src/main/java/com/docus/server/infrastructure/client/DownLoadAPI.java new file mode 100644 index 0000000..56f84f1 --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/infrastructure/client/DownLoadAPI.java @@ -0,0 +1,4 @@ +package com.docus.server.infrastructure.client; + +public class DownLoadAPI { +} diff --git a/docus-segmentation/src/main/java/com/docus/server/service/IFileUploadService.java b/docus-segmentation/src/main/java/com/docus/server/service/IFileUploadService.java new file mode 100644 index 0000000..00e46ec --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/service/IFileUploadService.java @@ -0,0 +1,14 @@ +package com.docus.server.service; + +import com.docus.server.vo.scheduling.management.schcollectorversionfile.UploadFileVO; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; + +public interface IFileUploadService { + + List uploadFile(MultipartFile[] multipartFiles, String pathKey) throws Exception; + + void downloadFile(String filePath, HttpServletResponse response); +} diff --git a/docus-segmentation/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java b/docus-segmentation/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java new file mode 100644 index 0000000..647a2bb --- /dev/null +++ b/docus-segmentation/src/main/java/com/docus/server/service/impl/FileUploadServiceImpl.java @@ -0,0 +1,137 @@ +package com.docus.server.service.impl; + +import cn.hutool.core.util.StrUtil; +import com.docus.core.util.Func; +import com.docus.infrastructure.web.exception.ApiException; +import com.docus.infrastructure.web.exception.ExceptionCode; +import com.docus.server.service.IFileUploadService; +import com.docus.server.vo.scheduling.management.schcollectorversionfile.UploadFileVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.net.URLEncoder; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +@Service +@Slf4j +public class FileUploadServiceImpl implements IFileUploadService { + + @Value("${file.uploadFolder:D://docus/}") + private String saveFilePath; + + private static final String COLLECTOR_PACKAGES = "collector_packages"; + + private static DateTimeFormatter ymdDtf = DateTimeFormatter.ofPattern("yyyyMMdd"); + + @Override + public List uploadFile(MultipartFile[] multipartFiles, String pathKey) throws Exception { + + if (Func.isBlank(pathKey)) { + pathKey = COLLECTOR_PACKAGES; + } + + List uploadFileVOList = new ArrayList<>(); + + for (MultipartFile multipartFile : multipartFiles) { + + valid(multipartFile); + + uploadFileVOList.add(actionUploadFile(multipartFile, pathKey)); + } + return uploadFileVOList; + + } + + private UploadFileVO actionUploadFile(MultipartFile multipartFile, String pathKey) throws Exception { + + String todayFormat = ymdDtf.format(LocalDateTime.now()); + + String path = pathKey + "/" + todayFormat + "/" + UUID.randomUUID().toString() + "/"; + File fileDir = new File(saveFilePath + path); + if (!fileDir.exists()) { + fileDir.mkdirs(); + } + String fileName = path + multipartFile.getOriginalFilename(); + + File dest = new File(saveFilePath + fileName); + + multipartFile.transferTo(dest); + + + UploadFileVO uploadFileVO = new UploadFileVO(); + uploadFileVO.setFileName(multipartFile.getOriginalFilename()); + uploadFileVO.setFileSize(multipartFile.getSize()); + uploadFileVO.setFileTitle(multipartFile.getName()); + uploadFileVO.setFileType(multipartFile.getContentType()); + uploadFileVO.setFilePath(fileName); + + return uploadFileVO; + } + + @Override + public void downloadFile(String path, HttpServletResponse response) { + + try { + //获取要下载的文件 + File file = new File(saveFilePath + "/" + path); + + //设置响应的内容类型为二进制流，即文件类型 + response.setContentType("application/octet-stream"); + + //设置文件名 + response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(file.getName(), "UTF-8")); + + //创建输入流 + FileInputStream inputStream = new FileInputStream(file); + BufferedInputStream buffInputStream = new BufferedInputStream(inputStream); + + //创建输出流 + ServletOutputStream outputStream = response.getOutputStream(); + BufferedOutputStream buffOutputStream = new BufferedOutputStream(outputStream); + + //循环读取数据并写入到响应输出流中 + byte[] buffer = new byte[1024]; + int len = -1; + while ((len = buffInputStream.read(buffer)) != -1) { + buffOutputStream.write(buffer, 0, len); + } + + //关闭流 + buffOutputStream.flush(); + buffOutputStream.close(); + outputStream.flush(); + outputStream.close(); + buffInputStream.close(); + inputStream.close(); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + + private void valid(MultipartFile multipartFile) { + if (multipartFile.isEmpty()) { + throw new ApiException(ExceptionCode.ParamIllegal.getCode(), "上传失败，请选择文件！"); + } + + String originalFilename = multipartFile.getOriginalFilename(); + if (StrUtil.isBlank(originalFilename)) { + throw new ApiException(ExceptionCode.ParamIllegal.getCode(), "未找到原文件文件名！"); + } + if (!originalFilename.contains(".")) { + throw new ApiException(ExceptionCode.ParamIllegal.getCode(), "原文件名未识别到后缀！"); + } + } + +} diff --git a/docus-segmentation/src/main/resources/application.properties b/docus-segmentation/src/main/resources/application.properties new file mode 100644 index 0000000..bde7d71 --- /dev/null +++ b/docus-segmentation/src/main/resources/application.properties @@ -0,0 +1,9 @@ +#当前项目的根package +api.base-package=com.docus.server + +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +spring.datasource.url=jdbc:mysql://db.docus.cn:3306/docus-collector-scheduling?autoReconnect=true&allowMultiQueries=true&useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai +spring.datasource.username=docus +spring.datasource.password=docus702 + +mybatis-plus.type-enums-package=com.docus.server.enums diff --git a/docus-segmentation/src/main/resources/bin/auto.bat b/docus-segmentation/src/main/resources/bin/auto.bat new file mode 100644 index 0000000..b07af42 --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/auto.bat @@ -0,0 +1,6 @@ +@echo off + + +WinSW.exe status|findstr NonExistent && winsw install + +WinSW.exe status|findstr stopped && winsw start diff --git a/docus-segmentation/src/main/resources/bin/jenkins-update.bat b/docus-segmentation/src/main/resources/bin/jenkins-update.bat new file mode 100644 index 0000000..19d3f53 --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/jenkins-update.bat @@ -0,0 +1,21 @@ +@echo off + +set deployDir=%1\@project.artifactId@ +if %deployDir%=="" set deployDir=d:\webroot\@project.artifactId@ + +set curr_file=%cd% +cd /d %deployDir% +call stop.bat +sc query @project.artifactId@ |Find "RUNNING" && ping 127.0.0.1 -n 10 >nul +cd %curr_file% +rd/s/q %deployDir%\lib +rd/s/q %deployDir%\dataConfig +rd/s/q %deployDir%\config +rd/s/q %deployDir%\mybatis.mapper +del /s/q %deployDir%\*.jar +xcopy /Y/E/I * %deployDir% + +cd /d %deployDir% +call auto.bat +cd %curr_file% + diff --git a/docus-segmentation/src/main/resources/bin/start.bat b/docus-segmentation/src/main/resources/bin/start.bat new file mode 100644 index 0000000..58c25cb --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/start.bat @@ -0,0 +1,21 @@ +set java_opts=-Xms512m -Xmx512m +set key="java_opts" + + +rem 文件不存在，就跳过 +if not exist java-ops.ini goto end + +for /f "tokens=1,2 delims==" %%i in (java-ops.ini) do ( + if "%%i"==%key% set java_opts=%%j) +echo java_opts is : %java_opts% + +:end + +rem 启动java + +java %java_opts% -Dfile.encoding=utf-8 -jar -Dspring.profiles.active=@profile.name@ -Dloader.path=config,lib @project.build.finalName@.jar + + + + + diff --git a/docus-segmentation/src/main/resources/bin/stop.bat b/docus-segmentation/src/main/resources/bin/stop.bat new file mode 100644 index 0000000..1e224c0 --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/stop.bat @@ -0,0 +1,3 @@ +@echo off + +winsw stop diff --git a/docus-segmentation/src/main/resources/bin/update.bat b/docus-segmentation/src/main/resources/bin/update.bat new file mode 100644 index 0000000..67730ec --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/update.bat @@ -0,0 +1,47 @@ +@echo off + +set curr_file=%cd% + +set parentDir=%1 +if %parentDir%=="" set deployDir=d:\webroot + + +set deployDir=%parentDir%\@project.artifactId@ + +set backupDir=%parentDir%\backup + +set dateStr=%date:~0,4%%date:~5,2%%date:~8,2%%time:~0,2%%time:~3,2%%time:~6,2% + + +cd /d %deployDir% + +call stop.bat +sc query @project.artifactId@ |Find "RUNNING" && ping 127.0.0.1 -n 10 >nul + +cd /d %parentDir% + +xcopy /Y/E/I project.artifactId@\lib %backupDir%\%dateStr%\@project.artifactId@lib +xcopy /Y/E/I project.artifactId@\config %backupDir%\%dateStr%\@project.artifactId@config +xcopy /Y/E/I project.artifactId@\*.jar %backupDir%\%dateStr%\@project.artifactId@ +xcopy /Y/E/I project.artifactId@\*.bat %backupDir%\%dateStr%\@project.artifactId@ +xcopy /Y/E/I project.artifactId@\*.exe %backupDir%\%dateStr%\@project.artifactId@ +xcopy /Y/E/I project.artifactId@\*.xml %backupDir%\%dateStr%\@project.artifactId@ + + +cd %curr_file% + +rd/s/q %deployDir%\lib +rd/s/q %deployDir%\config +del /s/q %deployDir%\*.jar + + +xcopy /Y/E/I lib %deployDir%\lib +xcopy /Y/E/I config %deployDir%\config +xcopy /Y/E/I *.jar %deployDir% +xcopy /Y/E/I *.bat %deployDir% +xcopy /Y/E/I *.exe %deployDir% +xcopy /Y/E/I *.xml %deployDir% + +cd /d %deployDir% +call auto.bat +cd %curr_file% \ No newline at end of file diff --git a/docus-segmentation/src/main/resources/bin/winsw.xml b/docus-segmentation/src/main/resources/bin/winsw.xml new file mode 100644 index 0000000..60dafa2 --- /dev/null +++ b/docus-segmentation/src/main/resources/bin/winsw.xml @@ -0,0 +1,10 @@ + + docus-segmentation + 生产-自动分段(docus-segmentation)-服务 + 生产-自动分段(docus-segmentation)-服务 + Automatic + %BASE%\start.bat + + nacos + seata-server + \ No newline at end of file diff --git a/docus-segmentation/src/main/resources/bootstrap.yml b/docus-segmentation/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..4546456 --- /dev/null +++ b/docus-segmentation/src/main/resources/bootstrap.yml @@ -0,0 +1,60 @@ +server: + port: 9116 +spring: + profiles: + active: dev + application: + name: @artifactId@ + datasource: + dynamic: + primary: master #设置默认的数据源，默认值为master + strict: false #是否弃用严格模式，如果启用在味匹配到指定数据源时抛出异常 + datasource: + #公司病案的文件服务数据库 + master: + url: jdbc:log4jdbc:mysql://db.docus.cn:3306/docus-collector-scheduling?autoReconnect=true&allowMultiQueries=true&useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai + username: docus + password: docus702 + driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + type: com.alibaba.druid.pool.DruidDataSource + servlet: + multipart: + max-file-size: 100MB + max-request-size: 200MB + redis: + host: redis.docus.cn + password: JSdocus@702 + cloud: + nacos: + discovery: + server-addr: nacos.docus.cn + namespace: 34acdf7a-9fc6-4bbd-8aea-9a47c8007ad5 + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: 34acdf7a-9fc6-4bbd-8aea-9a47c8007ad5 + file-extension: yml + shared-configs: + - comm.${spring.cloud.nacos.config.file-extension} + +mybatis-plus: + configuration: + map-underscore-to-camel-case: true + call-setters-on-nulls: true + jdbc-type-for-null: null + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + global-config: + db-config: + update-strategy: ignored + field-strategy: NOT_EMPTY + db-type: MYSQL + mapper-locations: classpath*:/mapper/*Mapper.xml + type-enums-package: com.docus.server.enums + +file: + uploadFolder: D://docus/ + +docus: + vm-task-cron: 0/30 * * * * ? + collector-package-download-url: http://192.168.16.110:9113 + collector-package-download-savePath: D:\\packages\\ + terminal-restart-bat: cmd /c start D:\webroot\collector-scheduling-management\jenkins-update.bat d:\webroot \ No newline at end of file diff --git a/docus-segmentation/src/main/resources/log4jdbc.log4j2.properties b/docus-segmentation/src/main/resources/log4jdbc.log4j2.properties new file mode 100644 index 0000000..5cb6f99 --- /dev/null +++ b/docus-segmentation/src/main/resources/log4jdbc.log4j2.properties @@ -0,0 +1,2 @@ +# If you use SLF4J. First, you need to tell log4jdbc-log4j2 that you want to use the SLF4J logger +log4jdbc.spylogdelegator.name=net.sf.log4jdbc.log.slf4j.Slf4jSpyLogDelegator \ No newline at end of file diff --git a/docus-segmentation/src/main/resources/logback.xml b/docus-segmentation/src/main/resources/logback.xml new file mode 100644 index 0000000..1c6a406 --- /dev/null +++ b/docus-segmentation/src/main/resources/logback.xml @@ -0,0 +1,104 @@ + + + collector-terminal-management[0] + + + + + [%d{yyyy-MM-dd' 'HH:mm:ss.sss}] [%contextName] [%thread] [%X{traceId}] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + [%d{yyyy-MM-dd' 'HH:mm:ss.sss}] [%C] [%t] [%X{traceId}] [%L] [%-5p] %m%n + utf-8 + + + + + ${log.path}%d.%i.log + + 500MB + + 30 + + + + + + [%d{yyyy-MM-dd' 'HH:mm:ss.sss}] [%C] [%t] [%X{traceId}] [%L] [%-5p] %m%n + utf-8 + + + + + ${log.path}external%d.%i.log + + 500MB + + 30 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docus-segmentation/src/test/java/com/docus/server/Example.java b/docus-segmentation/src/test/java/com/docus/server/Example.java new file mode 100644 index 0000000..5a836ed --- /dev/null +++ b/docus-segmentation/src/test/java/com/docus/server/Example.java @@ -0,0 +1,56 @@ +package com.docus.server; + +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.hardware.GlobalMemory; +import oshi.hardware.HardwareAbstractionLayer; +import oshi.hardware.NetworkIF; +import oshi.software.os.FileSystem; +import oshi.software.os.OSFileStore; + +import java.util.Arrays; +import java.util.List; + +public class Example { + public static void main(String[] args) { + SystemInfo systemInfo = new SystemInfo(); + CentralProcessor processor = systemInfo.getHardware().getProcessor(); + int logicalProcessorCount = processor.getLogicalProcessorCount(); + String cpuModel = processor.getProcessorIdentifier().getName(); + long[] systemCpuLoadTicks = processor.getSystemCpuLoadTicks(); + + System.out.println(logicalProcessorCount); + System.out.println(cpuModel); + System.out.println(Arrays.asList(systemCpuLoadTicks)); + + GlobalMemory memory = systemInfo.getHardware().getMemory(); + long totalMemorySize = memory.getTotal(); + long availableMemorySize = memory.getAvailable(); + double memoryUsage = (totalMemorySize - availableMemorySize) * 100.0 / totalMemorySize; + System.out.println(totalMemorySize); + System.out.println(availableMemorySize); + System.out.println(memoryUsage); + + HardwareAbstractionLayer hal = systemInfo.getHardware(); + FileSystem fileSystem = systemInfo.getOperatingSystem().getFileSystem(); + List fileStores = fileSystem.getFileStores(); + for (OSFileStore store : fileStores) { + long totalSpace = store.getTotalSpace(); + long usableSpace = store.getUsableSpace(); + double usage = (totalSpace - usableSpace) * 100.0 / totalSpace; + System.out.println(totalSpace); + System.out.println(usableSpace); + System.out.println(usage); + } + + List networkIFs = systemInfo.getHardware().getNetworkIFs(); + for (NetworkIF networkIF : networkIFs) { + String name = networkIF.getName(); + long receiveBytes = networkIF.getBytesRecv(); + long transmitBytes = networkIF.getBytesSent(); + System.out.println(name); + System.out.println(receiveBytes); + System.out.println(transmitBytes); + } + } +} diff --git a/docus-segmentation/src/test/java/com/docus/server/FileController.java b/docus-segmentation/src/test/java/com/docus/server/FileController.java new file mode 100644 index 0000000..cb39799 --- /dev/null +++ b/docus-segmentation/src/test/java/com/docus/server/FileController.java @@ -0,0 +1,46 @@ +package com.docus.server; + +import cn.hutool.core.util.ZipUtil; +import com.docus.server.common.download.downLoader.HttpDownloader; +import com.docus.server.common.utils.StartUpExeUtils; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * 文件上传下载 API + * + * @author AutoGenerator + * @since 2023-07-15 + */ +@SpringBootTest +public class FileController { + + public static void test() throws Exception { + /** + * 1、接收调度器下载采集器包命令 + * 2、下载保存到指定部署路径 + * 3、启动采集器 + */ + + HttpDownloader httpDownloader = new HttpDownloader(null); + String url = "http://192.168.16.110:9113/sch/file/download?filePath=collector_packages/20230718/91d930e6-0490-44e5-9756-caee3251d645/navicat.zip"; + String fileName = "collector.zip"; + String savePath = "H:\\docus\\1"; //部署路径 + String procName = "navicat.exe"; //进程名称 + + httpDownloader.downLoadFromUrl(url, fileName, savePath); + + ZipUtil.unzip(savePath + "\\" + fileName, savePath + "\\collector"); + + StartUpExeUtils.startUpExe(savePath + "\\collector\\" + procName, procName); + } + + public static void main(String[] args) throws Exception { + test(); + } + + public static void test1() throws Exception { + ZipUtil.unzip("/Users/linruifeng/Desktop/collector_packages/collector.zip", "/Users/linruifeng/Desktop/collector_packages"); +// ZipUtil.unGzip(new GZIPInputStream(new FileInputStream(new File("/Users/linruifeng/Desktop/collector_packages/index.tar.gz")))); +// FileUtils.unTarGz("/Users/linruifeng/Desktop/collector_packages/", "/Users/linruifeng/Desktop/collector_packages/test"); + } +} diff --git a/docus-segmentation/src/test/java/com/docus/server/FileUtils.java b/docus-segmentation/src/test/java/com/docus/server/FileUtils.java new file mode 100644 index 0000000..2010beb --- /dev/null +++ b/docus-segmentation/src/test/java/com/docus/server/FileUtils.java @@ -0,0 +1,162 @@ +package com.docus.server; + +import org.apache.tools.tar.TarInputStream; + +import java.io.*; +import java.util.zip.GZIPInputStream; + +/** + * @program: JavaCode + * @ClassName FileUtils + * @description: + * @author: ltcz99 + * @create: 2023-04-16 + * @Version 1.0 + **/ +public class FileUtils { + + + /** + * 解压tar.gz文件到指定目录 + * + * @param sourceDir 源文件夹 + * @param destDir 解压后的目标文件夹 + */ + public static void unTarGz(String sourceDir, String destDir) throws Exception { + File outFile = new File(sourceDir); + File[] files = outFile.listFiles(); + try { + //创建输出目录 + createDirectory(destDir, null); + TarInputStream tarIn; + int index = 1; + for (File file : files) { + if (file.getName().contains("tar.gz")) { + tarIn = new TarInputStream(new GZIPInputStream( + new BufferedInputStream(new FileInputStream(file))), + 1024 * 2); + + String outFileName = destDir + "/" + file.getName(); + OutputStream out = new FileOutputStream(new File(outFileName)); + int length = 0; + byte[] b = new byte[2048]; + while ((length = tarIn.read(b)) != -1) { + out.write(b, 0, length); + } + out.close(); + tarIn.close(); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /** + * 解压gz到指定的文件夹下面 + * + * @param sourceDir + * @param destDir + */ + public static void unGzipFile(String sourceDir, String destDir) { + //创建输出目录 + createDirectory(destDir, null); + File sourceFile = new File(sourceDir); + File[] files = sourceFile.listFiles(); + try { + int index = 1; + for (File file : files) { + if (file.getName().contains("gz")) { + FileInputStream fin = new FileInputStream(file); + //建立gzip解压工作流 + GZIPInputStream gzin = new GZIPInputStream(fin); + //建立解压文件输出流 + File tmpFile = new File(destDir + "/" + index++ + ".log"); + FileOutputStream fout = new FileOutputStream(tmpFile); + int length; + byte[] buf = new byte[2048]; + while ((length = gzin.read(buf, 0, buf.length)) != -1) { + fout.write(buf, 0, length); + } + gzin.close(); + fout.close(); + fin.close(); + } + } + } catch (Exception ex) { + System.err.println(ex); + } + } + + /** + * 读取文件到指定的文件夹下面 + * + * @param sourceLogPath + * @param destLogPath + */ + public static void readFileToDestLogPath(String sourceLogPath, String destLogPath) { + File sourceFile = new File(sourceLogPath); + File[] files = sourceFile.listFiles(); + for (File file : files) { + String fileName = destLogPath + "/" + file.getName(); + File destFile = new File(fileName); + if (file.getName().contains("log") && !fileName.contains("gz")) { + try { + if (destFile.exists()) { + destFile.delete(); + } + String logFile = sourceFile + "/" + file.getName(); + FileInputStream fis = new FileInputStream(logFile); + FileOutputStream fos = new FileOutputStream(destFile); + BufferedInputStream bis = new BufferedInputStream(fis); + BufferedOutputStream bos = new BufferedOutputStream(fos); + int len = 0; + while ((len = bis.read()) != -1) { + bos.write(len); + } + bos.flush(); + // 关闭资源 + fis.close(); + bis.close(); + fos.close(); + bos.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + } + + /** + * 创建目录 + * + * @param outputDir + * @param subDir + */ + public static void createDirectory(String outputDir, String subDir) { + File file = new File(outputDir); + //子目录不为空 + if (!(subDir == null || subDir.trim().equals(""))) { + file = new File(outputDir + "/" + subDir); + } + if (!file.exists()) { + if (!file.getParentFile().exists()) { + file.getParentFile().mkdirs(); + } + file.mkdirs(); + } + } + + public static void main(String[] args) throws Exception { + String sourceDir = "/Users/ltcz99/Downloads/templog"; + String destDir = "/Users/ltcz99/Downloads/unzip/"; + //解压.gz文件到指定的文件件下面 + unGzipFile(sourceDir, destDir); + // 解压tar.gz文件到指定的文件夹下面 + unTarGz(sourceDir, destDir); + //读取特定的文件到指定的文件夹下面 + readFileToDestLogPath(sourceDir, destDir); + } +} diff --git a/docus-segmentation/src/test/java/com/docus/server/SystemInfoUtils.java b/docus-segmentation/src/test/java/com/docus/server/SystemInfoUtils.java new file mode 100644 index 0000000..a3f2ceb --- /dev/null +++ b/docus-segmentation/src/test/java/com/docus/server/SystemInfoUtils.java @@ -0,0 +1,313 @@ +package com.docus.server; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.hardware.GlobalMemory; +import oshi.hardware.HardwareAbstractionLayer; +import oshi.hardware.NetworkIF; +import oshi.software.os.FileSystem; +import oshi.software.os.OSFileStore; +import oshi.software.os.OperatingSystem; +import oshi.util.Util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.DecimalFormat; +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.StringTokenizer; + +/** + * 系统消息工具类 + **/ +public class SystemInfoUtils { + + private static final int OSHI_WAIT_SECOND = 1000; + private static final int SLEEP_TIME = 2 * 1000; + private static SystemInfo systemInfo = new SystemInfo(); + private static HardwareAbstractionLayer hardware = systemInfo.getHardware(); + private static OperatingSystem operatingSystem = systemInfo.getOperatingSystem(); + + public static void main(String[] args) throws Exception { + JSONObject info = getInfo(); + System.out.println(info); + } + + public static JSONObject getCpuInfo() { + JSONObject cpuInfo = new JSONObject(); + CentralProcessor processor = hardware.getProcessor(); + // CPU信息 + long[] prevTicks = processor.getSystemCpuLoadTicks(); + Util.sleep(OSHI_WAIT_SECOND); + long[] ticks = processor.getSystemCpuLoadTicks(); + long nice = ticks[CentralProcessor.TickType.NICE.getIndex()] - prevTicks[CentralProcessor.TickType.NICE.getIndex()]; + long irq = ticks[CentralProcessor.TickType.IRQ.getIndex()] - prevTicks[CentralProcessor.TickType.IRQ.getIndex()]; + long softirq = ticks[CentralProcessor.TickType.SOFTIRQ.getIndex()] - prevTicks[CentralProcessor.TickType.SOFTIRQ.getIndex()]; + long steal = ticks[CentralProcessor.TickType.STEAL.getIndex()] - prevTicks[CentralProcessor.TickType.STEAL.getIndex()]; + long cSys = ticks[CentralProcessor.TickType.SYSTEM.getIndex()] - prevTicks[CentralProcessor.TickType.SYSTEM.getIndex()]; + long user = ticks[CentralProcessor.TickType.USER.getIndex()] - prevTicks[CentralProcessor.TickType.USER.getIndex()]; + long iowait = ticks[CentralProcessor.TickType.IOWAIT.getIndex()] - prevTicks[CentralProcessor.TickType.IOWAIT.getIndex()]; + long idle = ticks[CentralProcessor.TickType.IDLE.getIndex()] - prevTicks[CentralProcessor.TickType.IDLE.getIndex()]; + long totalCpu = user + nice + cSys + idle + iowait + irq + softirq + steal; + //cpu核数 + cpuInfo.put("cpuNum", processor.getLogicalProcessorCount()); + //cpu系统使用率 + cpuInfo.put("cSys", new DecimalFormat("#.##%").format(cSys * 1.0 / totalCpu)); + //cpu用户使用率 + cpuInfo.put("user", new DecimalFormat("#.##%").format(user * 1.0 / totalCpu)); + //cpu当前等待率 + cpuInfo.put("iowait", new DecimalFormat("#.##%").format(iowait * 1.0 / totalCpu)); + //cpu当前使用率 + cpuInfo.put("idle", new DecimalFormat("#.##%").format(1.0 - (idle * 1.0 / totalCpu))); + return cpuInfo; + } + + /** + * 系统jvm信息 + */ + public static JSONObject getJvmInfo() { + JSONObject cpuInfo = new JSONObject(); + Properties props = System.getProperties(); + Runtime runtime = Runtime.getRuntime(); + long jvmTotalMemoryByte = runtime.totalMemory(); + long freeMemoryByte = runtime.freeMemory(); + //jvm总内存 + cpuInfo.put("total", formatByte(runtime.totalMemory())); + //空闲空间 + cpuInfo.put("free", formatByte(runtime.freeMemory())); + //jvm最大可申请 + cpuInfo.put("max", formatByte(runtime.maxMemory())); + //vm已使用内存 + cpuInfo.put("user", formatByte(jvmTotalMemoryByte - freeMemoryByte)); + //jvm内存使用率 + cpuInfo.put("usageRate", new DecimalFormat("#.##%").format((jvmTotalMemoryByte - freeMemoryByte) * 1.0 / jvmTotalMemoryByte)); + //jdk版本 + cpuInfo.put("jdkVersion", props.getProperty("java.version")); + //jdk路径 + cpuInfo.put("jdkHome", props.getProperty("java.home")); + return cpuInfo; + } + + /** + * 系统内存信息 + */ + public static JSONObject getMemInfo() { + JSONObject cpuInfo = new JSONObject(); + GlobalMemory memory = systemInfo.getHardware().getMemory(); + //总内存 + long totalByte = memory.getTotal(); + //剩余 + long acaliableByte = memory.getAvailable(); + //总内存 + cpuInfo.put("total", formatByte(totalByte)); + //使用 + cpuInfo.put("used", formatByte(totalByte - acaliableByte)); + //剩余内存 + cpuInfo.put("free", formatByte(acaliableByte)); + //使用率 + cpuInfo.put("usageRate", new DecimalFormat("#.##%").format((totalByte - acaliableByte) * 1.0 / totalByte)); + return cpuInfo; + } + + /** + * 带宽 + */ + public static JSONArray networkIFs() throws Exception { + JSONObject cpuInfo; + JSONArray sysFiles = new JSONArray(); + + List networkIFs = systemInfo.getHardware().getNetworkIFs(); + for (NetworkIF networkIF : networkIFs) { + String name = networkIF.getName(); + long receiveBytes = networkIF.getBytesRecv(); + long transmitBytes = networkIF.getBytesSent(); + + cpuInfo = new JSONObject(); + //名称 + cpuInfo.put("name", name); + //网络接收 + cpuInfo.put("receiveBytes", formatNumber(receiveBytes / (1024.0 * (2 * 1000 / 1000)))); + //网络发送 + cpuInfo.put("transmitBytes", formatNumber(transmitBytes / (1024.0 * (2 * 1000 / 1000)))); + + sysFiles.add(cpuInfo); + } + return sysFiles; + } + + public static JSONObject getNetWork() { + JSONObject networkInfo = new JSONObject(); + Properties props = System.getProperties(); + String os = props.getProperty("os.name").toLowerCase(); + os = os.startsWith("win") ? "windows" : "linux"; + Map result = new HashMap<>(); + Process pro = null; + Runtime r = Runtime.getRuntime(); + BufferedReader input = null; + try { + String command = "windows".equals(os) ? "netstat -e" : "ifconfig"; + pro = r.exec(command); + input = new BufferedReader(new InputStreamReader(pro.getInputStream())); + long result1[] = readInLine(input, os); + Thread.sleep(SLEEP_TIME); + pro.destroy(); + input.close(); + pro = r.exec(command); + input = new BufferedReader(new InputStreamReader(pro.getInputStream())); + long result2[] = readInLine(input, os); + String rxPercent = formatNumber((result2[0] - result1[0]) / (1024.0 * (SLEEP_TIME / 1000)));// 下行速率(kB/s) + String txPercent = formatNumber((result2[1] - result1[1]) / (1024.0 * (SLEEP_TIME / 1000)));// 上行速率(kB/s) + networkInfo.put("rxPercent", rxPercent); + networkInfo.put("txPercent", txPercent); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (input != null) { + try { + input.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + Optional.ofNullable(pro).ifPresent(p -> p.destroy()); + } + return networkInfo; + } + + private static long[] readInLine(BufferedReader input, String osType) { + long arr[] = new long[2]; + StringTokenizer tokenStat = null; + try { + if (osType.equals("linux")) { // 获取linux环境下的网口上下行速率 + long rx = 0, tx = 0; + String line = null; + //RX packets:4171603 errors:0 dropped:0 overruns:0 frame:0 + //TX packets:4171603 errors:0 dropped:0 overruns:0 carrier:0 + while ((line = input.readLine()) != null) { + if (line.indexOf("RX packets") >= 0) { + rx += Long.parseLong(line.substring(line.indexOf("RX packets") + 11, line.indexOf(" ", line.indexOf("RX packets") + 11))); + } else if (line.indexOf("TX packets") >= 0) { + tx += Long.parseLong(line.substring(line.indexOf("TX packets") + 11, line.indexOf(" ", line.indexOf("TX packets") + 11))); + } + } + arr[0] = rx; + arr[1] = tx; + } else { // 获取windows环境下的网口上下行速率 + input.readLine(); + input.readLine(); + input.readLine(); + input.readLine(); + tokenStat = new StringTokenizer(input.readLine()); + tokenStat.nextToken(); + arr[0] = Long.parseLong(tokenStat.nextToken()); + arr[1] = Long.parseLong(tokenStat.nextToken()); + } + } catch (Exception e) { + e.printStackTrace(); + } + return arr; + } + + + private static String formatNumber(double f) { + return new Formatter().format("%.2f", f).toString(); + } + + /** + * 系统盘符信息 + */ + public static JSONArray getSysFileInfo() { + JSONObject cpuInfo; + JSONArray sysFiles = new JSONArray(); + FileSystem fileSystem = operatingSystem.getFileSystem(); + List fsArray = fileSystem.getFileStores(); + for (OSFileStore fs : fsArray) { + cpuInfo = new JSONObject(); + //盘符路径 + cpuInfo.put("dirName", fs.getMount()); + //盘符类型 + cpuInfo.put("sysTypeName", fs.getType()); + //文件类型 + cpuInfo.put("typeName", fs.getName()); + //总大小 + cpuInfo.put("total", formatByte(fs.getTotalSpace())); + //剩余大小 + cpuInfo.put("free", formatByte(fs.getUsableSpace())); + //已经使用量 + cpuInfo.put("used", formatByte(fs.getTotalSpace() - fs.getUsableSpace())); + if (fs.getTotalSpace() == 0) { + //资源的使用率 + cpuInfo.put("usage", 0); + } else { + cpuInfo.put("usage", new DecimalFormat("#.##%").format((fs.getTotalSpace() - fs.getUsableSpace()) * 1.0 / fs.getTotalSpace())); + } + sysFiles.add(cpuInfo); + } + return sysFiles; + } + + /** + * 系统信息 + */ + public static JSONObject getSysInfo() throws UnknownHostException { + JSONObject cpuInfo = new JSONObject(); + Properties props = System.getProperties(); + //操作系统名 + cpuInfo.put("osName", props.getProperty("os.name")); + //系统架构 + cpuInfo.put("osArch", props.getProperty("os.arch")); + //服务器名称 + cpuInfo.put("computerName", InetAddress.getLocalHost().getHostName()); + //服务器Ip + cpuInfo.put("computerIp", InetAddress.getLocalHost().getHostAddress()); + //项目路径 + cpuInfo.put("userDir", props.getProperty("user.dir")); + return cpuInfo; + } + + + /** + * 所有系统信息 + */ + public static JSONObject getInfo() throws Exception { + JSONObject info = new JSONObject(); + info.put("cpuInfo", getCpuInfo()); + info.put("jvmInfo", getJvmInfo()); + info.put("memInfo", getMemInfo()); + info.put("sysInfo", getSysInfo()); + info.put("sysFileInfo", getSysFileInfo()); + info.put("networkInfo", getNetWork()); + return info; + } + + /** + * 单位转换 + */ + private static String formatByte(long byteNumber) { + //换算单位 + double FORMAT = 1024.0; + double kbNumber = byteNumber / FORMAT; + if (kbNumber < FORMAT) { + return new DecimalFormat("#.##KB").format(kbNumber); + } + double mbNumber = kbNumber / FORMAT; + if (mbNumber < FORMAT) { + return new DecimalFormat("#.##MB").format(mbNumber); + } + double gbNumber = mbNumber / FORMAT; + if (gbNumber < FORMAT) { + return new DecimalFormat("#.##GB").format(gbNumber); + } + double tbNumber = gbNumber / FORMAT; + return new DecimalFormat("#.##TB").format(tbNumber); + } +} diff --git a/docus-segmentation/src/test/java/com/docus/server/TsmsAuthApiApplicationTests.java b/docus-segmentation/src/test/java/com/docus/server/TsmsAuthApiApplicationTests.java new file mode 100644 index 0000000..18fefea --- /dev/null +++ b/docus-segmentation/src/test/java/com/docus/server/TsmsAuthApiApplicationTests.java @@ -0,0 +1,79 @@ +package com.docus.server; + +import cn.smallbun.screw.core.Configuration; +import cn.smallbun.screw.core.engine.EngineConfig; +import cn.smallbun.screw.core.engine.EngineFileType; +import cn.smallbun.screw.core.engine.EngineTemplateType; +import cn.smallbun.screw.core.execute.DocumentationExecute; +import cn.smallbun.screw.core.process.ProcessConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationContext; + +import javax.sql.DataSource; +import java.util.ArrayList; + +@SpringBootTest +class ScrewTests { + + @Autowired + private ApplicationContext applicationContext; + + @Test + void test() { + + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setDriverClassName("com.mysql.cj.jdbc.Driver"); + hikariConfig.setJdbcUrl("jdbc:mysql://db.docus.cn:3306/docus-collector-scheduling?autoReconnect=true&allowMultiQueries=true&useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai"); + hikariConfig.setUsername("docus"); + hikariConfig.setPassword("docus702"); + //设置可以获取tables remarks信息 + hikariConfig.addDataSourceProperty("useInformationSchema", "true"); + hikariConfig.setMinimumIdle(2); + hikariConfig.setMaximumPoolSize(5); + DataSource dataSource = new HikariDataSource(hikariConfig); + + //生成文件配置 + EngineConfig engineConfig = EngineConfig + .builder() + //文件生成路径 + .fileOutputDir("h:\\file") + //打开目录 + .openOutputDir(false) + //文件类型 + .fileType(EngineFileType.HTML) + //生成模板实现 + .produceType(EngineTemplateType.freemarker) + .build(); + //配置想要生成的表 + ProcessConfig processConfig = ProcessConfig + .builder() + //根据名称指定表生成 + .designatedTableName(new ArrayList<>()) + //根据表前缀生成 + .designatedTablePrefix(new ArrayList<>()) + //根据表后缀生成 + .designatedTableSuffix(new ArrayList<>()) + //忽略表名 + .ignoreTableName(new ArrayList<>()) + //忽略表前缀 + .ignoreTablePrefix(new ArrayList<>()) + //忽略表后缀 + .ignoreTableSuffix(new ArrayList<>()) + .build(); + //生成文档配置 + Configuration configuration = Configuration.builder() + .version("1.0.0") + .description("描述") + .dataSource(dataSource) + .engineConfig(engineConfig) + .produceConfig(processConfig) + .build(); + + //生成 + new DocumentationExecute(configuration).execute(); + } +} diff --git a/pom.xml b/pom.xml index c30ebc2..72b3c7f 100644 --- a/pom.xml +++ b/pom.xml @@ -12,12 +12,7 @@ docus-collector-server pom - collect-sdry - collector-scheduling-management - collector-terminal-management - common-collect - common-docus - common-sysem + docus-segmentation docus-api-common docus-client-interface