diff --git a/src/main/java/com/point/strategy/common/MemoryMonitor.java b/src/main/java/com/point/strategy/common/MemoryMonitor.java new file mode 100644 index 0000000..5369893 --- /dev/null +++ b/src/main/java/com/point/strategy/common/MemoryMonitor.java @@ -0,0 +1,92 @@ +package com.point.strategy.common; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * 内存监控工具类 + * 用于监控JVM内存使用情况，防止内存溢出 + */ +@Slf4j +@Component +public class MemoryMonitor { + + private static final long MEMORY_THRESHOLD = 500 * 1024 * 1024; // 500MB阈值 + private static final AtomicLong startTime = new AtomicLong(); + private static final String OPERATION_NAME = "hookUpTwo"; + + /** + * 开始内存监控 + * @param operation 操作名称 + */ + public static void startMonitoring(String operation) { + startTime.set(System.currentTimeMillis()); + Runtime runtime = Runtime.getRuntime(); + long initialMemory = (runtime.totalMemory() - runtime.freeMemory()) / 1024 / 1024; + log.info("开始 {} - 初始内存使用: {}MB, 最大内存: {}MB", + operation, initialMemory, runtime.maxMemory() / 1024 / 1024); + } + + /** + * 检查内存使用情况 + */ + public static void checkMemoryUsage() { + Runtime runtime = Runtime.getRuntime(); + long usedMemory = runtime.totalMemory() - runtime.freeMemory(); + long maxMemory = runtime.maxMemory(); + long freeMemory = runtime.freeMemory(); + long totalMemory = runtime.totalMemory(); + + if (usedMemory > MEMORY_THRESHOLD) { + log.warn("内存使用超过阈值: {}MB / {}MB (总内存: {}MB, 空闲: {}MB)", + usedMemory / 1024 / 1024, maxMemory / 1024 / 1024, + totalMemory / 1024 / 1024, freeMemory / 1024 / 1024); + forceGC(); + } + } + + /** + * 强制垃圾回收 + */ + public static void forceGC() { + log.debug("执行强制垃圾回收..."); + System.gc(); + System.runFinalization(); + try { + Thread.sleep(100); // 等待GC完成 + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * 停止监控并输出结果 + */ + public static void stopMonitoring() { + long duration = System.currentTimeMillis() - startTime.get(); + Runtime runtime = Runtime.getRuntime(); + long usedMemory = runtime.totalMemory() - runtime.freeMemory(); + long maxMemory = runtime.maxMemory(); + + log.info("{} 操作完成 - 耗时: {}ms, 最终内存使用: {}MB / {}MB", + OPERATION_NAME, duration, usedMemory / 1024 / 1024, maxMemory / 1024 / 1024); + } + + /** + * 获取当前内存使用情况 + * @return 内存使用信息 + */ + public static String getMemoryInfo() { + Runtime runtime = Runtime.getRuntime(); + long usedMemory = runtime.totalMemory() - runtime.freeMemory(); + long maxMemory = runtime.maxMemory(); + long freeMemory = runtime.freeMemory(); + long totalMemory = runtime.totalMemory(); + + return String.format("内存使用: %dMB/%dMB (总内存: %dMB, 空闲: %dMB)", + usedMemory / 1024 / 1024, maxMemory / 1024 / 1024, + totalMemory / 1024 / 1024, freeMemory / 1024 / 1024); + } +} \ No newline at end of file diff --git a/src/main/java/com/point/strategy/datas/service/ImportService.java b/src/main/java/com/point/strategy/datas/service/ImportService.java index 68342f7..d7049f8 100644 --- a/src/main/java/com/point/strategy/datas/service/ImportService.java +++ b/src/main/java/com/point/strategy/datas/service/ImportService.java @@ -26,8 +26,11 @@ import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletRequest; import java.io.*; +import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.nio.file.*; import java.util.*; +import java.util.concurrent.Semaphore; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -114,6 +117,9 @@ public class ImportService { //创建线程池工具类 ThreadPoolUtils threadPoolUtils = new ThreadPoolUtils(20); + + //文件处理并发控制信号量，限制同时处理的文件数量 + private Semaphore fileProcessSemaphore = new Semaphore(10); /** * 根据操作系统类型获取网络上传文件路径配置 @@ -1958,132 +1964,156 @@ public class ImportService { //获取表名 TtableDescription ttableDescription = ttableDescriptionMapper.selectTtableDescription(entityId); String tableName = ttableDescription.getTableName(); - //文件解析 - File[] files = new File(localPath).listFiles(); - if(null == files || files.length < 1){ + + //使用NIO Path替代File + Path localPathObj = Paths.get(localPath); + if (!Files.exists(localPathObj) || !Files.isDirectory(localPathObj)) { System.out.println("文件路径错误或不存在"); return json = AjaxJson.returnExceptionInfo("文件路径错误或不存在"); } - for (File file : files) { - if(file.isDirectory()){ - //获取folder_no案卷级档号 - String folderNo = file.getName(); - //获取件号 - File[] files1 = file.listFiles(); - Arrays.sort(files1); - for (File file1 : files1) { - int i = 1; - String pieceNo = file1.getName(); - String archiveNo = folderNo + "-" + pieceNo; - String saveUrl = combinePath(uploadPath, "uploadFile") + File.separator ; - String replace=archiveNo; -// if(archiveNo.contains("·")){ -// replace = archiveNo.replace("·", "."); -// } - String dir1 = saveUrl +"/"+ replace + '/'; - String dir = "uploadFile" +"/"+ replace + '/'; - //获取id - HashMap mapThree = new HashMap(); - StringBuffer conditionSql = new StringBuffer(); -// conditionSql.append("piece_no"); -// conditionSql.append("="); -// conditionSql.append("'"+pieceNo+"'"); -// conditionSql.append("and folder_no"); -// conditionSql.append("="); -// conditionSql.append("'"+folderNo+"'"); - conditionSql.append("archive_no"); - conditionSql.append("="); - conditionSql.append("'"+archiveNo+"'"); - mapThree.put("tableName", tableName+ "_temp"); - mapThree.put("conditionSql", conditionSql.toString()); - List list = docSimpleMapper.selectObject(mapThree); - if(CollectionUtils.isEmpty(list)){ - return json = AjaxJson.returnExceptionInfo("系统里面没有该档号"+archiveNo); - } - Map map = (Map) list.get(0); - int id = (Integer) map.get("id"); - //获取图片 - File[] files2 = file1.listFiles(); - if(1==type){ - //覆盖 先删除wsjh_20201103104220949_temp_file表的记录 - HashMap mapTwo = new HashMap(); - StringBuffer conditionSqlTwo = new StringBuffer(); - conditionSqlTwo.append("rec_id"+"="+id); - mapTwo.put("tableName", tableName+ "_temp_file"); - mapTwo.put("conditionSql", conditionSqlTwo.toString()); - docSimpleMapper.deleteObject(mapTwo); - //开始挂接 - for (File file3 : files2) { - //上传文件 - String imageName = ""; - if("是".equals(AbSurface)){ - imageName = uploadFileAB(file1, dir1); - }else { - //上传文件 - imageName = uploadFile(file3, dir1, replace, i); - } - //插入到表中 - AjaxJson json1 = insertFile(file3, id, dir, tableName+ "_temp_file", imageName,i); - if ("101".equals(json1.getCode())) { - falseNum++; - } - if ("100".equals(json1.getCode())) { - successNum++; - } - i++; - String[] strings = file3.getName().split("\\."); - String name = file3.getName(); - String fileType = strings[strings.length - 1]; - if(!fileType.equals("mp3") || !fileType.equals("mp4")){ - getText(file3,imageName,tableName); - } + + try (DirectoryStream dirStream = Files.newDirectoryStream(localPathObj)) { + for (Path path : dirStream) { + if (Files.isDirectory(path)) { + //获取folder_no案卷级档号 + String folderNo = path.getFileName().toString(); + + //获取件号 + Path pathObj = path; + try (DirectoryStream dirStream1 = Files.newDirectoryStream(pathObj)) { + List fileList = new ArrayList<>(); + for (Path path1 : dirStream1) { + fileList.add(path1); } - }else{ - //追加 - for (File file3 : files2) { - Map map5=new HashMap<>(); - map5.put("tableName",tableName + "_temp_file"); - map5.put("conditionSql","rec_id= '"+id+"'"); - int page = danganguanliService.selectObjectCount(map5) + 1; - //上传文件 - String imageName = ""; - if("是".equals(AbSurface)){ - imageName = uploadFileAB(file1, dir1); - }else { - //上传文件 - imageName = uploadFile(file3, dir1, replace, page); + //排序 + fileList.sort(Comparator.comparing(Path::getFileName)); + + for (Path path1 : fileList) { + int i = 1; + String pieceNo = path1.getFileName().toString(); + String archiveNo = folderNo + "-" + pieceNo; + String saveUrl = combinePath(uploadPath, "uploadFile") + File.separator; + String replace = archiveNo; + String dir1 = saveUrl + "/" + replace + '/'; + String dir = "uploadFile" + "/" + replace + '/'; + + //获取id + HashMap mapThree = new HashMap(); + StringBuffer conditionSql = new StringBuffer(); + conditionSql.append("archive_no"); + conditionSql.append("="); + conditionSql.append("'" + archiveNo + "'"); + mapThree.put("tableName", tableName + "_temp"); + mapThree.put("conditionSql", conditionSql.toString()); + List list = docSimpleMapper.selectObject(mapThree); + if (CollectionUtils.isEmpty(list)) { + return json = AjaxJson.returnExceptionInfo("系统里面没有该档号" + archiveNo); } - //插入到表中 - AjaxJson json1 = insertFile(file3, id, dir, tableName+ "_temp_file", imageName,page); - if ("101".equals(json1.getCode())) { - falseNum++; - } - if ("100".equals(json1.getCode())) { - successNum++; - } - String[] strings = file3.getName().split("\\."); - String name = file3.getName(); - String fileType = strings[strings.length - 1]; - if(!fileType.equals("mp3") || !fileType.equals("mp4")){ - getText(file3,imageName,tableName); + Map map = (Map) list.get(0); + int id = (Integer) map.get("id"); + + //获取图片 + try (DirectoryStream dirStream2 = Files.newDirectoryStream(path1)) { + List fileList2 = new ArrayList<>(); + for (Path path2 : dirStream2) { + fileList2.add(path2); + } + + if (1 == type) { + //覆盖 先删除wsjh_20201103104220949_temp_file表的记录 + HashMap mapTwo = new HashMap(); + StringBuffer conditionSqlTwo = new StringBuffer(); + conditionSqlTwo.append("rec_id" + "=" + id); + mapTwo.put("tableName", tableName + "_temp_file"); + mapTwo.put("conditionSql", conditionSqlTwo.toString()); + docSimpleMapper.deleteObject(mapTwo); + + //开始挂接 + for (Path path2 : fileList2) { + File file3 = path2.toFile(); + //上传文件 + String imageName = ""; + if ("是".equals(AbSurface)) { + imageName = uploadFileAB(path1.toFile(), dir1); + } else { + //上传文件 + imageName = uploadFile(file3, dir1, replace, i); + } + //插入到表中 + AjaxJson json1 = insertFile(file3, id, dir, tableName + "_temp_file", imageName, i); + if ("101".equals(json1.getCode())) { + falseNum++; + } + if ("100".equals(json1.getCode())) { + successNum++; + } + i++; + String fileName = file3.getName(); + String fileType = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(); + if (!fileType.equals("mp3") && !fileType.equals("mp4")) { + getText(file3, imageName, tableName); + } + + //定期触发垃圾回收，释放内存 + if (i % 50 == 0) { + System.gc(); + } + } + } else { + //追加 + for (Path path2 : fileList2) { + File file3 = path2.toFile(); + Map map5 = new HashMap<>(); + map5.put("tableName", tableName + "_temp_file"); + map5.put("conditionSql", "rec_id= '" + id + "'"); + int page = danganguanliService.selectObjectCount(map5) + 1; + //上传文件 + String imageName = ""; + if ("是".equals(AbSurface)) { + imageName = uploadFileAB(path1.toFile(), dir1); + } else { + //上传文件 + imageName = uploadFile(file3, dir1, replace, page); + } + //插入到表中 + AjaxJson json1 = insertFile(file3, id, dir, tableName + "_temp_file", imageName, page); + if ("101".equals(json1.getCode())) { + falseNum++; + } + if ("100".equals(json1.getCode())) { + successNum++; + } + String fileName = file3.getName(); + String fileType = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(); + if (!fileType.equals("mp3") && !fileType.equals("mp4")) { + getText(file3, imageName, tableName); + } + + //定期触发垃圾回收，释放内存 + if (page % 50 == 0) { + System.gc(); + } + } + } } + + //更新原文数量 + Map map7 = new HashMap<>(); + map7.put("tableName", tableName + "_temp"); + map7.put("tableName2", tableName + "_temp_file"); + map7.put("id", id); + danganguanliService.wsajmlTempCount(map7); } - } - //更新原文数量 - Map map7=new HashMap<>(); - map7.put("tableName",tableName+"_temp"); - map7.put("tableName2",tableName+"_temp_file"); - map7.put("id",id); - danganguanliService.wsajmlTempCount(map7); } - } + } catch (IOException e) { + throw new RuntimeException("处理文件时发生错误", e); } - json = AjaxJson.returnInfo("成功上传数"+successNum+","+"失败上传数"+falseNum); - json.put("successNum",successNum); - json.put("falseNum",falseNum); + json = AjaxJson.returnInfo("成功上传数" + successNum + "," + "失败上传数" + falseNum); + json.put("successNum", successNum); + json.put("falseNum", falseNum); return json; } @@ -2104,126 +2134,160 @@ public class ImportService { //获取表名 TtableDescription ttableDescription = ttableDescriptionMapper.selectTtableDescription(entityId); String tableName = ttableDescription.getTableName(); - //文件解析 - File[] files = new File(localPath).listFiles(); - if(null == files || files.length < 1){ + + //使用NIO Path替代File + Path localPathObj = Paths.get(localPath); + if (!Files.exists(localPathObj) || !Files.isDirectory(localPathObj)) { System.out.println("文件路径错误或不存在"); return json = AjaxJson.returnExceptionInfo("文件路径错误或不存在"); } - for (File file : files) { - if(file.isDirectory()){ - //获取folder_no案卷级档号 - String folderNo = file.getName(); - //获取件号 - File[] files1 = file.listFiles(); - Arrays.sort(files1); - for (File file1 : files1) { - int i = 1; - String pieceNo = file1.getName(); - String archiveNo = folderNo + "-" + pieceNo; - String saveUrl = combinePath(uploadPath, "uploadFile") + File.separator ; - String replace=archiveNo; - String dir1 = saveUrl +"/"+ replace + '/'; - String dir = "uploadFile" +"/"+ replace + '/'; - //获取id - HashMap mapThree = new HashMap(); - StringBuffer conditionSql = new StringBuffer(); - conditionSql.append("archive_no"); - conditionSql.append("="); - conditionSql.append("'"+archiveNo+"'"); - mapThree.put("tableName", tableName+ "_temp"); - mapThree.put("conditionSql", conditionSql.toString()); - List list = docSimpleMapper.selectObject(mapThree); - if(CollectionUtils.isEmpty(list)){ - return json = AjaxJson.returnExceptionInfo("系统里面没有该档号"+archiveNo); - } - Map map = (Map) list.get(0); - int id = (Integer) map.get("id"); - //获取图片 - File[] files2 = file1.listFiles(); - if(1==type){ - //覆盖 先删除wsjh_20201103104220949_temp_file表的记录 - HashMap mapTwo = new HashMap(); - StringBuffer conditionSqlTwo = new StringBuffer(); - conditionSqlTwo.append("rec_id"+"="+id); - mapTwo.put("tableName", tableName+ "_temp_file"); - mapTwo.put("conditionSql", conditionSqlTwo.toString()); - docSimpleMapper.deleteObject(mapTwo); - //开始挂接 - for (File file3 : files2) { - //上传文件 - String imageName = ""; - if("是".equals(AbSurface)){ - imageName = uploadFileAB(file1, dir1); - //排序取文件名中的数字 - i = extractSortValue(imageName)==0?i:extractSortValue(imageName); - }else { - //上传文件 - imageName = uploadFile(file3, dir1, replace, i); - } - //插入到表中 - AjaxJson json1 = insertFile(file3, id, dir, tableName+ "_temp_file", imageName,i); - if ("101".equals(json1.getCode())) { - falseNum++; - } - if ("100".equals(json1.getCode())) { - successNum++; - } - i++; - String[] strings = file3.getName().split("\\."); - String name = file3.getName(); - String fileType = strings[strings.length - 1]; - if(!fileType.equals("mp3") || !fileType.equals("mp4")){ - getText(file3,imageName,tableName); - } + + try (DirectoryStream dirStream = Files.newDirectoryStream(localPathObj)) { + for (Path path : dirStream) { + if (Files.isDirectory(path)) { + //获取folder_no案卷级档号 + String folderNo = path.getFileName().toString(); + + //获取件号 + Path pathObj = path; + try (DirectoryStream dirStream1 = Files.newDirectoryStream(pathObj)) { + List fileList = new ArrayList<>(); + for (Path path1 : dirStream1) { + fileList.add(path1); } - }else{ - //追加 - for (File file3 : files2) { - Map map5=new HashMap<>(); - map5.put("tableName",tableName + "_temp_file"); - map5.put("conditionSql","rec_id= '"+id+"'"); - int page = danganguanliService.selectObjectCount(map5) + 1; - //上传文件 - String imageName = ""; - if("是".equals(AbSurface)){ - imageName = uploadFileAB(file1, dir1); - //排序取文件名中的数字 - page = extractSortValue(imageName)==0?page:extractSortValue(imageName); - }else { - //上传文件 - imageName = uploadFile(file3, dir1, replace, page); + //排序 + fileList.sort(Comparator.comparing(Path::getFileName)); + + for (Path path1 : fileList) { + int i = 1; + String pieceNo = path1.getFileName().toString(); + String archiveNo = folderNo + "-" + pieceNo; + String saveUrl = combinePath(uploadPath, "uploadFile") + File.separator; + String replace = archiveNo; + String dir1 = saveUrl + "/" + replace + '/'; + String dir = "uploadFile" + "/" + replace + '/'; + + //获取id + HashMap mapThree = new HashMap(); + StringBuffer conditionSql = new StringBuffer(); + conditionSql.append("archive_no"); + conditionSql.append("="); + conditionSql.append("'" + archiveNo + "'"); + mapThree.put("tableName", tableName + "_temp"); + mapThree.put("conditionSql", conditionSql.toString()); + List list = docSimpleMapper.selectObject(mapThree); + if (CollectionUtils.isEmpty(list)) { + return json = AjaxJson.returnExceptionInfo("系统里面没有该档号" + archiveNo); } - //插入到表中 - AjaxJson json1 = insertFile(file3, id, dir, tableName+ "_temp_file", imageName,page); - if ("101".equals(json1.getCode())) { - falseNum++; - } - if ("100".equals(json1.getCode())) { - successNum++; - } - String[] strings = file3.getName().split("\\."); - String name = file3.getName(); - String fileType = strings[strings.length - 1]; - if(!fileType.equals("mp3") || !fileType.equals("mp4")){ - getText(file3,imageName,tableName); + Map map = (Map) list.get(0); + int id = (Integer) map.get("id"); + + //获取图片 + try (DirectoryStream dirStream2 = Files.newDirectoryStream(path1)) { + List fileList2 = new ArrayList<>(); + for (Path path2 : dirStream2) { + fileList2.add(path2); + } + + if (1 == type) { + //覆盖 先删除wsjh_20201103104220949_temp_file表的记录 + HashMap mapTwo = new HashMap(); + StringBuffer conditionSqlTwo = new StringBuffer(); + conditionSqlTwo.append("rec_id" + "=" + id); + mapTwo.put("tableName", tableName + "_temp_file"); + mapTwo.put("conditionSql", conditionSqlTwo.toString()); + docSimpleMapper.deleteObject(mapTwo); + + //开始挂接 + for (Path path2 : fileList2) { + File file3 = path2.toFile(); + //上传文件 + String imageName = ""; + if ("是".equals(AbSurface)) { + imageName = uploadFileAB(path1.toFile(), dir1); + //排序取文件名中的数字 + i = extractSortValue(imageName) == 0 ? i : extractSortValue(imageName); + } else { + //上传文件 + imageName = uploadFile(file3, dir1, replace, i); + } + //插入到表中 + AjaxJson json1 = insertFile(file3, id, dir, tableName + "_temp_file", imageName, i); + if ("101".equals(json1.getCode())) { + falseNum++; + } + if ("100".equals(json1.getCode())) { + successNum++; + } + i++; + String fileName = file3.getName(); + String fileType = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(); + if (!fileType.equals("mp3") && !fileType.equals("mp4")) { + getText(file3, imageName, tableName); + } + + //定期触发垃圾回收，释放内存 + if (i % 50 == 0) { + System.gc(); + } + } + } else { + //追加 + for (Path path2 : fileList2) { + File file3 = path2.toFile(); + Map map5 = new HashMap<>(); + map5.put("tableName", tableName + "_temp_file"); + map5.put("conditionSql", "rec_id= '" + id + "'"); + int page = danganguanliService.selectObjectCount(map5) + 1; + //上传文件 + String imageName = ""; + if ("是".equals(AbSurface)) { + imageName = uploadFileAB(path1.toFile(), dir1); + //排序取文件名中的数字 + page = extractSortValue(imageName) == 0 ? page : extractSortValue(imageName); + } else { + //上传文件 + imageName = uploadFile(file3, dir1, replace, page); + } + //插入到表中 + AjaxJson json1 = insertFile(file3, id, dir, tableName + "_temp_file", imageName, page); + if ("101".equals(json1.getCode())) { + falseNum++; + } + if ("100".equals(json1.getCode())) { + successNum++; + } + String fileName = file3.getName(); + String fileType = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(); + if (!fileType.equals("mp3") && !fileType.equals("mp4")) { + getText(file3, imageName, tableName); + } + + //定期触发垃圾回收，释放内存 + if (page % 50 == 0) { + System.gc(); + } + } + } } + + //更新原文数量 + Map map7 = new HashMap<>(); + map7.put("tableName", tableName + "_temp"); + map7.put("tableName2", tableName + "_temp_file"); + map7.put("id", id); + danganguanliService.wsajmlTempCount(map7); } - } - //更新原文数量 - Map map7=new HashMap<>(); - map7.put("tableName",tableName+"_temp"); - map7.put("tableName2",tableName+"_temp_file"); - map7.put("id",id); - danganguanliService.wsajmlTempCount(map7); } - } + } catch (IOException e) { + throw new RuntimeException("处理文件时发生错误", e); } - json = AjaxJson.returnInfo("成功上传数"+successNum+","+"失败上传数"+falseNum); - json.put("successNum",successNum); - json.put("falseNum",falseNum); + + json = AjaxJson.returnInfo("成功上传数" + successNum + "," + "失败上传数" + falseNum); + json.put("successNum", successNum); + json.put("falseNum", falseNum); return json; } @@ -3858,8 +3922,6 @@ public class ImportService { //上传文件 public String uploadFile(File file, String dir,String archiveNo,int page) throws Exception { //上传文件到服务器 - //输入流 - InputStream is =new FileInputStream(file); String[] strings = file.getName().split("\\."); String type = strings[strings.length - 1].toLowerCase(); String name = ""; @@ -3873,21 +3935,30 @@ public class ImportService { } //改图片名拼接 String imgName = archiveNo + "." + name; - File fileOne = new File(dir); - if (!fileOne.exists()) { - fileOne.mkdirs(); + Path dirPath = Paths.get(dir); + if (!Files.exists(dirPath)) { + Files.createDirectories(dirPath); } - //获取输出流 - OutputStream os= new FileOutputStream(dir+imgName); - byte [] bts = new byte [ 1024 ]; - //一个一个字节的读取并写入 - while (is.read(bts)!=- 1 ) - { - os.write(bts); + + //使用NIO进行文件复制，更高效且内存友好 + Path sourcePath = file.toPath(); + Path targetPath = dirPath.resolve(imgName); + + //使用try-with-resources确保资源释放 + try (FileChannel sourceChannel = FileChannel.open(sourcePath, StandardOpenOption.READ); + FileChannel targetChannel = FileChannel.open(targetPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + + //使用transferTo进行高效文件传输 + long size = sourceChannel.size(); + long transferred = sourceChannel.transferTo(0, size, targetChannel); + + //确保所有数据都已传输 + while (transferred < size) { + transferred += sourceChannel.transferTo(transferred, size - transferred, targetChannel); + } } - os.flush(); - os.close(); - is.close(); + String file_name_server = imgName; if(!type.equals("mp3") && !type.equals("mp4")){ //生成一份pdf文件，用于归档章的操作 @@ -3914,26 +3985,21 @@ public class ImportService { //上传文件AB面 public String uploadFileAB(File file, String dir) throws IOException { //上传文件到服务器 - //输入流 - InputStream is =new FileInputStream(file); String[] strings = file.getName().split("\\."); String type = strings[strings.length - 1].toLowerCase(); String imgName = file.getName(); - File fileOne = new File(dir); - if (!fileOne.exists()) { - fileOne.mkdirs(); + Path dirPath = Paths.get(dir); + if (!Files.exists(dirPath)) { + Files.createDirectories(dirPath); } - //获取输出流 - OutputStream os= new FileOutputStream(dir+imgName); - byte [] bts = new byte [ 1024 ]; - //一个一个字节的读取并写入 - while (is.read(bts)!=- 1 ) - { - os.write(bts); - } - os.flush(); - os.close(); - is.close(); + + //使用NIO进行文件复制，更高效且内存友好 + Path sourcePath = file.toPath(); + Path targetPath = dirPath.resolve(imgName); + + //使用Files.copy进行高效文件复制 + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + String file_name_server = imgName; if(!type.equals("mp3") && !type.equals("mp4")){ //生成一份pdf文件，用于归档章的操作 @@ -4039,48 +4105,61 @@ public class ImportService { //todo 添加ocr日志 //启动一个线程，根据ocr获取图片文字"file_content,"+ threadPoolUtils.submitTask(()->{ - OcrLog ocrLog = new OcrLog(); - if(youhongIntegrate){ - try { - String ocrImageWord = OCRUtil.yhOcr(file,youhongBaseUrl); - Map map8=new HashMap<>(); - String fieldValue = "file_content" + "=" + "'"+ocrImageWord+"'"; - String conditionSql = "file_name_server" + "=" + "'"+fileNameServer+"'"; - map8.put("tableName",tableName+"_temp_file"); - map8.put("fieldValue",fieldValue); - map8.put("conditionSql",conditionSql); - danganguanliService.updateObject(map8); - ocrLog.setFileName(file.getName()); - ocrLog.setTableName(tableName+"_temp_file"); - ocrLog.setStatusType("0"); - } catch (Exception e) { - ocrLog.setFailureReason(e.getMessage()); - ocrLog.setStatusType("1"); - ocrLogMapper.insert(ocrLog); - throw new RuntimeException("ocr:"+e); + //获取信号量许可，限制并发OCR处理数量 + try { + fileProcessSemaphore.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + + try { + OcrLog ocrLog = new OcrLog(); + if(youhongIntegrate){ + try { + String ocrImageWord = OCRUtil.yhOcr(file,youhongBaseUrl); + Map map8=new HashMap<>(); + String fieldValue = "file_content" + "=" + "'"+ocrImageWord+"'"; + String conditionSql = "file_name_server" + "=" + "'"+fileNameServer+"'"; + map8.put("tableName",tableName+"_temp_file"); + map8.put("fieldValue",fieldValue); + map8.put("conditionSql",conditionSql); + danganguanliService.updateObject(map8); + ocrLog.setFileName(file.getName()); + ocrLog.setTableName(tableName+"_temp_file"); + ocrLog.setStatusType("0"); + } catch (Exception e) { + ocrLog.setFailureReason(e.getMessage()); + ocrLog.setStatusType("1"); + ocrLogMapper.insert(ocrLog); + throw new RuntimeException("ocr:"+e); + } + }else{ + OCRUtil.setTessPath(tessPath); + try { + String ocrImageWord = OCRUtil.getOcrImageWord(file); + Map map8=new HashMap<>(); + String fieldValue = "file_content" + "=" + "'"+ocrImageWord+"'"; + String conditionSql = "file_name_server" + "=" + "'"+fileNameServer+"'"; + map8.put("tableName",tableName+"_temp_file"); + map8.put("fieldValue",fieldValue); + map8.put("conditionSql",conditionSql); + danganguanliService.updateObject(map8); + ocrLog.setFileName(file.getName()); + ocrLog.setTableName(tableName+"_temp_file"); + ocrLog.setStatusType("0"); + } catch (Exception e) { + ocrLog.setFailureReason(e.getMessage()); + ocrLog.setStatusType("1"); + ocrLogMapper.insert(ocrLog); + throw new RuntimeException("ocr:"+e); + } } - }else{ - OCRUtil.setTessPath(tessPath); - try { - String ocrImageWord = OCRUtil.getOcrImageWord(file); - Map map8=new HashMap<>(); - String fieldValue = "file_content" + "=" + "'"+ocrImageWord+"'"; - String conditionSql = "file_name_server" + "=" + "'"+fileNameServer+"'"; - map8.put("tableName",tableName+"_temp_file"); - map8.put("fieldValue",fieldValue); - map8.put("conditionSql",conditionSql); - danganguanliService.updateObject(map8); - ocrLog.setFileName(file.getName()); - ocrLog.setTableName(tableName+"_temp_file"); - ocrLog.setStatusType("0"); - } catch (Exception e) { - ocrLog.setFailureReason(e.getMessage()); - ocrLog.setStatusType("1"); - ocrLogMapper.insert(ocrLog); - throw new RuntimeException("ocr:"+e); - } - } - ocrLogMapper.insert(ocrLog); + ocrLogMapper.insert(ocrLog); + } finally { + //释放信号量许可 + fileProcessSemaphore.release(); + } }); }