package org.example; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import java.io.File; import java.io.FileInputStream; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * 表1 */ public class IaShzhHdpDataImporter { // 防灾对象表插入SQL private static final String INSERT_HDP_SQL = "INSERT INTO SHZH_JCSJ.IA_SHZH_HDP(ADCD, ADNM, TYPE, PCOUNT, RVNM, RVCD, BRNAME, BRCD, " + "DAMNAME, DAMCD, DZRVNM, DZRVCD, ISSZ, ISJW, ISDW, ISGT, ISYS, ISDT, ISGD, ISML, " + "REMARK, ISENABLE, IMPORTYEAR, ISKJ, PRENAME, PRECODE, ISLHHP, ISNSL, SIGNER, CADCD, " + "GUID, AUDBATCH, INPUTTYPE, AUDID, STATUS, WSCD) \n" + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?)"; // 防灾对象河流信息表插入SQL private static final String INSERT_RV_INFO_SQL = "INSERT INTO SHZH_JCSJ.IA_RV_INFO (" + "PRECODE, PRENAME, RVCD, RVNM, WSCD" + ") VALUES (?, ?, ?, ?, ?)"; // 检查防灾对象重复的SQL(基于县编码、名称、编码、类型、人口) private static final String CHECK_HDP_DUPLICATE_SQL = "SELECT COUNT(*) FROM SHZH_JCSJ.IA_SHZH_HDP " + "WHERE ADCD = ? AND PRENAME = ? AND PRECODE = ? AND TYPE = ? AND " + "(PCOUNT = ? OR (PCOUNT IS NULL AND ? IS NULL))"; // 检查河流信息重复的SQL(基于precode, prename, rvcd, rvnm, wscd) private static final String CHECK_RV_INFO_DUPLICATE_SQL = "SELECT COUNT(*) FROM SHZH_JCSJ.IA_RV_INFO " + "WHERE PRECODE = ? AND PRENAME = ? AND RVCD = ? AND RVNM = ? AND WSCD = ?"; public static void main(String[] args) { // 先测试数据库连接 if (!testDatabaseConnection()) { System.err.println("数据库连接测试失败,程序退出"); return; } // 测试用的固定路径 // args = new String[]{"F:\\1111-湖工大50条成果-已通过初检-待入库"}; // args = new String[]{"F:\\3333-2024年度咸宁市8条20251017-已通过初检-待入库"}; // args = new String[]{"F:\\4444-长江科学院-黄冈28条-已通过初检-已统一表头-待入库"}; // args = new String[]{"F:\\3333-华科-地大9条-20251021-已通过初检-已统一表头-待入库"}; // args = new String[]{"F:\\小流域\\5555-长江科学院20条10.23-已通过初检-已统一表头-待入库\\5555-长江科学院20条10.23-已通过初检-已统一表头-待入库"}; args = new String[]{"I:\\8888-长科院-孝昌县8条1028-已通过初检-已统一表头-待入库"}; if (args.length == 0) { System.out.println("请提供湖工大9.29目录路径作为参数"); return; } String baseDir = args[0]; File baseDirectory = new File(baseDir); if (!baseDirectory.exists() || !baseDirectory.isDirectory()) { System.out.println("指定的目录不存在或不是有效目录: " + baseDir); return; } System.out.println("开始处理目录: " + baseDir); try { processAllCounties(baseDirectory); } catch (Exception e) { System.err.println("处理过程中发生错误: " + e.getMessage()); e.printStackTrace(); } } private static boolean testDatabaseConnection() { try { Connection conn = DatabaseUtil.getConnection(); if (conn != null && !conn.isClosed()) { System.out.println("✓ 数据库连接测试成功"); conn.close(); return true; } } catch (Exception e) { System.err.println("✗ 数据库连接测试失败: " + e.getMessage()); } return false; } private static void processAllCounties(File baseDir) { File[] countyDirs = baseDir.listFiles(File::isDirectory); if (countyDirs == null) { System.out.println("在基础目录下未找到任何县市文件夹"); return; } int totalProcessed = 0; int totalSuccess = 0; int totalDuplicates = 0; int totalRvDuplicates = 0; System.out.println("找到 " + countyDirs.length + " 个县市文件夹"); for (File countyDir : countyDirs) { String countyName = extractCountyName(countyDir.getName()); System.out.println("\n=== 处理县市: " + countyName + " ==="); try { ProcessResult result = processCountyData(countyDir, countyName); totalProcessed += result.processed; totalDuplicates += result.duplicates; totalRvDuplicates += result.rvDuplicates; totalSuccess++; System.out.println("✓ 成功处理 " + countyName + ",导入 " + result.processed + " 条记录,跳过 " + result.duplicates + " 条重复记录,河流信息表跳过 " + result.rvDuplicates + " 条重复记录"); } catch (Exception e) { System.err.println("✗ 处理县市 " + countyName + " 时出错: " + e.getMessage()); } } System.out.println("\n=== 处理完成 ==="); System.out.println("成功处理县市: " + totalSuccess + " 个"); System.out.println("总导入记录: " + totalProcessed + " 条"); System.out.println("防灾对象表跳过重复记录: " + totalDuplicates + " 条"); System.out.println("河流信息表跳过重复记录: " + totalRvDuplicates + " 条"); } private static ProcessResult processCountyData(File countyDir, String countyName) throws Exception { int totalProcessed = 0; int totalDuplicates = 0; int totalRvDuplicates = 0; // 查找小流域目录 File watershedDir = findWatershedDirectory(countyDir); if (watershedDir == null) { System.out.println("在 " + countyName + " 下未找到小流域目录"); return new ProcessResult(0, 0, 0); } System.out.println("找到小流域目录: " + watershedDir.getName()); // 查找所有水系目录 File[] sxDirs = findSxDirectories(watershedDir); if (sxDirs == null || sxDirs.length == 0) { System.out.println("在 " + watershedDir.getPath() + " 下未找到水系目录"); return new ProcessResult(0, 0, 0); } System.out.println("找到 " + sxDirs.length + " 个水系文件夹:"); for (File sxDir : sxDirs) { System.out.println(" - " + sxDir.getName()); } // 遍历处理每个水系文件夹 for (File sxDir : sxDirs) { System.out.println("\n处理水系: " + sxDir.getName()); ProcessResult result = processSxData(sxDir, countyName); totalProcessed += result.processed; totalDuplicates += result.duplicates; totalRvDuplicates += result.rvDuplicates; System.out.println("水系 " + sxDir.getName() + " 导入 " + result.processed + " 条记录,跳过 " + result.duplicates + " 条重复记录,河流信息表跳过 " + result.rvDuplicates + " 条重复记录"); } return new ProcessResult(totalProcessed, totalDuplicates, totalRvDuplicates); } private static ProcessResult processSxData(File sxDir, String countyName) throws Exception { int processedCount = 0; int duplicateCount = 0; int rvDuplicateCount = 0; // 查找成果报表目录 File reportDir = findReportDirectory(sxDir); if (reportDir == null) { System.out.println("在 " + sxDir.getPath() + " 下未找到成果报表目录"); return new ProcessResult(0, 0, 0); } System.out.println("找到成果报表目录: " + reportDir.getPath()); // 查找Excel文件 File excelFile = findExcelFile(reportDir); if (excelFile == null) { System.out.println("在 " + reportDir.getPath() + " 下未找到表1山洪灾害防治对象名录.xlsx"); return new ProcessResult(0, 0, 0); } System.out.println("找到Excel文件: " + excelFile.getPath()); // 提取流域编码 String watershedCode = CommonUtils.extractWatershedCode(sxDir.getName()); System.out.println("流域编码: " + watershedCode); // 读取Excel文件并导入数据库 ProcessResult result = readExcelAndImport(excelFile, countyName, watershedCode); return result; } private static ProcessResult readExcelAndImport(File excelFile, String countyName, String watershedCode) throws Exception { int importedCount = 0; int duplicateCount = 0; int rvDuplicateCount = 0; // 先读取所有数据到内存 List hdpDataList = new ArrayList<>(); List rvInfoDataList = new ArrayList<>(); readExcelData(excelFile, countyName, watershedCode, hdpDataList, rvInfoDataList); if (hdpDataList.isEmpty()) { System.out.println("未读取到有效数据"); return new ProcessResult(0, 0, 0); } System.out.println("成功读取 " + hdpDataList.size() + " 条数据,开始导入数据库..."); // 批量导入数据库 ProcessResult result = batchInsertData(hdpDataList, rvInfoDataList); return result; } private static String extractCountyCode(Sheet sheet) { // 从第二行提取县代码 Row headerRow = sheet.getRow(1); if (headerRow != null) { Cell codeCell = headerRow.getCell(9); // H列 - 县代码 if (codeCell != null) { return getCellStringValue(codeCell); } } return ""; // 默认值 } private static String extractAdcd(Sheet sheet) { // 从第二行提取乡镇代码 Row headerRow = sheet.getRow(1); if (headerRow != null) { Cell codeCell = headerRow.getCell(22); // H列 - 乡镇代码 if (codeCell != null) { return getCellStringValue(codeCell); } } return ""; // 默认值 } private static String extractAdnm(Sheet sheet) { // 从第二行提取乡镇名称 Row headerRow = sheet.getRow(1); if (headerRow != null) { Cell codeCell = headerRow.getCell(14); // H列 - 乡镇名称 if (codeCell != null) { return getCellStringValue(codeCell); } } return ""; // 默认值 } private static void readExcelData(File excelFile, String countyName, String watershedCode, List hdpDataList, List rvInfoDataList) throws Exception { try (FileInputStream fis = new FileInputStream(excelFile); Workbook workbook = new XSSFWorkbook(fis)) { // 遍历所有工作表 for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) { Sheet sheet = workbook.getSheetAt(sheetIndex); String sheetName = sheet.getSheetName().replaceAll("[\\u4e00-\\u9fa5]+$", ""); // 跳过填表说明等非数据工作表 if (sheetName.contains("填表说明") || sheetName.contains("说明")) { continue; } System.out.println("处理工作表: " + sheetName); String countyCode = extractCountyCode(sheet); String adcd = extractAdcd(sheet); String adnm = extractAdnm(sheet); int startRow = findDataStartRow(sheet); if (startRow == -1) { System.out.println("未找到数据开始行"); return; } System.out.println("数据从第 " + (startRow + 1) + " 行开始,共 " + (sheet.getLastRowNum() - startRow + 1) + " 行"); for (int i = startRow; i <= sheet.getLastRowNum(); i++) { Row row = sheet.getRow(i); if (row == null) continue; Cell serialCell = row.getCell(0); if (serialCell == null || isCellEmpty(serialCell)) { continue; } Object[] hdpData = parseRowData(row, countyName, watershedCode, countyCode, adcd, adnm); if (hdpData != null) { hdpDataList.add(hdpData); // 同时生成河流信息表数据 Object[] rvInfoData = parseRvInfoData(hdpData); if (rvInfoData != null) { rvInfoDataList.add(rvInfoData); } System.out.println("✓ 解析第 " + (i + 1) + " 行数据成功"); } else { System.out.println("✗ 解析第 " + (i + 1) + " 行数据失败"); } } } } } private static ProcessResult batchInsertData(List hdpDataList, List rvInfoDataList) { int importedCount = 0; int duplicateCount = 0; int rvDuplicateCount = 0; try (Connection conn = DatabaseUtil.getConnection(); PreparedStatement hdpPstmt = conn.prepareStatement(INSERT_HDP_SQL); PreparedStatement rvInfoPstmt = conn.prepareStatement(INSERT_RV_INFO_SQL); PreparedStatement checkHdpDupPstmt = conn.prepareStatement(CHECK_HDP_DUPLICATE_SQL); PreparedStatement checkRvInfoDupPstmt = conn.prepareStatement(CHECK_RV_INFO_DUPLICATE_SQL)) { conn.setAutoCommit(false); // 开启事务 // 使用Set来跟踪已处理的数据,避免内存中重复 Set processedHdpKeys = new HashSet<>(); Set processedRvInfoKeys = new HashSet<>(); // 处理防灾对象表数据 for (Object[] rowData : hdpDataList) { try { // 检查是否重复 String adcd = (String) rowData[0]; String prename = (String) rowData[24]; // PRENAME在索引24 String precode = (String) rowData[25]; // PRECODE在索引25 String type = (String) rowData[2]; // TYPE在索引2 Integer pcount = (Integer) rowData[3]; // PCOUNT在索引3 // 检查必要字段是否为空 if (adcd == null || prename == null || precode == null || type == null) { System.out.println("跳过数据不完整的记录: " + prename + " (" + precode + ")"); continue; } // 生成唯一键,先检查内存中是否已处理 String hdpKey = adcd + "|" + prename + "|" + precode + "|" + type + "|" + pcount; if (processedHdpKeys.contains(hdpKey)) { duplicateCount++; System.out.println("跳过内存中重复记录: " + prename + " (" + precode + ")"); continue; } // 检查数据库中是否已存在 checkHdpDupPstmt.setString(1, adcd); checkHdpDupPstmt.setString(2, prename); checkHdpDupPstmt.setString(3, precode); checkHdpDupPstmt.setString(4, type); if (pcount != null) { checkHdpDupPstmt.setInt(5, pcount); checkHdpDupPstmt.setInt(6, pcount); } else { checkHdpDupPstmt.setNull(5, java.sql.Types.INTEGER); checkHdpDupPstmt.setNull(6, java.sql.Types.INTEGER); } try (ResultSet rs = checkHdpDupPstmt.executeQuery()) { if (rs.next() && rs.getInt(1) > 0) { duplicateCount++; System.out.println("跳过数据库中重复记录: " + prename + " (" + precode + ")"); continue; } } // 插入数据 for (int i = 0; i < rowData.length; i++) { hdpPstmt.setObject(i + 1, rowData[i]); } hdpPstmt.addBatch(); importedCount++; processedHdpKeys.add(hdpKey); // 记录已处理的键 // 每10条执行一次批量插入 if (importedCount % 10 == 0) { hdpPstmt.executeBatch(); System.out.println("已批量插入 " + importedCount + " 条记录到防灾对象表"); } } catch (Exception e) { System.err.println("单条数据插入失败: " + e.getMessage()); e.printStackTrace(); // 继续处理下一条数据 } } // 执行剩余的批量插入 if (importedCount > 0 && importedCount % 10 != 0) { hdpPstmt.executeBatch(); System.out.println("最终批量插入完成,共 " + importedCount + " 条记录到防灾对象表"); } // 处理河流信息表数据 int rvImportedCount = 0; for (Object[] rvData : rvInfoDataList) { try { // 检查河流信息是否重复 String precode = (String) rvData[0]; String prename = (String) rvData[1]; String rvcd = (String) rvData[2]; String rvnm = (String) rvData[3]; String wscd = (String) rvData[4]; // 检查必要字段是否为空 if (precode == null || prename == null || rvcd == null || rvnm == null || wscd == null) { System.out.println("跳过河流信息数据不完整的记录: " + prename + " (" + precode + ")"); continue; } // 生成唯一键,先检查内存中是否已处理 String rvInfoKey = precode + "|" + prename + "|" + rvcd + "|" + rvnm + "|" + wscd; if (processedRvInfoKeys.contains(rvInfoKey)) { rvDuplicateCount++; System.out.println("跳过内存中河流信息重复记录: " + prename + " (" + precode + ") - " + rvnm); continue; } // 检查数据库中是否已存在 checkRvInfoDupPstmt.setString(1, precode); checkRvInfoDupPstmt.setString(2, prename); checkRvInfoDupPstmt.setString(3, rvcd); checkRvInfoDupPstmt.setString(4, rvnm); checkRvInfoDupPstmt.setString(5, wscd); try (ResultSet rs = checkRvInfoDupPstmt.executeQuery()) { if (rs.next() && rs.getInt(1) > 0) { rvDuplicateCount++; System.out.println("跳过数据库中河流信息重复记录: " + prename + " (" + precode + ") - " + rvnm); continue; } } for (int i = 0; i < rvData.length; i++) { rvInfoPstmt.setObject(i + 1, rvData[i]); } rvInfoPstmt.addBatch(); rvImportedCount++; processedRvInfoKeys.add(rvInfoKey); // 记录已处理的键 if (rvImportedCount % 10 == 0) { rvInfoPstmt.executeBatch(); System.out.println("已批量插入 " + rvImportedCount + " 条记录到河流信息表"); } } catch (Exception e) { System.err.println("河流信息表数据插入失败: " + e.getMessage()); e.printStackTrace(); } } // 执行剩余的河流信息批量插入 if (rvImportedCount > 0 && rvImportedCount % 10 != 0) { rvInfoPstmt.executeBatch(); System.out.println("最终批量插入完成,共 " + rvImportedCount + " 条记录到河流信息表"); } conn.commit(); // 提交事务 System.out.println("事务提交成功"); } catch (Exception e) { System.err.println("批量插入数据失败: " + e.getMessage()); e.printStackTrace(); } return new ProcessResult(importedCount, duplicateCount, rvDuplicateCount); } // 修改ProcessResult类,添加河流信息重复统计 private static class ProcessResult { int processed; int duplicates; int rvDuplicates; ProcessResult(int processed, int duplicates, int rvDuplicates) { this.processed = processed; this.duplicates = duplicates; this.rvDuplicates = rvDuplicates; } } // 解析河流信息表数据 private static Object[] parseRvInfoData(Object[] hdpData) { try { Object[] rvData = new Object[5]; // PRECODE rvData[0] = hdpData[25]; // PRECODE在索引25 // PRENAME rvData[1] = hdpData[24]; // PRENAME在索引24 // RVCD - 河流代码 rvData[2] = hdpData[5]; // RVCD在索引5 // RVNM - 河流名称 rvData[3] = hdpData[4]; // RVNM在索引4 // WSCD - 流域编码 rvData[4] = hdpData[35]; // WSCD在索引35 return rvData; } catch (Exception e) { System.err.println("解析河流信息数据失败: " + e.getMessage()); return null; } } // 其他辅助方法保持不变... private static File findWatershedDirectory(File countyDir) { File[] subDirs = countyDir.listFiles(File::isDirectory); if (subDirs != null) { for (File subDir : subDirs) { if (subDir.getName().contains("小流域")) { return subDir; } } if (subDirs.length > 0) return subDirs[0]; } return null; } private static File[] findSxDirectories(File watershedDir) { File[] subDirs = watershedDir.listFiles(File::isDirectory); if (subDirs != null) { List sxDirs = new ArrayList<>(); for (File subDir : subDirs) { // 包含"水系"或者是类似HBWF开头的文件夹都认为是水系文件夹 if (subDir.getName().contains("水系") || subDir.getName().matches("^HBWF.*")) { sxDirs.add(subDir); } } return sxDirs.toArray(new File[0]); } return null; } private static File findReportDirectory(File sxDir) { File[] subDirs = sxDir.listFiles(File::isDirectory); if (subDirs != null) { for (File subDir : subDirs) { if (subDir.getName().contains("成果报表")) { return subDir; } } if (subDirs.length > 0) return subDirs[0]; } return null; } private static File findExcelFile(File reportDir) { File[] files = reportDir.listFiles((dir, name) -> name.equals("表1山洪灾害防治对象名录.xlsx") || name.toLowerCase().endsWith(".xlsx")); if (files != null && files.length > 0) { for (File file : files) { if (file.getName().equals("表1山洪灾害防治对象名录.xlsx")) { return file; } } return files[0]; } return null; } private static String extractCountyName(String dirName) { return dirName.contains("、") ? dirName.split("、")[1].trim() : dirName; } private static int findDataStartRow(Sheet sheet) { for (int i = 0; i <= sheet.getLastRowNum(); i++) { Row row = sheet.getRow(i); if (row != null) { Cell cell = row.getCell(0); if (cell != null && !isCellEmpty(cell)) { if (cell.getCellType() == CellType.NUMERIC) { double value = cell.getNumericCellValue(); if (value >= 1 && value <= 1000) return i; } else if (cell.getCellType() == CellType.STRING) { String value = cell.getStringCellValue().trim(); if (value.matches("\\d+")) return i; } } } } return -1; } private static String getCellStringValue(Cell cell) { if (cell == null) return null; try { switch (cell.getCellType()) { case STRING: return cell.getStringCellValue().trim(); case NUMERIC: double num = cell.getNumericCellValue(); return (num == (long) num) ? String.valueOf((long) num) : String.valueOf(num); case BOOLEAN: return String.valueOf(cell.getBooleanCellValue()); default: return null; } } catch (Exception e) { return null; } } private static boolean isCellEmpty(Cell cell) { if (cell == null) return true; switch (cell.getCellType()) { case BLANK: return true; case STRING: return cell.getStringCellValue().trim().isEmpty(); default: return false; } } // parseRowData方法保持不变... private static Object[] parseRowData(Row row, String countyName, String watershedCode, String countyCode,String adcd ,String adnm) { try { Object[] data = new Object[36]; // 对应36个参数 int index = 0; // ADCD - 行政区划代码 (C列) String objectCode = getCellStringValue(row.getCell(2)); data[index++] = adcd; // ADNM - 行政区划名称 (B列 + 县市名称) String objectName = getCellStringValue(row.getCell(1)); data[index++] = adnm; // TYPE - 类型 (D列) data[index++] = getCellStringValue(row.getCell(3)); // PCOUNT - 人口 (E列) String populationStr = getCellStringValue(row.getCell(4)); if (populationStr != null && !populationStr.trim().isEmpty()) { try { data[index++] = Integer.parseInt(populationStr.trim()); } catch (NumberFormatException e) { data[index++] = null; } } else { data[index++] = null; } // RVNM - 河流名称 (F列) data[index++] = getCellStringValue(row.getCell(5)); // RVCD - 河流代码 (G列) data[index++] = getCellStringValue(row.getCell(6)); // BRNAME - 桥梁名称 (H列) data[index++] = getCellStringValue(row.getCell(7)); // BRCD - 桥梁编码 (I列) data[index++] = getCellStringValue(row.getCell(8)); // DAMNAME - 塘坝名称 (J列) data[index++] = getCellStringValue(row.getCell(9)); // DAMCD - 塘坝编码 (K列) data[index++] = getCellStringValue(row.getCell(10)); // DZRVNM - 多支河流名称 (L列) data[index++] = getCellStringValue(row.getCell(11)); // DZRVCD - 多支河流代码 (M列) data[index++] = getCellStringValue(row.getCell(12)); // 风险隐患要素 (N~S列) data[index++] = getCellStringValue(row.getCell(13)); // ISSZ束窄 data[index++] = getCellStringValue(row.getCell(14)); // ISJW急弯 data[index++] = getCellStringValue(row.getCell(15)); // ISDW低洼地 data[index++] = getCellStringValue(row.getCell(18)); // ISGT沟滩占地 data[index++] = getCellStringValue(row.getCell(20)); // ISYS壅水 data[index++] = getCellStringValue(row.getCell(21)); // ISDT顶托 // 风险隐患影响类型 (T~Y列) data[index++] = getCellStringValue(row.getCell(22)); // ISGD改道 data[index++] = getCellStringValue(row.getCell(23)); // ISML漫流 // REMARK - 备注 (Z列) data[index++] = getCellStringValue(row.getCell(24)); // 固定值字段 data[index++] = "1"; // ISENABLE data[index++] = "2025"; // IMPORTYEAR data[index++] = getCellStringValue(row.getCell(19)); // ISKJ溃决 data[index++] = objectName; // PRENAME data[index++] = objectCode; // PRECODE data[index++] = getCellStringValue(row.getCell(16)); // ISLHHP临河滑坡 data[index++] = getCellStringValue(row.getCell(17)); // ISNSL泥石流 data[index++] = null; // SIGNER data[index++] = countyCode; // CADCD // GUID - 唯一标识 data[index++] = UUID.randomUUID().toString(); // 其他固定字段 data[index++] = "BATCH_" + System.currentTimeMillis(); // AUDBATCH data[index++] = "1"; // INPUTTYPE data[index++] = null; // AUDID data[index++] = "1"; // STATUS // WSCD - 流域编码 data[index++] = watershedCode; return data; } catch (Exception e) { System.err.println("解析行数据失败: " + e.getMessage()); return null; } } }