From 59c32acf7c18ab0a593c3edae3512793b631e148 Mon Sep 17 00:00:00 2001 From: zhangluping <1801075613@qq.com> Date: Sat, 28 Dec 2024 11:23:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../demo/controller/ExportExcelController.java | 36 +++++++++++ .../com/example/demo/mapper/DetailYMapper.java | 25 ++++++-- .../com/example/demo/sevice/DataExportService.java | 71 ++++++++++++++++++++++ .../example/demo/sevice/ExportExcelService.java | 41 +++++++++++++ 4 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/example/demo/controller/ExportExcelController.java create mode 100644 src/main/java/com/example/demo/sevice/DataExportService.java create mode 100644 src/main/java/com/example/demo/sevice/ExportExcelService.java diff --git a/src/main/java/com/example/demo/controller/ExportExcelController.java b/src/main/java/com/example/demo/controller/ExportExcelController.java new file mode 100644 index 0000000..884a7ae --- /dev/null +++ b/src/main/java/com/example/demo/controller/ExportExcelController.java @@ -0,0 +1,36 @@ +package com.example.demo.controller; + +import com.example.demo.sevice.DataExportService; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; + +@RestController +@RequestMapping("/export") +@RequiredArgsConstructor +@Transactional +@Slf4j +@CrossOrigin +public class ExportExcelController { + + @Autowired + private DataExportService dataExportService; + + @PostMapping("/exportUserExcel") + public void exportUserExcel(HttpServletResponse response) { + try { + dataExportService.exportData(response); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("导出 Excel 文件失败", e); + } + } +} diff --git a/src/main/java/com/example/demo/mapper/DetailYMapper.java b/src/main/java/com/example/demo/mapper/DetailYMapper.java index c025cda..371bc1b 100644 --- a/src/main/java/com/example/demo/mapper/DetailYMapper.java +++ b/src/main/java/com/example/demo/mapper/DetailYMapper.java @@ -1,11 +1,9 @@ package com.example.demo.mapper; import com.example.demo.domain.entity.DetailY; +import com.example.demo.domain.entity.UserDetailExport; import com.example.demo.domain.vo.DetailYgold; -import org.apache.ibatis.annotations.Insert; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Options; -import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.*; import java.util.List; @@ -79,8 +77,8 @@ public interface DetailYMapper { "" }) DetailY getCount(DetailY detailY); + + //导出数据的SQL + @Select({ + "" + }) + List searchExport(@Param("offset") int offset, @Param("pageSize") int pageSize); + + @Select("SELECT COUNT(*) FROM detail_y") + int getTotalCount(); } diff --git a/src/main/java/com/example/demo/sevice/DataExportService.java b/src/main/java/com/example/demo/sevice/DataExportService.java new file mode 100644 index 0000000..66d5f15 --- /dev/null +++ b/src/main/java/com/example/demo/sevice/DataExportService.java @@ -0,0 +1,71 @@ +package com.example.demo.sevice; + +import com.example.demo.domain.entity.UserDetailExport; +import com.example.demo.mapper.DetailYMapper; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@Service +public class DataExportService { + // 设置线程池 + private final ExecutorService executorService = Executors.newFixedThreadPool(3); // 使用10个线程的线程池 + + // 每次查询20w条数据 + private static final int PAGE_SIZE = 100000; + + @Autowired + private DetailYMapper detailYMapper; + + @Autowired + private ExportExcelService exportExcelService; + + public void exportData(HttpServletResponse response) throws IOException { + // 获取总记录数 + int totalRecords = detailYMapper.getTotalCount(); // 获取总记录数 + int totalPages = (totalRecords + PAGE_SIZE - 1) / PAGE_SIZE; // 计算总页 + // 设置Excel响应头 + exportExcelService.setExcelResponseProp(response, "userDetail"); + + // 创建一个集合用于收集所有查询结果 + List allData = new ArrayList<>(); + List>> futures = new ArrayList<>(); + + // 使用多线程导出数据 + for (int page = 0; page < totalPages; page++) { + final int currentPage = page; + futures.add(executorService.submit(() -> { + try { + return fetchData(currentPage, PAGE_SIZE); // 查询当前页的数据 + } catch (Exception e) { + e.printStackTrace(); + return new ArrayList(); // 出错时返回空数据 + } + })); + } + + // 等待所有任务完成，并收集结果 + for (Future> future : futures) { + try { + allData.addAll(future.get()); // 将每个线程的结果合并 + } catch (Exception e) { + e.printStackTrace(); + } + } + + // 一次性写入Excel + exportExcelService.writeDataToSheet(response, allData, 0); + } + + private List fetchData(int page, int pageSize) { + int offset = page * pageSize; // 计算偏移量 + return detailYMapper.searchExport(offset, pageSize); // 调用Mapper查询数据 + } +} diff --git a/src/main/java/com/example/demo/sevice/ExportExcelService.java b/src/main/java/com/example/demo/sevice/ExportExcelService.java new file mode 100644 index 0000000..2b6fdb1 --- /dev/null +++ b/src/main/java/com/example/demo/sevice/ExportExcelService.java @@ -0,0 +1,41 @@ +package com.example.demo.sevice; + +import com.alibaba.excel.EasyExcel; +import com.example.demo.domain.entity.UserDetailExport; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.List; + +@Service +public class ExportExcelService { + + // 设置响应头以导出 Excel 文件 + public void setExcelResponseProp(HttpServletResponse response, String fileName) throws IOException { + response.setContentType("application/vnd.ms-excel"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Disposition", "attachment; filename=" + fileName + ".xlsx"); + } + + // 使用 EasyExcel 写入数据到 Excel + public void writeDataToSheet(HttpServletResponse response, List allData, int sheetIndex) throws IOException { + int batchSize = 2000000; // 每次写入200w条 + int totalDataSize = allData.size(); + int currentSheetIndex = sheetIndex; // 用来区分不同的Sheet + + for (int i = 0; i < totalDataSize; i += batchSize) { + int endIndex = Math.min(i + batchSize, totalDataSize); + List batchData = allData.subList(i, endIndex); + + // 每次写入一个新的 Sheet，Sheet 名称使用 currentSheetIndex + EasyExcel.write(response.getOutputStream(), UserDetailExport.class) + .sheet("Sheet " + (currentSheetIndex + 1)) // 设置Sheet名称 + .doWrite(batchData); // 写入当前批次的数据 + + currentSheetIndex++; // 增加 Sheet 索引 + } + } +} + +