超过100万行数据导出execl,springboot-kotlin项目

x33g5p2x  于2022-06-29 转载在 Spring  
字(4.5k)|赞(0)|评价(0)|浏览(686)

项目中的用量详情,一个月的数据已经超过100万行,需要能够导出来,普通的方法不进导出比较慢,而且会出现OOM问题。

介绍一款可以导出100万行的方法:
https://github.com/alibaba/easyexcel

1、第一种方法:写入10万行,采用分而治之的思想处理

  1. @RequestMapping("/xxxexportcsv100")
  2. fun export(response: HttpServletResponse) {
  3. val outputStream: OutputStream = response.outputStream
  4. val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
  5. response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
  6. response.contentType = "application/octet-stream;charset=UTF-8"
  7. response.setHeader("Pragma", "No-cache")
  8. response.setHeader("Cache-Control", "no-cache")
  9. response.setDateHeader("Expires", 0)
  10. val count: Int = billingDetailService.countByIdIsNotNull()
  11. val totalStopWatch = StopWatch()
  12. totalStopWatch.start()
  13. if (count > 100000) {
  14. val max = 100000
  15. val excelWriter: ExcelWriter = EasyExcel.write(outputStream).build()
  16. for (i in 0 until count / max + 1) {
  17. val stopWatch = StopWatch()
  18. stopWatch.start()
  19. val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, max)
  20. stopWatch.stop()
  21. logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
  22. val writerStopWatch = StopWatch()
  23. writerStopWatch.start()
  24. val writeSheet: WriteSheet =
  25. EasyExcel.writerSheet(i, "账单" + (i + 1)).head(BillingDetailResourceDto::class.java)
  26. .registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).build()
  27. excelWriter.write(exportList, writeSheet)
  28. writerStopWatch.stop()
  29. logger.info("执行一次100000行写出到execl,花费的时间${writerStopWatch.totalTimeSeconds}")
  30. }
  31. //刷新流
  32. excelWriter.finish()
  33. } else {
  34. val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(0, count)
  35. EasyExcel.write(outputStream, BillingDetailResourceDto::class.java)
  36. .registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).sheet("账单").doWrite(exportList)
  37. }
  38. outputStream.flush()
  39. totalStopWatch.stop()
  40. logger.info("导出execl总耗时=${totalStopWatch.totalTimeSeconds}")
  41. response.outputStream.close()
  42. }

导出的execl列:

测试结果如下:
475276行数据,平局耗时38秒左右

测试结果如下:
1261138行数据,平局耗时113秒左右

第二种方法:每个sheet,写入10万行,每个sheet再次分多次写入

  1. @RequestMapping("/yyyexportcsv100")
  2. fun writeExcelByMulSheetAndWriteChange(response: HttpServletResponse) {
  3. val outputStream: OutputStream = response.outputStream
  4. val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
  5. response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
  6. response.contentType = "application/octet-stream;charset=UTF-8"
  7. response.setHeader("Pragma", "No-cache")
  8. response.setHeader("Cache-Control", "no-cache")
  9. response.setDateHeader("Expires", 0)
  10. //每个sheet写入的数据
  11. val NUM_PER_SHEET = 100000
  12. //每次向sheet中写入的数据
  13. val NUM_BY_TIMES = 50000
  14. val startTime = System.currentTimeMillis()
  15. // 获取数据
  16. val count: Int = billingDetailService.countByIdIsNotNull()
  17. // 获取 sheet 的个数
  18. val sheetNum: Int =
  19. if (count % NUM_PER_SHEET === 0) count / NUM_PER_SHEET else count / NUM_PER_SHEET + 1
  20. // 获取每个sheet 写入的次数
  21. val writeNumPerSheet: Int =
  22. if (NUM_PER_SHEET % NUM_BY_TIMES === 0) NUM_PER_SHEET / NUM_BY_TIMES else NUM_PER_SHEET / NUM_BY_TIMES + 1
  23. // 最后一个 sheet 写入的数量
  24. val writeNumLastSheet: Int = count - (sheetNum - 1) * NUM_PER_SHEET
  25. // 最后一个 sheet 写入的次数
  26. val writeNumPerLastSheet: Int =
  27. if (writeNumLastSheet % NUM_BY_TIMES === 0) writeNumLastSheet / NUM_BY_TIMES else writeNumLastSheet / NUM_BY_TIMES + 1
  28. // 指定写入的文件
  29. val excelWriter = EasyExcel.write(outputStream, BillingDetailResourceDto::class.java).build()
  30. for (i in 0 until sheetNum) {
  31. val sheetName = "sheet$i"
  32. val writeSheet = EasyExcel.writerSheet(i, sheetName).build()
  33. val writeNum = if (i == sheetNum - 1) writeNumPerLastSheet else writeNumPerSheet // 每个sheet 写入的次数
  34. val endEndNum = if (i == sheetNum - 1) count else (i + 1) * NUM_PER_SHEET // 每个sheet 最后一次写入的最后行数
  35. for (j in 0 until writeNum) {
  36. val l = System.currentTimeMillis()
  37. val startNum: Int = i * NUM_PER_SHEET + j * NUM_BY_TIMES
  38. val endNum = if (j == writeNum - 1) endEndNum else i * NUM_PER_SHEET + (j + 1) * NUM_BY_TIMES
  39. val stopWatch = StopWatch()
  40. stopWatch.start()
  41. val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, endNum - startNum)
  42. stopWatch.stop()
  43. logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
  44. excelWriter!!.write(exportList, writeSheet)
  45. logger.info(
  46. "写入sheet={},数据量={}-{}={},耗时={}",
  47. sheetName,
  48. endNum,
  49. startNum,
  50. endNum - startNum,
  51. System.currentTimeMillis() - l
  52. )
  53. }
  54. }
  55. // 需要放入 finally 中
  56. excelWriter?.finish()
  57. logger.info("导出excel结束,总数据量={},耗时={}ms", count, System.currentTimeMillis() - startTime)
  58. response.outputStream.close()
  59. }

测试结果如下:
475276行数据,平局耗时37秒左右

测试结果如下:
1261138行数据,平局耗时106秒左右

两种方法相差不多,100万行的数据导出,还是需要从产品角度考虑,不能让用户一直在页面等待。

相关文章