项目中的用量详情,一个月的数据已经超过100万行,需要能够导出来,普通的方法不进导出比较慢,而且会出现OOM问题。
介绍一款可以导出100万行的方法:
https://github.com/alibaba/easyexcel
@RequestMapping("/xxxexportcsv100")
fun export(response: HttpServletResponse) {
val outputStream: OutputStream = response.outputStream
val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
response.contentType = "application/octet-stream;charset=UTF-8"
response.setHeader("Pragma", "No-cache")
response.setHeader("Cache-Control", "no-cache")
response.setDateHeader("Expires", 0)
val count: Int = billingDetailService.countByIdIsNotNull()
val totalStopWatch = StopWatch()
totalStopWatch.start()
if (count > 100000) {
val max = 100000
val excelWriter: ExcelWriter = EasyExcel.write(outputStream).build()
for (i in 0 until count / max + 1) {
val stopWatch = StopWatch()
stopWatch.start()
val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, max)
stopWatch.stop()
logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
val writerStopWatch = StopWatch()
writerStopWatch.start()
val writeSheet: WriteSheet =
EasyExcel.writerSheet(i, "账单" + (i + 1)).head(BillingDetailResourceDto::class.java)
.registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).build()
excelWriter.write(exportList, writeSheet)
writerStopWatch.stop()
logger.info("执行一次100000行写出到execl,花费的时间${writerStopWatch.totalTimeSeconds}")
}
//刷新流
excelWriter.finish()
} else {
val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(0, count)
EasyExcel.write(outputStream, BillingDetailResourceDto::class.java)
.registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).sheet("账单").doWrite(exportList)
}
outputStream.flush()
totalStopWatch.stop()
logger.info("导出execl总耗时=${totalStopWatch.totalTimeSeconds}")
response.outputStream.close()
}
导出的execl列:
测试结果如下:
475276行数据,平局耗时38秒左右
测试结果如下:
1261138行数据,平局耗时113秒左右
@RequestMapping("/yyyexportcsv100")
fun writeExcelByMulSheetAndWriteChange(response: HttpServletResponse) {
val outputStream: OutputStream = response.outputStream
val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
response.contentType = "application/octet-stream;charset=UTF-8"
response.setHeader("Pragma", "No-cache")
response.setHeader("Cache-Control", "no-cache")
response.setDateHeader("Expires", 0)
//每个sheet写入的数据
val NUM_PER_SHEET = 100000
//每次向sheet中写入的数据
val NUM_BY_TIMES = 50000
val startTime = System.currentTimeMillis()
// 获取数据
val count: Int = billingDetailService.countByIdIsNotNull()
// 获取 sheet 的个数
val sheetNum: Int =
if (count % NUM_PER_SHEET === 0) count / NUM_PER_SHEET else count / NUM_PER_SHEET + 1
// 获取每个sheet 写入的次数
val writeNumPerSheet: Int =
if (NUM_PER_SHEET % NUM_BY_TIMES === 0) NUM_PER_SHEET / NUM_BY_TIMES else NUM_PER_SHEET / NUM_BY_TIMES + 1
// 最后一个 sheet 写入的数量
val writeNumLastSheet: Int = count - (sheetNum - 1) * NUM_PER_SHEET
// 最后一个 sheet 写入的次数
val writeNumPerLastSheet: Int =
if (writeNumLastSheet % NUM_BY_TIMES === 0) writeNumLastSheet / NUM_BY_TIMES else writeNumLastSheet / NUM_BY_TIMES + 1
// 指定写入的文件
val excelWriter = EasyExcel.write(outputStream, BillingDetailResourceDto::class.java).build()
for (i in 0 until sheetNum) {
val sheetName = "sheet$i"
val writeSheet = EasyExcel.writerSheet(i, sheetName).build()
val writeNum = if (i == sheetNum - 1) writeNumPerLastSheet else writeNumPerSheet // 每个sheet 写入的次数
val endEndNum = if (i == sheetNum - 1) count else (i + 1) * NUM_PER_SHEET // 每个sheet 最后一次写入的最后行数
for (j in 0 until writeNum) {
val l = System.currentTimeMillis()
val startNum: Int = i * NUM_PER_SHEET + j * NUM_BY_TIMES
val endNum = if (j == writeNum - 1) endEndNum else i * NUM_PER_SHEET + (j + 1) * NUM_BY_TIMES
val stopWatch = StopWatch()
stopWatch.start()
val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, endNum - startNum)
stopWatch.stop()
logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
excelWriter!!.write(exportList, writeSheet)
logger.info(
"写入sheet={},数据量={}-{}={},耗时={}",
sheetName,
endNum,
startNum,
endNum - startNum,
System.currentTimeMillis() - l
)
}
}
// 需要放入 finally 中
excelWriter?.finish()
logger.info("导出excel结束,总数据量={},耗时={}ms", count, System.currentTimeMillis() - startTime)
response.outputStream.close()
}
测试结果如下:
475276行数据,平局耗时37秒左右
测试结果如下:
1261138行数据,平局耗时106秒左右
两种方法相差不多,100万行的数据导出,还是需要从产品角度考虑,不能让用户一直在页面等待。
版权说明 : 本文为转载文章, 版权归原作者所有 版权申明
原文链接 : https://blog.csdn.net/sunyuhua_keyboard/article/details/125489154
内容来源于网络,如有侵权,请联系作者删除!