情景:處理歷史流量數(shù)據(jù),由于數(shù)據(jù)量比較大,可以按照時(shí)間或者分頁(yè)進(jìn)行分批處理
設(shè)計(jì)邏輯:定時(shí)任務(wù)處理前一天數(shù)據(jù),分頁(yè)查詢分批處理數(shù)據(jù)
Boolean haveData = true;
Integer pageNum = 1;
String endDayStr = DateUtil.format(start, Constants.DAY_FORMAT);
String startDayStr = DateUtil.format(DateUtil.offsetDay(start, dayOffset), Constants.DAY_FORMAT);
while (haveData){
Page<dto> detailInfo = null;
detailInfo = service.getInfoByPage(pageNum,
Constants.pageSize,startDayStr, endDayStr);
assert detailInfo != null;
List<dto> dtoList = detailInfo .getRecords();
if(CollUtil.isNotEmpty(dtoList )){
//具體處理邏輯
pageNum++;
}else {
haveData = false;
}
}
}
//這里直接用mybatis plus的分頁(yè)查詢進(jìn)行處理的,可以自己寫個(gè)mapper或者直接用LambdaQueryWrapper做條件查詢都可以
@Override
public Page<dto> getInfoByPage(Integer pageNum, Integer pageSize, String startDayStr, String endDayStr) {
Page<dto> page = new Page<>(pageNum, pageSize);
baseMapper.getInfoByPage(page,startDayStr,endDayStr);
return page;
}
pom依賴
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.5.5</version>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>dynamic-datasource-spring-boot3-starter</artifactId>
<version>4.3.0</version>
</dependency>