demand: TMS系统 - 运输管理 - 通过多线程分批处理，优化漏组批次订单一次性查询/导出10万+数据的效率
committer: heyu
This commit is contained in:
parent
019de52f66
commit
a0280d35d4
@ -5,6 +5,10 @@ import java.util.stream.Collectors;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import com.xdadan.erp.emis.domain.*;
|
||||
@ -73,6 +77,36 @@ public class EmisTmsSiteBatchMissServiceImpl extends EmisBaseService implements
|
||||
@Override
|
||||
public List<EmisTmsSiteBatchMiss> selectEmisTmsSiteBatchMissList(EmisTmsSiteBatchMiss emisTmsSiteBatchMiss) {
|
||||
log.info("Selecting TMS site batch miss list with params: {}", emisTmsSiteBatchMiss);
|
||||
|
||||
// 检查是否需要多线程优化
|
||||
// 通过检查请求参数中是否包含大分页标识
|
||||
boolean useMultiThread = false;
|
||||
Integer pageSize = null;
|
||||
Integer pageNum = null;
|
||||
|
||||
// 尝试从params中获取分页参数
|
||||
if (emisTmsSiteBatchMiss.getParams() != null) {
|
||||
Object pageSizeObj = emisTmsSiteBatchMiss.getParams().get("pageSize");
|
||||
Object pageNumObj = emisTmsSiteBatchMiss.getParams().get("pageNum");
|
||||
|
||||
if (pageSizeObj instanceof Integer) {
|
||||
pageSize = (Integer) pageSizeObj;
|
||||
if (pageSize > 1000) {
|
||||
useMultiThread = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (pageNumObj instanceof Integer) {
|
||||
pageNum = (Integer) pageNumObj;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果pageSize > 1000，使用多线程查询优化
|
||||
if (useMultiThread && pageSize != null && pageNum != null) {
|
||||
log.info("Using multi-thread optimization for large pageSize: {}", pageSize);
|
||||
return selectEmisTmsSiteBatchMissListWithMultiThread(emisTmsSiteBatchMiss, pageSize, pageNum);
|
||||
}
|
||||
|
||||
List<EmisTmsSiteBatchMiss> list = null;
|
||||
// 如果单号不为空，则去除其他参数
|
||||
if (!StringUtils.isEmpty(emisTmsSiteBatchMiss.getBillCode())) {
|
||||
@ -130,6 +164,184 @@ public class EmisTmsSiteBatchMissServiceImpl extends EmisBaseService implements
|
||||
return list;
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用多线程查询TMS漏组批次信息列表
|
||||
* 当pageSize > 1000时，将查询任务拆分为多个线程，每个线程查询1000条记录
|
||||
*
|
||||
* @param emisTmsSiteBatchMiss TMS漏组批次信息
|
||||
* @param pageSize 分页大小
|
||||
* @param pageNum 页码
|
||||
* @return TMS漏组批次信息列表
|
||||
*/
|
||||
private List<EmisTmsSiteBatchMiss> selectEmisTmsSiteBatchMissListWithMultiThread(
|
||||
EmisTmsSiteBatchMiss emisTmsSiteBatchMiss, Integer pageSize, Integer pageNum) {
|
||||
|
||||
log.info("Using multi-thread query optimization for pageSize: {}, pageNum: {}", pageSize, pageNum);
|
||||
|
||||
try {
|
||||
// 计算需要的线程数，每个线程处理1000条记录
|
||||
final int RECORDS_PER_THREAD = 1000;
|
||||
int threadCount = (int) Math.ceil((double) pageSize / RECORDS_PER_THREAD);
|
||||
|
||||
// 创建线程池，线程数不超过CPU核心数的2倍
|
||||
int maxThreads = Math.min(threadCount, Runtime.getRuntime().availableProcessors() * 2);
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxThreads);
|
||||
|
||||
List<Future<List<EmisTmsSiteBatchMiss>>> futures = new ArrayList<>();
|
||||
|
||||
try {
|
||||
// 为每个线程分配查询任务
|
||||
for (int i = 0; i < threadCount; i++) {
|
||||
int start = i * RECORDS_PER_THREAD;
|
||||
int end = Math.min(start + RECORDS_PER_THREAD, pageSize);
|
||||
|
||||
// 创建查询任务
|
||||
Callable<List<EmisTmsSiteBatchMiss>> task = () -> {
|
||||
try {
|
||||
// 创建查询参数的副本，避免线程间冲突
|
||||
EmisTmsSiteBatchMiss queryParam = createQueryParamCopy(emisTmsSiteBatchMiss);
|
||||
|
||||
// 设置分页参数
|
||||
PageHelper.startPage(1, end - start);
|
||||
|
||||
// 执行查询
|
||||
List<EmisTmsSiteBatchMiss> result = emisTmsSiteBatchMissMapper
|
||||
.selectEmisTmsSiteBatchMissList(queryParam);
|
||||
|
||||
// 只返回当前线程负责的数据范围
|
||||
if (result.size() > start) {
|
||||
int actualEnd = Math.min(start + (end - start), result.size());
|
||||
return result.subList(start, actualEnd);
|
||||
}
|
||||
|
||||
return new ArrayList<>();
|
||||
} catch (Exception e) {
|
||||
log.error("Error in thread query task", e);
|
||||
return new ArrayList<>();
|
||||
}
|
||||
};
|
||||
|
||||
futures.add(executorService.submit(task));
|
||||
}
|
||||
|
||||
// 收集所有线程的查询结果
|
||||
List<EmisTmsSiteBatchMiss> allResults = new ArrayList<>();
|
||||
for (Future<List<EmisTmsSiteBatchMiss>> future : futures) {
|
||||
try {
|
||||
List<EmisTmsSiteBatchMiss> threadResult = future.get();
|
||||
allResults.addAll(threadResult);
|
||||
} catch (Exception e) {
|
||||
log.error("Error getting thread result", e);
|
||||
}
|
||||
}
|
||||
|
||||
// 根据pageNum和pageSize进行最终分页
|
||||
int startIndex = (pageNum - 1) * pageSize;
|
||||
int endIndex = Math.min(startIndex + pageSize, allResults.size());
|
||||
|
||||
if (startIndex >= allResults.size()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
return allResults.subList(startIndex, endIndex);
|
||||
|
||||
} finally {
|
||||
executorService.shutdown();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error in multi-thread query, fallback to normal query", e);
|
||||
// 如果多线程查询失败，回退到普通查询
|
||||
return performNormalQuery(emisTmsSiteBatchMiss);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行普通查询（多线程查询失败时的回退方案）
|
||||
*/
|
||||
private List<EmisTmsSiteBatchMiss> performNormalQuery(EmisTmsSiteBatchMiss emisTmsSiteBatchMiss) {
|
||||
log.info("Performing normal query as fallback");
|
||||
|
||||
List<EmisTmsSiteBatchMiss> list = null;
|
||||
// 如果单号不为空，则去除其他参数
|
||||
if (!StringUtils.isEmpty(emisTmsSiteBatchMiss.getBillCode())) {
|
||||
emisTmsSiteBatchMiss.setBillCode(WaybillHelper.formatQueryValue(emisTmsSiteBatchMiss.getBillCode()));
|
||||
String[] billCodeSortList = emisTmsSiteBatchMiss.getBillCode().split(",");
|
||||
if (billCodeSortList.length > 1) {
|
||||
emisTmsSiteBatchMiss.getParams().put("billCodeSortList", Arrays.asList(billCodeSortList));
|
||||
}
|
||||
list = emisTmsSiteBatchMissMapper.selectEmisTmsSiteBatchMissList(emisTmsSiteBatchMiss);
|
||||
} else {
|
||||
list = emisTmsSiteBatchMissMapper.selectEmisTmsSiteBatchMissList(emisTmsSiteBatchMiss);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建查询参数的深拷贝，避免线程间冲突
|
||||
*/
|
||||
private EmisTmsSiteBatchMiss createQueryParamCopy(EmisTmsSiteBatchMiss original) {
|
||||
EmisTmsSiteBatchMiss copy = new EmisTmsSiteBatchMiss();
|
||||
|
||||
// 复制基本属性
|
||||
copy.setId(original.getId());
|
||||
copy.setBillCode(original.getBillCode());
|
||||
copy.setManager(original.getManager());
|
||||
copy.setTransManager(original.getTransManager());
|
||||
copy.setSendSiteCode(original.getSendSiteCode());
|
||||
copy.setSendSiteName(original.getSendSiteName());
|
||||
copy.setSendCountry(original.getSendCountry());
|
||||
copy.setSendCountryName(original.getSendCountryName());
|
||||
copy.setDestinationCode(original.getDestinationCode());
|
||||
copy.setDestinationName(original.getDestinationName());
|
||||
copy.setReceiveCountry(original.getReceiveCountry());
|
||||
copy.setReceiveCountryName(original.getReceiveCountryName());
|
||||
copy.setSendDate(original.getSendDate());
|
||||
copy.setBlSign(original.getBlSign());
|
||||
copy.setSignDate(original.getSignDate());
|
||||
copy.setPaymentType(original.getPaymentType());
|
||||
copy.setParcelQty(original.getParcelQty());
|
||||
copy.setBillWeight(original.getBillWeight());
|
||||
copy.setTotalVolume(original.getTotalVolume());
|
||||
copy.setVolumeWeight(original.getVolumeWeight());
|
||||
copy.setSettlementWeight(original.getSettlementWeight());
|
||||
copy.setFreight(original.getFreight());
|
||||
copy.setWaybillRemark(original.getWaybillRemark());
|
||||
copy.setFeeRemark(original.getFeeRemark());
|
||||
copy.setMissReason(original.getMissReason());
|
||||
copy.setLineCode(original.getLineCode());
|
||||
copy.setLineName(original.getLineName());
|
||||
copy.setProductType(original.getProductType());
|
||||
copy.setProductTypeName(original.getProductTypeName());
|
||||
copy.setStartSiteCode(original.getStartSiteCode());
|
||||
copy.setNextSiteCode(original.getNextSiteCode());
|
||||
copy.setStartSiteName(original.getStartSiteName());
|
||||
copy.setNextSiteName(original.getNextSiteName());
|
||||
copy.setSupplierCode(original.getSupplierCode());
|
||||
copy.setSupplierName(original.getSupplierName());
|
||||
copy.setStartDate(original.getStartDate());
|
||||
copy.setEndDate(original.getEndDate());
|
||||
copy.setMissCount(original.getMissCount());
|
||||
copy.setMissType(original.getMissType());
|
||||
copy.setBlIsQuestion(original.getBlIsQuestion());
|
||||
copy.setVirtualRemark(original.getVirtualRemark());
|
||||
copy.setProblemType(original.getProblemType());
|
||||
copy.setDispatchUnderlingSiteCode(original.getDispatchUnderlingSiteCode());
|
||||
copy.setDispatchUnderlingSiteName(original.getDispatchUnderlingSiteName());
|
||||
copy.setManualRemark(original.getManualRemark());
|
||||
copy.setBlManual(original.getBlManual());
|
||||
copy.setPayee(original.getPayee());
|
||||
copy.setSalesmen(original.getSalesmen());
|
||||
copy.setRegisterManName(original.getRegisterManName());
|
||||
|
||||
// 复制params参数
|
||||
if (original.getParams() != null) {
|
||||
copy.setParams(new HashMap<>(original.getParams()));
|
||||
}
|
||||
|
||||
return copy;
|
||||
}
|
||||
|
||||
/**
|
||||
* 新增TMS漏组批次信息
|
||||
*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user