elf
/
honor-dd-light-ds-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of codeup.aliyun.com:64d9c5feeceb191898f636d7/honor-dd-light-ds-java

Browse Source
20230922-ljl-fixBug
ljl 1 year ago
parent a6535d8ca2 d8f90bf52e
commit 1f2f26774b
6 changed files with 283 additions and 27 deletions
Show Stats Download Patch File Download Diff File

11
ms-biz/src/main/java/com/ms/api/bo/CompareAndGetNeedProcessCidsBO.java
Unescape Escape View File

@ -0,0 +1,11 @@
package com.ms.api.bo;
import lombok.Data;
import java.util.List;
@Data
public class CompareAndGetNeedProcessCidsBO {
private List<Long> needInsertCategoryCids;
private List<Long> needUpdateCategoryCids;
}

9
ms-biz/src/main/java/com/ms/api/service/CategoryService.java
Unescape Escape View File

@ -5,6 +5,7 @@ import com.ms.dal.entity.Category;
import com.ms.dal.entity.RsyncCategoryQueue;
import java.util.List;
import java.util.Map;
/**
*
@ -29,7 +30,11 @@ public interface CategoryService {
List<DataItem> getAllShopCategorysFromDd(Long cid);
void recordShopCategorys(Integer shopId, List<DataItem> shopAllCategoryList);
List<Long> recordShopCategorys(Integer shopId, List<DataItem> shopAllCategoryList);
void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache);
boolean buildProductCatePath(List<Long> recordCategoryIds);
Map<Long, Category> getAllCategoryMap();
List<Category> getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache);
}

224
ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java
Unescape Escape View File

@ -1,8 +1,14 @@
package com.ms.api.service.impl;
import cn.hutool.core.util.ObjectUtil;
import com.alibaba.fastjson.JSONObject;
import com.doudian.open.api.shop_getShopCategory.data.DataItem;
import com.ms.api.bo.CompareAndGetNeedProcessCidsBO;
import com.ms.api.common.Ret;
import com.ms.api.consts.RedisKeyConst;
import com.ms.api.paas.StorageService;
import com.ms.api.tool.CommonTool;
import com.ms.api.tool.MoveTool;
import com.ms.api.util.DdRequestUtil;
import com.ms.dal.entity.Category;
import com.ms.api.service.CategoryService;
@ -17,10 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@ -34,6 +37,9 @@ public class CategoryServiceImpl implements CategoryService{
@Autowired
private RedisTemplate<String, String> redisTemplate;
@Autowired
private StorageService storageService;
@Autowired
private RsyncCategoryQueueMapper rsyncCategoryQueueMapper;
@ -122,7 +128,13 @@ public class CategoryServiceImpl implements CategoryService{
}
@Override
public void recordShopCategorys(Integer shopId, List<DataItem> shopAllCategoryList) {
public List<Long> recordShopCategorys(Integer shopId, List<DataItem> shopAllCategoryList) {
recordCategoryShop(shopId, shopAllCategoryList);
List<Long> recordCategoryIds = recordCategory(shopAllCategoryList);
return recordCategoryIds;
}
private void recordCategoryShop(Integer shopId, List<DataItem> shopAllCategoryList) {
// Map<Long, DataItem> shopAllCategoryMap = CommonTool.convertListToMap(shopAllCategoryList, DataItem::getId);
List<Long> existCids = categoryShopMapper.selectCategoryIdListByShopId(shopId);
List<Long> realCids = shopAllCategoryList.stream().map(DataItem::getId).collect(Collectors.toList());
@ -143,9 +155,209 @@ public class CategoryServiceImpl implements CategoryService{
}
}
private List<Long> recordCategory(List<DataItem> shopAllCategoryList) {
List<Long> recordCategoryIds = new ArrayList<>();
List<Long> realCids = shopAllCategoryList.stream().map(DataItem::getId).collect(Collectors.toList());
if (realCids.isEmpty()) {
return recordCategoryIds;
}
Map<Long, DataItem> shopAllCategoryMap = CommonTool.convertListToMap(shopAllCategoryList, DataItem::getId);
CompareAndGetNeedProcessCidsBO compareAndGetNeedProcessCidsBO = compareAndGetNeedProcessCids(shopAllCategoryMap, realCids);
for (Long cid : compareAndGetNeedProcessCidsBO.getNeedInsertCategoryCids()) {
try {
DataItem category = shopAllCategoryMap.get(cid);
Category obj = new Category();
obj.setCategoryId(category.getId());
obj.setCategoryName(category.getName());
obj.setParentCategoryId(category.getParentId().toString());
obj.setLevel(category.getLevel().intValue());
obj.setHasChildren(ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1);
obj.setGmtModified(new Date());
obj.setGmtCreate(new Date());
categoryMapper.insertSelective(obj);
recordCategoryIds.add(category.getId());
} catch (Exception e) {
log.error("recordCategory insert error: ", e);
}
}
for (Long cid : compareAndGetNeedProcessCidsBO.getNeedUpdateCategoryCids()) {
try {
DataItem category = shopAllCategoryMap.get(cid);
Category obj = new Category();
obj.setCategoryId(category.getId());
obj.setCategoryName(category.getName());
obj.setParentCategoryId(category.getParentId().toString());
obj.setLevel(category.getLevel().intValue());
obj.setHasChildren(ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1);
obj.setGmtModified(new Date());
categoryMapper.updateByPrimaryKeySelective(obj);
recordCategoryIds.add(category.getId());
} catch (Exception e) {
log.error("recordCategory update error: ", e);
}
}
return recordCategoryIds;
}
private CompareAndGetNeedProcessCidsBO compareAndGetNeedProcessCids(Map<Long, DataItem> shopAllCategoryMap, List<Long> realCids) {
final int chunkLen = 100;
List<Long> needInsertCategoryCids = new ArrayList<>();
List<Long> needUpdateCategoryCids = new ArrayList<>();
for (int i = 0; i < realCids.size(); i += chunkLen) {
List<Long> tempRealCids = realCids.subList(i, Math.min(i + chunkLen, realCids.size()));
List<Category> existCategoryInfos = categoryMapper.selectByCids(tempRealCids);
List<Long> existCids = existCategoryInfos.stream().map(Category::getCategoryId).collect(Collectors.toList());
tempRealCids.removeAll(existCids);
needInsertCategoryCids.addAll(tempRealCids);
for (Category existCategoryInfo : existCategoryInfos) {
DataItem category = shopAllCategoryMap.get(existCategoryInfo.getCategoryId());
Integer hasChildren = (ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1);
if (category.getName().equals(existCategoryInfo.getCategoryName())
&& category.getParentId() == Long.valueOf(existCategoryInfo.getParentCategoryId())
&& category.getLevel() == existCategoryInfo.getLevel()
&& hasChildren == existCategoryInfo.getHasChildren()) {
continue;
}
needUpdateCategoryCids.add(existCategoryInfo.getCategoryId());
}
}
CompareAndGetNeedProcessCidsBO ret = new CompareAndGetNeedProcessCidsBO();
ret.setNeedInsertCategoryCids(needInsertCategoryCids);
ret.setNeedUpdateCategoryCids(needUpdateCategoryCids);
return ret;
}
@Override
public boolean buildProductCatePath(List<Long> categoryIds) {
int successCnt = 0;
Map<Long, Category> categoryMap = getAllCategoryMap();
log.info("start buildProductCatePath");
for (Map.Entry<Long, Category> entry : categoryMap.entrySet()) {
Long categoryId = entry.getKey();
Category categoryInfo = entry.getValue();
// todo: 之前有用个本地文件控制同步停止，可有可无先忽略
if (categoryInfo.getLevel() <= 1) {
continue;
}
if (categoryIds.size() > 0 && !categoryIds.contains(categoryId)) {
continue;
}
String realPath = findCategoryRealPath(categoryMap, categoryId, "");
if (StringUtils.isEmpty(realPath) || realPath.equals(categoryInfo.getPath())) {
continue;
}
Category updateData = new Category();
updateData.setCategoryId(categoryId);
updateData.setPath(realPath);
updateData.setGmtModified(new Date());
categoryMapper.updateByPrimaryKeySelective(updateData);
successCnt++;
}
log.info("end buildProductCatePath");
return successCnt > 0;
}
private String findCategoryRealPath(Map<Long, Category> categoryMap, Long categoryId, String path) {
if (!categoryMap.containsKey(categoryId)) {
return path;
}
Category categoryInfo = categoryMap.get(categoryId);
path = !StringUtils.isEmpty(path) ? (categoryInfo.getCategoryName() + ">" + path) : categoryInfo.getCategoryName();
if (!StringUtils.isEmpty(categoryInfo.getParentCategoryId()) && Long.valueOf(categoryInfo.getParentCategoryId()) != 0L) {
path = findCategoryRealPath(categoryMap, Long.valueOf(categoryInfo.getParentCategoryId()), path);
}
return path;
}
@Override
public Map<Long, Category> getAllCategoryMap() {
final int limit = 100;
long startCateId = 0L;
Map<Long, Category> retMap = new HashMap<>();
while (true) {
List<Category> rows = categoryMapper.selectByPositionPage(startCateId, limit);
if (rows.isEmpty()) {
break;
}
for (Category row : rows) {
retMap.put(row.getCategoryId(), row);
startCateId = row.getCategoryId();
}
if (rows.size() < limit) {
break;
}
}
return retMap;
}
@Override
public void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache) {
public List<Category> getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache) {
String tosPath = MoveTool.getAllLeafProductCatListOssPath(shopId);
List<Category> cateListAll = null;
try {
Ret getTosContentRet = storageService.getContent(tosPath);
if (getTosContentRet.getResult().equals("success") && !isFlushCache) {
cateListAll = JSONObject.parseArray(getTosContentRet.getData().get("data").toString(), Category.class);
if (ObjectUtil.isNotEmpty(cateListAll)) {
return cateListAll;
}
}
} catch (Exception e) {
log.error("商品类目TOS读取操作过程异常：", e);
}
cateListAll = getAllLeafProductCatListFromDb(shopId);
try {
storageService.uploadContent(tosPath, JSONObject.toJSONString(cateListAll));
} catch (Exception e) {
log.error("商品类目TOS写入操作过程异常：", e);
}
// todo: tosPath文件 加入 tos自动清理队列，目前isFlushCache用的都是true,不清理也基本没影响，用到false的时候再说，有效期1天后自动删除，
return cateListAll;
}
private List<Category> getAllLeafProductCatListFromDb(Integer shopId) {
List<Category> cateListAll = new ArrayList<>();
final int stepLen = 200;
Long maxCid = 0L;
while (true) {
List<Category> cateList = categoryMapper.selectLeafByPositionPage(shopId, maxCid, stepLen);
if (cateList.isEmpty()) {
break;
}
cateListAll.addAll(cateList);
if (cateList.size() < stepLen) {
break;
}
maxCid = cateList.get(cateList.size() - 1).getCategoryId();
}
return cateListAll;
}
}

23
ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java
Unescape Escape View File

@ -88,22 +88,17 @@ public class SyncShopCategoryQueueTaskService extends TaskBaseService {
log.info("getAllShopCategorysFromDd");
List<DataItem> shopAllCategoryList = categoryService.getAllShopCategorysFromDd(0L);
log.info("recordShopCategorys");
categoryService.recordShopCategorys(shopId, shopAllCategoryList);
List<Long> recordCategoryIds = categoryService.recordShopCategorys(shopId, shopAllCategoryList);
log.info("end recordShopCategorys");
if (!recordCategoryIds.isEmpty()) {
log.info("start buildProductCatePath");
categoryService.buildProductCatePath(recordCategoryIds);
log.info("end buildProductCatePath");
}
log.info("start flush ShopCategorys oss");
// oss
log.info("start flush ShopCategorys tos");
categoryService.getAllLeafProductCatListFromTos(shopId, true);
// String ossPath = String.format("move_product_publish_to_pic_queue/%s/%s.txt", shopId, detailId);
// Ret ret = storageService.uploadContent(ossPath, JSON.toJSONString(data));
// if (CommonTool.isFailRet(ret)) {
// return ret;
// }
// String ossUrl = storageService.getPubOssUrlByOssPath(ossPath);
// log.info("uploadMoveDataToOss success ossUrl: " + ossUrl);
// return CommonTool.successResult("ossUrl", ossUrl);
log.info("end flush ShopCategorys oss");
log.info("end flush ShopCategorys tos");
return queue;
}

7
ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java
Unescape Escape View File

@ -3,6 +3,8 @@ package com.ms.dal.mapper;
import com.ms.dal.entity.Category;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
/**
* @Entity com.ms.dal.entity.Category
*/
@ -21,4 +23,9 @@ public interface CategoryMapper {
int updateByPrimaryKey(Category record);
List<Category> selectByCids(List<Long> categoryIds);
List<Category> selectByPositionPage(Long startId, Integer limit);
List<Category> selectLeafByPositionPage(Integer shopId, Long startId, Integer limit);
}

36
ms-dal/src/main/resources/mapper/CategoryMapper.xml
Unescape Escape View File

@ -26,12 +26,38 @@
select
<include refid="Base_Column_List" />
from category
where category_id = #{categoryId,jdbcType=BIGINT}
where category_id = #{categoryId,jdbcType=BIGINT}
</select>
<select id="selectByCids" resultMap="BaseResultMap">
select
<include refid="Base_Column_List" />
from category
where category_id in
<foreach collection="categoryIds" index="index" item="item" open="(" separator="," close=")">
#{item}
</foreach>
</select>
<select id="selectByPositionPage" resultMap="BaseResultMap">
select
<include refid="Base_Column_List" />
from category
where category_id > #{startId}
order by category_id asc
limit #{limit}
</select>
<select id="selectLeafByPositionPage" resultMap="BaseResultMap">
select c.*
from category c
left join category_shop cs on cs.category_id = c.category_id and cs.shop_id = #{shopId}
where c.category_id > #{startId}
and c.has_children = 0
order by c.category_id asc
limit #{limit}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete from category
where category_id = #{categoryId,jdbcType=BIGINT}
where category_id = #{categoryId,jdbcType=BIGINT}
</delete>
<insert id="insert" keyColumn="category_id" keyProperty="categoryId" parameterType="com.ms.dal.entity.Category" useGeneratedKeys="true">
insert into category
@ -97,11 +123,11 @@
gmt_modified = #{gmtModified,jdbcType=TIMESTAMP},
</if>
</set>
where category_id = #{categoryId,jdbcType=BIGINT}
where category_id = #{categoryId,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.ms.dal.entity.Category">
update category
set
set
category_name = #{categoryName,jdbcType=VARCHAR},
parent_category_id = #{parentCategoryId,jdbcType=VARCHAR},
path = #{path,jdbcType=VARCHAR},
@ -110,6 +136,6 @@
ts = #{ts,jdbcType=VARCHAR},
gmt_create = #{gmtCreate,jdbcType=TIMESTAMP},
gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}
where category_id = #{categoryId,jdbcType=BIGINT}
where category_id = #{categoryId,jdbcType=BIGINT}
</update>
</mapper>

Write Preview
Loading…
Cancel
Save