From 265bcba0cc96a82435c75f08f9690ff182b01d6e Mon Sep 17 00:00:00 2001 From: wangchaoxu Date: Mon, 11 Sep 2023 21:44:07 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E7=B1=BB=E7=9B=AE=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bo/CompareAndGetNeedProcessCidsBO.java | 11 ++ .../com/ms/api/service/CategoryService.java | 7 +- .../api/service/impl/CategoryServiceImpl.java | 132 +++++++++++++++++- .../SyncShopCategoryQueueTaskService.java | 9 +- .../com/ms/dal/mapper/CategoryMapper.java | 5 + .../main/resources/mapper/CategoryMapper.xml | 27 +++- 6 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 ms-biz/src/main/java/com/ms/api/bo/CompareAndGetNeedProcessCidsBO.java diff --git a/ms-biz/src/main/java/com/ms/api/bo/CompareAndGetNeedProcessCidsBO.java b/ms-biz/src/main/java/com/ms/api/bo/CompareAndGetNeedProcessCidsBO.java new file mode 100644 index 00000000..8b9ff4fc --- /dev/null +++ b/ms-biz/src/main/java/com/ms/api/bo/CompareAndGetNeedProcessCidsBO.java @@ -0,0 +1,11 @@ +package com.ms.api.bo; + +import lombok.Data; + +import java.util.List; + +@Data +public class CompareAndGetNeedProcessCidsBO { + private List needInsertCategoryCids; + private List needUpdateCategoryCids; +} diff --git a/ms-biz/src/main/java/com/ms/api/service/CategoryService.java b/ms-biz/src/main/java/com/ms/api/service/CategoryService.java index 2ec68b84..aa1c9d55 100644 --- a/ms-biz/src/main/java/com/ms/api/service/CategoryService.java +++ b/ms-biz/src/main/java/com/ms/api/service/CategoryService.java @@ -5,6 +5,7 @@ import com.ms.dal.entity.Category; import com.ms.dal.entity.RsyncCategoryQueue; import java.util.List; +import java.util.Map; /** * @@ -29,7 +30,11 @@ public interface CategoryService { List getAllShopCategorysFromDd(Long cid); - void recordShopCategorys(Integer shopId, List shopAllCategoryList); + List recordShopCategorys(Integer shopId, List shopAllCategoryList); void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache); + + boolean buildProductCatePath(List recordCategoryIds); + + Map getAllCategoryMap(); } diff --git a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java index df3b745c..20ca511e 100644 --- a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java +++ b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java @@ -1,6 +1,8 @@ package com.ms.api.service.impl; +import cn.hutool.core.util.ObjectUtil; import com.doudian.open.api.shop_getShopCategory.data.DataItem; +import com.ms.api.bo.CompareAndGetNeedProcessCidsBO; import com.ms.api.consts.RedisKeyConst; import com.ms.api.tool.CommonTool; import com.ms.api.util.DdRequestUtil; @@ -17,10 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @@ -122,7 +121,13 @@ public class CategoryServiceImpl implements CategoryService{ } @Override - public void recordShopCategorys(Integer shopId, List shopAllCategoryList) { + public List recordShopCategorys(Integer shopId, List shopAllCategoryList) { + recordCategoryShop(shopId, shopAllCategoryList); + List recordCategoryIds = recordCategory(shopAllCategoryList); + return recordCategoryIds; + } + + private void recordCategoryShop(Integer shopId, List shopAllCategoryList) { // Map shopAllCategoryMap = CommonTool.convertListToMap(shopAllCategoryList, DataItem::getId); List existCids = categoryShopMapper.selectCategoryIdListByShopId(shopId); List realCids = shopAllCategoryList.stream().map(DataItem::getId).collect(Collectors.toList()); @@ -143,6 +148,123 @@ public class CategoryServiceImpl implements CategoryService{ } } + private List recordCategory(List shopAllCategoryList) { + List recordCategoryIds = new ArrayList<>(); + + List realCids = shopAllCategoryList.stream().map(DataItem::getId).collect(Collectors.toList()); + if (realCids.isEmpty()) { + return recordCategoryIds; + } + + Map shopAllCategoryMap = CommonTool.convertListToMap(shopAllCategoryList, DataItem::getId); + + CompareAndGetNeedProcessCidsBO compareAndGetNeedProcessCidsBO = compareAndGetNeedProcessCids(shopAllCategoryMap, realCids); + + for (Long cid : compareAndGetNeedProcessCidsBO.getNeedInsertCategoryCids()) { + try { + DataItem category = shopAllCategoryMap.get(cid); + Category obj = new Category(); + obj.setCategoryId(category.getId()); + obj.setCategoryName(category.getName()); + obj.setParentCategoryId(category.getParentId().toString()); + obj.setLevel(category.getLevel().intValue()); + obj.setHasChildren(ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1); + obj.setGmtModified(new Date()); + obj.setGmtCreate(new Date()); + categoryMapper.insertSelective(obj); + recordCategoryIds.add(category.getId()); + } catch (Exception e) { + log.error("recordCategory insert error: ", e); + } + + } + + for (Long cid : compareAndGetNeedProcessCidsBO.getNeedUpdateCategoryCids()) { + try { + DataItem category = shopAllCategoryMap.get(cid); + Category obj = new Category(); + obj.setCategoryId(category.getId()); + obj.setCategoryName(category.getName()); + obj.setParentCategoryId(category.getParentId().toString()); + obj.setLevel(category.getLevel().intValue()); + obj.setHasChildren(ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1); + obj.setGmtModified(new Date()); + categoryMapper.updateByPrimaryKeySelective(obj); + recordCategoryIds.add(category.getId()); + } catch (Exception e) { + log.error("recordCategory update error: ", e); + } + } + + return recordCategoryIds; + + } + + private CompareAndGetNeedProcessCidsBO compareAndGetNeedProcessCids(Map shopAllCategoryMap, List realCids) { + final int chunkLen = 100; + List needInsertCategoryCids = new ArrayList<>(); + List needUpdateCategoryCids = new ArrayList<>(); + for (int i = 0; i < realCids.size(); i += chunkLen) { + List tempRealCids = realCids.subList(i, Math.min(i + chunkLen, realCids.size())); + List existCategoryInfos = categoryMapper.selectByCids(tempRealCids); + List existCids = existCategoryInfos.stream().map(Category::getCategoryId).collect(Collectors.toList()); + tempRealCids.removeAll(existCids); + needInsertCategoryCids.addAll(tempRealCids); + + for (Category existCategoryInfo : existCategoryInfos) { + DataItem category = shopAllCategoryMap.get(existCategoryInfo.getCategoryId()); + Integer hasChildren = (ObjectUtil.isNotNull(category.getIsLeaf()) && category.getIsLeaf() ? 0 : 1); + if (category.getName().equals(existCategoryInfo.getCategoryName()) + && category.getParentId() == Long.valueOf(existCategoryInfo.getParentCategoryId()) + && category.getLevel() == existCategoryInfo.getLevel() + && hasChildren == existCategoryInfo.getHasChildren()) { + continue; + } + needUpdateCategoryCids.add(existCategoryInfo.getCategoryId()); + } + } + CompareAndGetNeedProcessCidsBO ret = new CompareAndGetNeedProcessCidsBO(); + ret.setNeedInsertCategoryCids(needInsertCategoryCids); + ret.setNeedUpdateCategoryCids(needUpdateCategoryCids); + return ret; + } + + @Override + public boolean buildProductCatePath(List recordCategoryIds) { + int successCnt = 0; + + Map categoryMap = getAllCategoryMap(); + +// for () + + + // todo: + + + return successCnt > 0; + } + + @Override + public Map getAllCategoryMap() { + final int limit = 100; + long startCateId = 0L; + Map retMap = new HashMap<>(); + while (true) { + List rows = categoryMapper.selectByPositionPage(startCateId, limit); + if (rows.isEmpty()) { + break; + } + for (Category row : rows) { + retMap.put(row.getCategoryId(), row); + startCateId = row.getCategoryId(); + } + if (rows.size() < limit) { + break; + } + } + return retMap; + } + @Override public void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache) { diff --git a/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java b/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java index f445807c..0ca97bc8 100644 --- a/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java +++ b/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java @@ -88,7 +88,14 @@ public class SyncShopCategoryQueueTaskService extends TaskBaseService { log.info("getAllShopCategorysFromDd"); List shopAllCategoryList = categoryService.getAllShopCategorysFromDd(0L); log.info("recordShopCategorys"); - categoryService.recordShopCategorys(shopId, shopAllCategoryList); + List recordCategoryIds = categoryService.recordShopCategorys(shopId, shopAllCategoryList); + log.info("end recordShopCategorys"); + if (!recordCategoryIds.isEmpty()) { + log.info("start buildProductCatePath"); + categoryService.buildProductCatePath(recordCategoryIds); + log.info("end buildProductCatePath"); + } + log.info("start flush ShopCategorys oss"); // oss diff --git a/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java b/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java index b00954e0..6256d1e3 100644 --- a/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java +++ b/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java @@ -3,6 +3,8 @@ package com.ms.dal.mapper; import com.ms.dal.entity.Category; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + /** * @Entity com.ms.dal.entity.Category */ @@ -21,4 +23,7 @@ public interface CategoryMapper { int updateByPrimaryKey(Category record); + List selectByCids(List categoryIds); + + List selectByPositionPage(Long startId, Integer limit); } diff --git a/ms-dal/src/main/resources/mapper/CategoryMapper.xml b/ms-dal/src/main/resources/mapper/CategoryMapper.xml index 736c7f8e..20c92daf 100644 --- a/ms-dal/src/main/resources/mapper/CategoryMapper.xml +++ b/ms-dal/src/main/resources/mapper/CategoryMapper.xml @@ -26,12 +26,29 @@ select from category - where category_id = #{categoryId,jdbcType=BIGINT} + where category_id = #{categoryId,jdbcType=BIGINT} + + + delete from category - where category_id = #{categoryId,jdbcType=BIGINT} + where category_id = #{categoryId,jdbcType=BIGINT} insert into category @@ -97,11 +114,11 @@ gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - where category_id = #{categoryId,jdbcType=BIGINT} + where category_id = #{categoryId,jdbcType=BIGINT} update category - set + set category_name = #{categoryName,jdbcType=VARCHAR}, parent_category_id = #{parentCategoryId,jdbcType=VARCHAR}, path = #{path,jdbcType=VARCHAR}, @@ -110,6 +127,6 @@ ts = #{ts,jdbcType=VARCHAR}, gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, gmt_modified = #{gmtModified,jdbcType=TIMESTAMP} - where category_id = #{categoryId,jdbcType=BIGINT} + where category_id = #{categoryId,jdbcType=BIGINT} From 1cfb737c34f207cb9aa91dea76e2085a68a06b70 Mon Sep 17 00:00:00 2001 From: wangchaoxu Date: Mon, 11 Sep 2023 21:52:06 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E7=B1=BB=E7=9B=AE=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/service/impl/CategoryServiceImpl.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java index 20ca511e..a367f9b6 100644 --- a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java +++ b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java @@ -230,12 +230,27 @@ public class CategoryServiceImpl implements CategoryService{ } @Override - public boolean buildProductCatePath(List recordCategoryIds) { + public boolean buildProductCatePath(List categoryIds) { int successCnt = 0; Map categoryMap = getAllCategoryMap(); -// for () + for (Map.Entry entry : categoryMap.entrySet()) { + Long categoryId = entry.getKey(); + Category categoryInfo = entry.getValue(); + + if (categoryInfo.getLevel() <= 1) { + continue; + } + + if (categoryIds.size() > 0 && !categoryIds.contains(categoryId)) { + continue; + } + + // todo: + + + } // todo: From d8f90bf52e1f9d4bb3ed2a1aa04587c3884544b6 Mon Sep 17 00:00:00 2001 From: wangchaoxu Date: Tue, 12 Sep 2023 02:34:17 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E7=B1=BB=E7=9B=AE=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ms/api/service/CategoryService.java | 4 +- .../api/service/impl/CategoryServiceImpl.java | 85 +++++++++++++++++-- .../SyncShopCategoryQueueTaskService.java | 16 +--- .../com/ms/dal/mapper/CategoryMapper.java | 2 + .../main/resources/mapper/CategoryMapper.xml | 9 ++ 5 files changed, 95 insertions(+), 21 deletions(-) diff --git a/ms-biz/src/main/java/com/ms/api/service/CategoryService.java b/ms-biz/src/main/java/com/ms/api/service/CategoryService.java index aa1c9d55..3b2cf4da 100644 --- a/ms-biz/src/main/java/com/ms/api/service/CategoryService.java +++ b/ms-biz/src/main/java/com/ms/api/service/CategoryService.java @@ -32,9 +32,9 @@ public interface CategoryService { List recordShopCategorys(Integer shopId, List shopAllCategoryList); - void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache); - boolean buildProductCatePath(List recordCategoryIds); Map getAllCategoryMap(); + + List getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache); } diff --git a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java index a367f9b6..2dc45380 100644 --- a/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java +++ b/ms-biz/src/main/java/com/ms/api/service/impl/CategoryServiceImpl.java @@ -1,10 +1,14 @@ package com.ms.api.service.impl; import cn.hutool.core.util.ObjectUtil; +import com.alibaba.fastjson.JSONObject; import com.doudian.open.api.shop_getShopCategory.data.DataItem; import com.ms.api.bo.CompareAndGetNeedProcessCidsBO; +import com.ms.api.common.Ret; import com.ms.api.consts.RedisKeyConst; +import com.ms.api.paas.StorageService; import com.ms.api.tool.CommonTool; +import com.ms.api.tool.MoveTool; import com.ms.api.util.DdRequestUtil; import com.ms.dal.entity.Category; import com.ms.api.service.CategoryService; @@ -33,6 +37,9 @@ public class CategoryServiceImpl implements CategoryService{ @Autowired private RedisTemplate redisTemplate; + @Autowired + private StorageService storageService; + @Autowired private RsyncCategoryQueueMapper rsyncCategoryQueueMapper; @@ -234,11 +241,13 @@ public class CategoryServiceImpl implements CategoryService{ int successCnt = 0; Map categoryMap = getAllCategoryMap(); - + log.info("start buildProductCatePath"); for (Map.Entry entry : categoryMap.entrySet()) { Long categoryId = entry.getKey(); Category categoryInfo = entry.getValue(); + // todo: 之前有用个本地文件控制同步停止，可有可无先忽略 + if (categoryInfo.getLevel() <= 1) { continue; } @@ -247,16 +256,37 @@ public class CategoryServiceImpl implements CategoryService{ continue; } - // todo: + String realPath = findCategoryRealPath(categoryMap, categoryId, ""); + if (StringUtils.isEmpty(realPath) || realPath.equals(categoryInfo.getPath())) { + continue; + } + Category updateData = new Category(); + updateData.setCategoryId(categoryId); + updateData.setPath(realPath); + updateData.setGmtModified(new Date()); + categoryMapper.updateByPrimaryKeySelective(updateData); + + successCnt++; } + log.info("end buildProductCatePath"); + return successCnt > 0; + } - // todo: + private String findCategoryRealPath(Map categoryMap, Long categoryId, String path) { + if (!categoryMap.containsKey(categoryId)) { + return path; + } + Category categoryInfo = categoryMap.get(categoryId); + path = !StringUtils.isEmpty(path) ? (categoryInfo.getCategoryName() + ">" + path) : categoryInfo.getCategoryName(); + if (!StringUtils.isEmpty(categoryInfo.getParentCategoryId()) && Long.valueOf(categoryInfo.getParentCategoryId()) != 0L) { + path = findCategoryRealPath(categoryMap, Long.valueOf(categoryInfo.getParentCategoryId()), path); + } - return successCnt > 0; + return path; } @Override @@ -281,8 +311,53 @@ public class CategoryServiceImpl implements CategoryService{ } @Override - public void getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache) { + public List getAllLeafProductCatListFromTos(Integer shopId, boolean isFlushCache) { + String tosPath = MoveTool.getAllLeafProductCatListOssPath(shopId); + + List cateListAll = null; + + try { + Ret getTosContentRet = storageService.getContent(tosPath); + if (getTosContentRet.getResult().equals("success") && !isFlushCache) { + cateListAll = JSONObject.parseArray(getTosContentRet.getData().get("data").toString(), Category.class); + if (ObjectUtil.isNotEmpty(cateListAll)) { + return cateListAll; + } + } + } catch (Exception e) { + log.error("商品类目TOS读取操作过程异常：", e); + } + + cateListAll = getAllLeafProductCatListFromDb(shopId); + + try { + storageService.uploadContent(tosPath, JSONObject.toJSONString(cateListAll)); + } catch (Exception e) { + log.error("商品类目TOS写入操作过程异常：", e); + } + + // todo: tosPath文件 加入 tos自动清理队列，目前isFlushCache用的都是true,不清理也基本没影响，用到false的时候再说，有效期1天后自动删除， + + return cateListAll; + } + + private List getAllLeafProductCatListFromDb(Integer shopId) { + List cateListAll = new ArrayList<>(); + final int stepLen = 200; + Long maxCid = 0L; + while (true) { + List cateList = categoryMapper.selectLeafByPositionPage(shopId, maxCid, stepLen); + if (cateList.isEmpty()) { + break; + } + cateListAll.addAll(cateList); + if (cateList.size() < stepLen) { + break; + } + maxCid = cateList.get(cateList.size() - 1).getCategoryId(); + } + return cateListAll; } } diff --git a/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java b/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java index 0ca97bc8..2a19a771 100644 --- a/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java +++ b/ms-biz/src/main/java/com/ms/api/task/SyncShopCategoryQueueTaskService.java @@ -96,21 +96,9 @@ public class SyncShopCategoryQueueTaskService extends TaskBaseService { log.info("end buildProductCatePath"); } - - log.info("start flush ShopCategorys oss"); - // oss + log.info("start flush ShopCategorys tos"); categoryService.getAllLeafProductCatListFromTos(shopId, true); - -// String ossPath = String.format("move_product_publish_to_pic_queue/%s/%s.txt", shopId, detailId); -// Ret ret = storageService.uploadContent(ossPath, JSON.toJSONString(data)); -// if (CommonTool.isFailRet(ret)) { -// return ret; -// } -// String ossUrl = storageService.getPubOssUrlByOssPath(ossPath); -// log.info("uploadMoveDataToOss success ossUrl: " + ossUrl); -// return CommonTool.successResult("ossUrl", ossUrl); - - log.info("end flush ShopCategorys oss"); + log.info("end flush ShopCategorys tos"); return queue; } diff --git a/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java b/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java index 6256d1e3..7063956e 100644 --- a/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java +++ b/ms-dal/src/main/java/com/ms/dal/mapper/CategoryMapper.java @@ -26,4 +26,6 @@ public interface CategoryMapper { List selectByCids(List categoryIds); List selectByPositionPage(Long startId, Integer limit); + + List selectLeafByPositionPage(Integer shopId, Long startId, Integer limit); } diff --git a/ms-dal/src/main/resources/mapper/CategoryMapper.xml b/ms-dal/src/main/resources/mapper/CategoryMapper.xml index 20c92daf..d30324e7 100644 --- a/ms-dal/src/main/resources/mapper/CategoryMapper.xml +++ b/ms-dal/src/main/resources/mapper/CategoryMapper.xml @@ -45,6 +45,15 @@ order by category_id asc limit #{limit} + delete from category