db = Zc::getDb(); $this->dd = DdFactory::getDdClient(); $this->redis = RedisExt::factory('bizCache'); $this->materialLog = Zc::getLog('material/materialLog'); } public function getMaterialQueryMaterialDetailFromDd($materialId, $accessToken) { $req = new MaterialQueryMaterialDetailRequest(); $req->setMaterialId($materialId); $ret = $this->dd->execute($req, $accessToken); $checkRet = CommonTool::checkDdApiRetStatus($ret); if (CommonTool::isFailRet($checkRet)) { return $checkRet; } return CommonTool::successResult('data', $ret['data']); } public function uploadMaterialImageSyncToDd($shopId, $filter, $accessToken) { for($i = 0; $i < 3; $i++) { $retryRet = $this->realUploadMaterialImageSyncToDd($shopId, $filter, $accessToken); if(CommonTool::isFailRet($retryRet) && ($retryRet['code'] == MaterialConst::folderIdUnExist || strpos($retryRet['reason'], '该文件夹已经被删除') !== false)) { $folderId = $this->getUploadImgFolderId($shopId, $accessToken, '', true); if ($folderId) { $filter['folderId'] = $folderId; continue; } } return $retryRet; } } public function realUploadMaterialImageSyncToDd($shopId, $filter, $accessToken) { $req = new MaterialUploadImageSyncRequest(); $url = $filter['url']; $req->setUrl($url); $folderId = $filter['folderId'] ? : 0; $req->setFolderId($folderId); $req->setMaterialName($filter['materialName']); $ret = $this->dd->execute($req, $accessToken); $checkRet = CommonTool::checkDdApiRetStatus($ret); if (CommonTool::isFailRet($checkRet)) { return $checkRet; } $materialId = $ret['data']['material_id']; $saveRet = $this->saveMaterialAuditStatusBuffer($shopId, [$materialId], [$materialId => $url]); if(CommonTool::isFailRet($saveRet)) { return $saveRet; } return CommonTool::successResult('data', $ret['data']); } public function createFolderMaterialToDd($filter, $accessToken) { $req = new MaterialCreateFolderRequest(); $req->setName($filter['name']); $parentId = $filter['parentId'] ? $filter['parentId'] : 0; $req->setParentId($parentId); $type = is_numeric($filter['type']) ? $filter['type'] : 0; $req->setType($type); $ret = $this->dd->execute($req, $accessToken); $checkRet = CommonTool::checkDdApiRetStatus($ret); if (CommonTool::isFailRet($checkRet)) { return $checkRet; } return CommonTool::successResult('data', $ret['data']); } public function getMaterialFolderInfoFromDd($filter, $accessToken) { $req = new MaterialGetFolderInfoRequest(); $pageNum = $filter['pageNum'] ? : 1; $req->setPageNum($pageNum); $pageSize = $filter['pageSize'] ? : 50; $req->setPageSize($pageSize); $req->setFolderId($filter['folderId']); $ret = $this->dd->execute($req, $accessToken); $checkRet = CommonTool::checkDdApiRetStatus($ret); if (CommonTool::isFailRet($checkRet)) { return $checkRet; } return CommonTool::successResult('data', $ret['data']); } public function uploadMoveProductImgToDd($shopId, $imgUrls, $folderId, $distinct, $accessToken) { $ret = $this->realUploadMoveProductImgToDd($shopId, $imgUrls, $folderId, $distinct, $accessToken); if (CommonTool::isFailRet($ret) && strpos($ret['reason'], '该文件夹已经被删除') !== false) { $folderId = $this->getUploadImgFolderId($shopId, $accessToken, '', true); return $this->realUploadMoveProductImgToDd($shopId, $imgUrls, $folderId, $distinct, $accessToken); } return $ret; } public function realUploadMoveProductImgToDd($shopId, $imgUrls, $folderId, $distinct, $accessToken) { $materials = []; $imgUrls = array_splice($imgUrls, 0, 50); foreach ($imgUrls as $imgUrl) { $uniqid = uniqid(); $materials[] = [ 'request_id' => $uniqid, 'folder_id' => $folderId, 'material_type' => 'photo', 'name' => $uniqid, 'url' => $imgUrl, ]; } $req = new MaterialBatchUploadImageSyncRequest(); $req->setDistinct($distinct); $req->setMaterials($materials); $ret = $this->dd->execute($req, $accessToken); $checkRet = CommonTool::checkDdApiRetStatus($ret); if (CommonTool::isFailRet($checkRet)) { return $checkRet; } $successMap = $ret['data']['success_map']; $materialIds = array_column($successMap, 'MaterialId'); $materialIdAndOriginUrlMap = array_column($successMap, 'OriginUrl', 'MaterialId'); $saveRet = $this->saveMaterialAuditStatusBuffer($shopId, $materialIds, $materialIdAndOriginUrlMap); if(CommonTool::isFailRet($saveRet)) { return $saveRet; } return CommonTool::successResult([ 'data' => $ret['data'], 'logId' => $ret['log_id'], ]); } public function getUploadImgFolderId($shopId, $accessToken, $log = '', $forceApi = false) { $retry = 0; while (true) { $folderId = $this->realGetUploadImgFolderId($shopId, $accessToken, $log, $forceApi); if ($folderId) { return $folderId; } sleep(1); $retry++; if ($retry > 3) { return ''; } } } private function realGetUploadImgFolderId($shopId, $accessToken, $log = '', $forceApi = false) { if ($forceApi == true) { $this->redis->del(RedisKeyConst::getShopMoveUploadImgFolderId($shopId)); } $folderId = $this->redis->get(RedisKeyConst::getShopMoveUploadImgFolderId($shopId)); if ($log) { $log->info('getUploadImgFolderId from redis ret :'. print_r($folderId, true)); } if ($folderId) { return $folderId; } $filter = [ 'folderId' => 0, 'pageNum' => 1, 'pageSize' => 20, ]; $ret = $this->getMaterialFolderInfoFromDd($filter, $accessToken); foreach ($ret['data']['folder_info']['child_folder'] as $childFolder) { if ($childFolder['folder_name'] == MaterialConst::getDefaultFolderName()) { $folderId = $childFolder['folder_id']; break; } } if ($log) { $log->info('getUploadImgFolderId from dd ret :'. print_r($folderId, true)); } if ($folderId) { $this->redis->set(RedisKeyConst::getShopMoveUploadImgFolderId($shopId), $folderId, 3600 * 24); return $folderId; } // 并发 $folderId = $this->redis->get(RedisKeyConst::getShopMoveUploadImgFolderId($shopId)); if ($log) { $log->info('getUploadImgFolderId from redis2 ret :'. print_r($folderId, true)); } if ($folderId) { return $folderId; } $filter = [ 'name' => MaterialConst::getDefaultFolderName(), 'parentId' => 0, ]; $ret = $this->createFolderMaterialToDd($filter, $accessToken); if ($log) { $log->info('createFolderMaterialToDd ret :'. print_r($ret, true)); } $this->redis->set(RedisKeyConst::getShopMoveUploadImgFolderId($shopId), $ret['data']['folder_id'], 3600 * 24); return $ret['data']['folder_id']; } public function getUploadImgSuccessMapAndFailUrls($data, $uploadImgs = []) { $sourceImgUrlToDdImgUrl = []; foreach ($data['success_map'] as $temp) { if(empty($temp['OriginUrl'])) { continue; } $sourceImgUrlToDdImgUrl[$temp['OriginUrl']] = $temp['ByteUrl']; } $failUrls = array_diff($uploadImgs, (array)array_keys($sourceImgUrlToDdImgUrl)); return [$sourceImgUrlToDdImgUrl, $failUrls]; } public function saveMaterialAuditStatusBuffer($shopId, $materialIds, $materialIdAndOriginUrlMap) { $oldErrorMode = $this->db->setErrorMode(ZcDb::ERROR_MODE_EXCEPTION); $transStatus = $this->db->startTransaction(); try { $materialIdMap = $this->getShopMaterialBizAuditStatusLog($shopId, ['materialIds' => $materialIds], true); foreach ($materialIds as $idx => $materialId) { if(isset($materialIdMap[$materialId])) { unset($materialIds[$idx]); } } $materialIds = array_values($materialIds); $this->addMaterialAuditStatusBuffer($shopId, $materialIds); $this->addMaterialBizAuditStatusLog($shopId, $materialIds, $materialIdAndOriginUrlMap); $this->db->commit($transStatus); $this->db->setErrorMode($oldErrorMode); } catch (Exception $e) { $this->db->rollback($transStatus); $this->db->setErrorMode($oldErrorMode); $this->materialLog->info(sprintf('Method: %s Exception: %s', __METHOD__, $e->getMessage())); return CommonTool::failResult("操作异常"); } return CommonTool::successResult(); } private function addMaterialAuditStatusBuffer($shopId, $materialIds) { $chunkMaterialIds = array_chunk($materialIds, 50); foreach ($chunkMaterialIds as $smallMaterialIds) { $insert = []; foreach ($smallMaterialIds as $smallMaterialId) { $insert[] = [ 'material_id' => $smallMaterialId, 'shop_id' => $shopId, 'status' => StatusConst::wait, 'gmt_exec' => date('Y-m-d H:i:s', time() + 120), 'gmt_create' => ZcDbEval::now(), 'gmt_modified' => ZcDbEval::now(), ]; } $this->db->insert('material_audit_status_buffer', $insert); } } private function addMaterialBizAuditStatusLog($shopId, $materialIds, $materialIdAndOriginUrlMap) { $chunkMaterialIds = array_chunk($materialIds, 50); foreach ($chunkMaterialIds as $smallMaterialIds) { $insert = []; foreach ($smallMaterialIds as $smallMaterialId) { $originUrl = $materialIdAndOriginUrlMap[$smallMaterialId]; $insert[] = [ 'shop_id' => $shopId, 'material_id' => $smallMaterialId, 'status' => StatusConst::wait, 'fail_reason' => '', 'origin_url' => $originUrl, 'byte_url' => '', 'md5_material' => '', //230209之前有存,发现作用不大,反而导致带宽高,所以不存了 'gmt_create' => ZcDbEval::now(), 'gmt_modified' => ZcDbEval::now(), ]; } $this->db->insert('material_biz_audit_status_log', $insert); } } public function getShopMaterialBizAuditStatusLog($shopId, $filter, $isList) { $filter = array_filter($filter); if (empty($filter)) { return []; } $where = []; $where[] = $this->db->prepare('and shop_id = %i', $shopId); $useIndex = ''; if($filter['materialIds']) { $where[] = $this->db->prepare('and material_id in %ls', $filter['materialIds']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'udx_material_id'); } if($filter['materialId']) { $where[] = $this->db->prepare('and material_id = %s', $filter['materialId']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'udx_material_id'); } if($filter['md5Material']) { $where[] = $this->db->prepare('and md5_material = %s', $filter['md5Material']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'idx_shop_id_md5_material'); } if($filter['md5Materials']) { $where[] = $this->db->prepare('and md5_material in %ls', $filter['md5Materials']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'idx_shop_id_md5_material'); } if($filter['originUrl']) { $where[] = $this->db->prepare('and origin_url = %s', $filter['originUrl']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'idx_shop_id_origin_url'); } if($filter['originUrls']) { $where[] = $this->db->prepare('and origin_url in %ls', $filter['originUrls']); $useIndex = $this->db->prepare("USE INDEX(`%l`)", 'idx_shop_id_origin_url'); } $where = implode(' ', $where); if($isList) { $materialList = $this->db->query('select * from material_biz_audit_status_log %l where 1 %l order by material_biz_audit_status_log_id desc', $useIndex, $where); $materialList = ZcArrayHelper::changeKeyRow($materialList, 'material_id'); return $materialList; } return $this->db->queryFirstRow('select * from material_biz_audit_status_log %l where 1 %l order by material_biz_audit_status_log_id desc', $useIndex, $where); } public function uploadImgUrlToDdV2($shopId, $sourceImgUrl, $accessToken) { $folderId = $this->getUploadImgFolderId($shopId, $accessToken); $filter = [ 'folderId' => $folderId, 'url' => $sourceImgUrl, 'materialName' => AppConst::getAppName() . CommonTool::getUuidName() ]; $uploadRet = $this->uploadMaterialImageSyncToDd($shopId, $filter, $accessToken); if (CommonTool::isSuccessRet($uploadRet)) { return CommonTool::successResult([ 'ddImgUrl' => $uploadRet['data']['byte_url'], 'materialId' => $uploadRet['data']['material_id'] ]); } else { return CommonTool::failResult($uploadRet['reason']); } } public function addMaterialBizQueue($shopId, $biz, $bidId, $requestData) { $insert = [ 'shop_id' => $shopId, 'biz' => $biz, 'biz_id' => $bidId, 'request_data' => serialize($requestData), 'status' => StatusConst::wait, 'try_times' => 0, 'gmt_exec' => date('Y-m-d H:i:s', time() + 20), 'gmt_create' => ZcDbEval::now(), 'gmt_modified' => ZcDbEval::now(), ]; $aff = $this->db->insert('material_biz_queue', $insert); if(!$aff) { return CommonTool::failResult('插入数据库失败!'); } $bizQueueId = $this->db->lastInsertId(); return CommonTool::successResult(['bizQueueId' => $bizQueueId]); } public function getUrlAndMaterialIdMapBySourceUrl($shopId, $sourceUrls) { $materialList = $this->getShopMaterialBizAuditStatusLog($shopId, ['originUrls' => $sourceUrls], true); $urlAndMaterialIdMap = []; foreach ($materialList as $material) { $materialId = $material['material_id']; $url = $material['origin_url']; if(isset($urlAndMaterialIdMap[$url])) { continue; } $urlAndMaterialIdMap[$url] = $materialId; } return $urlAndMaterialIdMap; } public function addMaterialBizToMaterial($shopId, $bizQueueId, $biz, $materialIds) { $materialIds = array_unique($materialIds); $chunkMaterialIds = array_chunk($materialIds, 30); foreach ($chunkMaterialIds as $smallMaterialIds) { $insert = []; foreach ($smallMaterialIds as $materialId) { $insert[] = [ 'material_biz_queue_id' => $bizQueueId, 'shop_id' => $shopId, 'biz' => $biz, 'material_id' => $materialId, 'gmt_create' => ZcDbEval::now(), 'gmt_modified' => ZcDbEval::now(), ]; } $this->db->insert('material_biz_to_material', $insert); } } public function saveMaterialBiz($shopId, $biz, $bidId, $materialIds, $requestData) { $oldErrorMode = $this->db->setErrorMode(ZcDb::ERROR_MODE_EXCEPTION); $transStatus = $this->db->startTransaction(); try { $ret = $this->addMaterialBizQueue($shopId, $biz, $bidId, $requestData); $this->materialLog->info("addMaterialBizQueue shopId {$shopId} bizId {$bidId} ret" . print_r($ret, true)); $bizQueueId = $ret['bizQueueId']; $this->addMaterialBizToMaterial($shopId, $bizQueueId, $biz, $materialIds); $this->db->commit($transStatus); $this->db->setErrorMode($oldErrorMode); } catch (Exception $e) { $this->db->rollback($transStatus); $this->db->setErrorMode($oldErrorMode); $this->materialLog->info(sprintf('Method: %s Exception: %s', __METHOD__, $e->getMessage())); return CommonTool::failResult("操作异常"); } return CommonTool::successResult(); } public function uploadImgsFilterExist($shopId, $imgUrls) { $imgUrls = array_unique($imgUrls); $logs = $this->db->query('select * from material_biz_audit_status_log where shop_id = %i and origin_url in %ls', $shopId, $imgUrls); foreach ($logs as $log) { if ($log['status'] == StatusConst::wait) { $key = array_search($log['origin_url'], $imgUrls); if ($key !== false) { unset($imgUrls[$key]); } continue; } else if ($log['status'] == StatusConst::success) { if ($log['gmt_create'] > date('Y-m-d H:i:s', time() - 3600 * 24)) { $key = array_search($log['origin_url'], $imgUrls); if ($key !== false) { unset($imgUrls[$key]); } continue; } else { $ret = CommonTool::getImageDataFromUrlUseProxy($log['byte_url']); if (CommonTool::isSuccessRet($ret) && !empty(strlen($ret['imageData']))) { $key = array_search($log['origin_url'], $imgUrls); if ($key !== false) { unset($imgUrls[$key]); } continue; } else { $this->db->delete('material_biz_audit_status_log', 'material_biz_audit_status_log_id = %i', $log['material_biz_audit_status_log_id']); } } } else { $this->db->delete('material_biz_audit_status_log', 'material_biz_audit_status_log_id = %i', $log['material_biz_audit_status_log_id']); } } return $imgUrls; } public function addMaterialBizMoveConfigImageTask($shopId, $moveShopSettingId, $fieldName, $extField, $needAuditUrls) { $oldErrorMode = $this->db->setErrorMode(ZcDb::ERROR_MODE_EXCEPTION); $transStatus = $this->db->startTransaction(); try { $this->db->insert('move_config_image_task', array( 'shop_id' => $shopId, 'move_shop_setting_id' => $moveShopSettingId, 'field_name' => $fieldName, 'ext_field' => $extField, 'status' => StatusConst::waitAudit, 'try_times' => 0, 'gmt_create' => ZcDbEval::now(), 'gmt_modified' => ZcDbEval::now(), )); $taskId = $this->db->lastInsertId(); $urlAndMaterialIdMap = $this->getUrlAndMaterialIdMapBySourceUrl($shopId, $needAuditUrls); $requestData = [ 'urlAndMaterialIdMap' => $urlAndMaterialIdMap, ]; $materialIds = array_values($urlAndMaterialIdMap); $ret = $this->addMaterialBizQueue($shopId, BizConst::bizMoveConfigImage, $taskId, $requestData); $bizQueueId = $ret['bizQueueId']; $this->addMaterialBizToMaterial($shopId, $bizQueueId, BizConst::bizMoveConfigImage, $materialIds); $this->db->commit($transStatus); $this->db->setErrorMode($oldErrorMode); } catch (Exception $e) { $this->db->rollback($transStatus); $this->db->setErrorMode($oldErrorMode); Zc::getLog('db_error')->error(sprintf('Method: %s Exception: %s', __METHOD__, $e->getMessage())); return CommonTool::failResult("操作异常"); } return CommonTool::successResult(); } public function updateMaterialBizQueue($queueId, $update) { $fields = array ( 'status', 'gmt_last_heartbeat', 'locked', 'try_times', 'gmt_exec', 'request_data', ); $updateData = ZcArrayHelper::filterColumns($update, $fields); if (empty($updateData)) { return false; } $updateData['gmt_modified'] = ZcDbEval::now(); $ret = $this->db->update('material_biz_queue', $updateData, 'material_biz_queue_id = %i', $queueId); return ($ret === false) ? false : true; } public function deleteMaterialBizQueueAndDelMaterialBizToMaterial($queueId) { $this->db->delete('material_biz_to_material', 'material_biz_queue_id = %i', $queueId); return $this->db->delete('material_biz_queue', 'material_biz_queue_id = %i', $queueId); } }