Skip to content

A
aidso-data
Commit b3a35136 authored by Yaowentong's avatar Yaowentong
Browse files
Options

宝洁品牌映射 

电商spuname修复
parent a58e2b76
Show whitespace changes
Inline Side-by-side
Showing with 538 additions and 120 deletions
aidso_geo/config/base_config.py
View file @ b3a35136
......@@ -280,7 +280,7 @@ if __name__ == '__main__':
# mt:snipaste_v3:only_content
init_redis_4 = init_redis4()
# print(init_redis_4.delete('mt:snipaste_v3:only_content'))
print(init_redis_4.llen('mt:snipaste_v3:only_content'))
print(init_redis_4.delete('image_video_ocr2'))
# old_count, new_count = deduplicate_redis_list(
# redis_client8,
# "geo:task_commit:list",
......
aidso_geo/core/down_load_bot.py
View file @ b3a35136
......@@ -1357,5 +1357,18 @@ def qian_report2(req_ids,plan_name):
# 使用示例
# =========================
if __name__ == "__main__":
req_list = ['05e40120-018e-430d-9fb7-c921bde61920','09b179f5-8de3-4e10-a2f4-576e7e1af661','3716b2e7-41f3-45bb-beb2-1329e6c44670','585e35c9-2783-4706-87f5-469085349a3f','eccc4af3-2320-4fee-a415-2c6d26a6b5e8','0320b50f-ebfc-42d5-9050-01071b5205bd','75f9fc76-c27f-4573-8954-cea9e915b618','72e688fa-2d73-438e-81aa-1460290f6a82','cb18a066-8c88-4821-9d22-9e517332cf05','d8b3a651-abec-450f-8052-72e3d17a3503','b0ffffa2-5900-46ba-a049-ead25a895d96','1cc4a4a1-2923-47bb-b392-f4ffdde112c7','e352fd0e-7f70-4e88-9034-547c39267d61','440b52a2-beba-4c84-9cce-c09c7998d14c','91f485f3-1a3d-4ed5-a733-8197254f3f3a','c8990254-ba68-4790-bea9-ba40bf1c9774','24bf1e91-eb2f-4e4a-96fe-d1d91fda7889','90054909-aa64-444e-b935-d59044f9407d','0e5b86ba-ac03-4179-827f-696ff6fba15f','1ef447ef-aa86-44ec-b4fd-4cd20834edd9','4e09725d-228f-4bd7-9547-b484cbd9efdd','3a6e128e-7c77-4fdf-b0f4-0881fe6dfeee','d0fba64d-c2ab-438a-b5f2-47fb7eaffcd1','49288de5-9cf8-4cfe-8c7f-d81c8909ca35','e50d3df5-395b-4c26-a885-66088ba1c8a5','ed79787c-78be-41df-866d-777b29d0fe67','be59190a-452d-4e58-873b-a65dbfd87945','ad827a32-9c04-4b1f-b314-a4f3520f6cd9','91cb4eed-e8d9-4666-975b-6e079d279c72','19781ed3-fee9-4848-b936-0dc1517682f5','c6b962a9-6851-4238-83c6-4be6fc09d299','38f0eb5b-ed46-4ab9-8d57-3b6f5ebee2a3','55769d35-310f-4d81-8685-dbecd928e14a','cdc316ad-c8bb-407b-8587-e8f01deb306c','821a68df-7f6c-491d-9dc9-5b550e307aaf','dc1ee718-6e8c-4a99-8bda-aaa518523705','0ea1f698-d9fb-4085-9334-f72226566a70']
qian_report2(req_list,'养生酒')
\ No newline at end of file
phone = '15100000026'
begin = '2026-08-05'
end = '2026-08-05'
plan_name = '豆包PC/APP_快速_0805'
req_list = get_req_id(phone, begin, end)
req_ids = []
req_time_map = {}
for item in req_list:
if item.get('plan_name') == plan_name:
req_id = item.get("req_id")
created_at = item.get("created_at")
req_ids.append(req_id)
req_time_map[req_id] = created_at
print(req_ids)
\ No newline at end of file
aidso_geo/core/feishu_snipaste_v4.py
View file @ b3a35136
......@@ -851,15 +851,16 @@ def doubao_process_original_data(file_path, original_content):
except (IndexError, json.JSONDecodeError):
continue
if json_content.get('query_list'):
extra = json_content.get('ack_client_meta').get('conversation_info').get('extra')
if json_content.get('ack_client_meta').get('conversation_info').get('extra'):
extra_json = json.loads(extra)
inner_user_ip = extra_json.get('inner_user_ip')
inner_pc_version = extra_json.get('inner_pc_version')
if inner_user_ip:
tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
if inner_pc_version:
tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
...
# extra = json_content.get('ack_client_meta').get('conversation_info').get('extra')
# if json_content.get('ack_client_meta').get('conversation_info').get('extra'):
# extra_json = json.loads(extra)
# inner_user_ip = extra_json.get('inner_user_ip')
# inner_pc_version = extra_json.get('inner_pc_version')
# if inner_user_ip:
# tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
# if inner_pc_version:
# tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
if json_content.get('event_type') == 2001:
even_data = json.loads(json_content.get('event_data'))
......@@ -936,7 +937,7 @@ def doubao_process_original_data(file_path, original_content):
index = next(
(
item["text_card"]["index"]
for item in url_list
for item in (url_list or [])
if item.get("text_card", {}).get("title") == target_title
),
None
......@@ -3511,7 +3512,7 @@ def run_daily_pipeline_safely(
if __name__ == "__main__":
# platform = "DB"
# try:
# run_daily_pipeline(pt="20260811", platform="DB")
# run_daily_pipeline(pt="20260813", platform="DB")
# except Exception:
# logger.exception(
# f"[每日任务 platform={platform} count=1-99] "
......
aidso_geo/models/eco_data_process.py
View file @ b3a35136
......@@ -6,17 +6,14 @@ from aidso_geo.utils import bh_utils,ai_utils
def process_eco_product_relation(
eco_result,
keyword,
batch_size=500,
batch_size=100,
):
"""
每100条商品调用一次AI，并将识别结果合并到原商品数据中。
produce_name只用于匹配，不写入最终结果。
"""
每batch_size条商品调用一次AI，并将识别结果合并到原商品数据中。
def normalize_title(value):
return " ".join(
str(value or "").strip().split()
)
row_id只用于关联AI结果，不写入最终数据库。
produce_name只用于传给AI和调试，不再用于匹配。
"""
final_result = []
......@@ -29,10 +26,18 @@ def process_eco_product_relation(
start_index:start_index + batch_size
]
# 给每条商品增加临时row_id。
# 使用全局位置，保证整个任务内不重复。
product_list = [
str(item.get("eco_title") or "").strip()
for item in batch
{
"row_id": start_index + batch_index,
"produce_name": str(
item.get("eco_title") or ""
).strip(),
}
for batch_index, item in enumerate(batch)
]
try:
ai_result = (
ai_utils.ai_get_product_relation_spu(
......@@ -51,38 +56,56 @@ def process_eco_product_relation(
if not isinstance(ai_result, list):
ai_result = []
# produce_name只用于建立匹配关系。
expected_row_ids = {
product["row_id"]
for product in product_list
}
# 使用row_id建立AI结果映射。
ai_result_map = {}
for ai_item in ai_result:
if not isinstance(ai_item, dict):
continue
produce_name = str(
ai_item.get("produce_name") or ""
).strip()
raw_row_id = ai_item.get("row_id")
normalized_name = normalize_title(
produce_name
)
if not normalized_name:
# 兼容AI返回数字字符串的情况。
if isinstance(raw_row_id, bool):
continue
if normalized_name not in ai_result_map:
ai_result_map[normalized_name] = ai_item
try:
row_id = int(raw_row_id)
except (TypeError, ValueError):
continue
# 忽略本批次不存在的row_id。
if row_id not in expected_row_ids:
continue
# 如果AI重复返回同一个row_id，保留第一条。
if row_id not in ai_result_map:
ai_result_map[row_id] = ai_item
returned_row_ids = set(ai_result_map.keys())
missing_row_ids = (
expected_row_ids - returned_row_ids
)
for batch_index, original_item in enumerate(batch):
row_id = start_index + batch_index
for original_item in batch:
eco_title = str(
original_item.get("eco_title") or ""
).strip()
matched_item = ai_result_map.get(
normalize_title(eco_title)
)
matched_item = ai_result_map.get(row_id)
# 未匹配时的默认值。
# 没有匹配到row_id时使用默认值。
relation_data = {
"brand": eco_title,
"search_goods_word": keyword,
"spu_name": "",
"current": 0,
}
......@@ -93,42 +116,60 @@ def process_eco_product_relation(
or eco_title
).strip()
spu_name = str(
ai_spu_name = str(
matched_item.get("spu_name")
or ""
).strip()
try:
current = int(
matched_item.get("current", 0)
matched_item.get(
"current",
0,
)
)
except (TypeError, ValueError):
current = 0
if brand == spu_name:
# 生成“品牌 + SPU名称”，同时避免重复品牌。
if not ai_spu_name:
spu_name = brand
elif ai_spu_name.startswith(brand):
spu_name = ai_spu_name
else:
spu_name = brand+" "+spu_name
spu_name = (
f"{brand} {ai_spu_name}"
).strip()
relation_data = {
"brand": brand,
"search_goods_word":keyword,
"search_goods_word": keyword,
"spu_name": spu_name,
"current": (
1 if current == 1 else 0
),
}
final_result.append(
{
final_result.append({
**original_item,
**relation_data,
}
)
print(final_result)
})
# 保留你当前的状态处理方式。
for item in final_result:
item["isCalculateSearchWord"] = 1
bh_utils.insert_data('geo_eco_data', final_result)
insert_success = bh_utils.insert_data(
"geo_eco_data",
final_result,
)
if not insert_success:
print(
"商品关系结果写入失败: "
f"rows={len(final_result)}"
)
return final_result
......@@ -398,9 +439,11 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
# eco_result = process_eco_product_relation(eco_result, search_goods_word)
bh_utils.insert_data('geo_eco_data',eco_result)
if __name__ == '__main__':
result = bh_utils.query_data("select * from geo_eco_data where req_id in ('0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','10640b76d87747eaa6230d6b9f594e85','10640b76d87747eaa6230d6b9f594e85','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','3bb66c2799514f52995cd2511d1e3daf','3bb66c2799514f52995cd2511d1e3daf','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b31ac2d7b69248f8bc0253c2e0d27c3b','b31ac2d7b69248f8bc0253c2e0d27c3b','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8')")
result = bh_utils.query_data("select * from geo_eco_data where req_id ='c23acb75eb114cb085f01fb744b1337f'")
eco_result = process_eco_product_relation(result, "小米洗地机")
for item in eco_result:
item["isCalculateSearchWord"] = None
# for item in eco_result:
# item["isCalculateSearchWord"] = None
print(eco_result)
bh_utils.insert_data('geo_eco_data', eco_result)
aidso_geo/models/process.py
View file @ b3a35136
......@@ -9,7 +9,8 @@ import requests
from copy import deepcopy
from datetime import datetime
from enum import Enum
import re
import unicodedata
import json
from concurrent.futures import ThreadPoolExecutor, as_completed
......@@ -49,43 +50,201 @@ def commit_task(data, process):
def get_keyword_ranks(text, keywords):
"""
统计关键词在正文中的出现次数和出现顺序。
规则：
1. 正文和关键词统一标准化后再匹配。
2. 同一关键词的不同字符格式只计算一次。
3. 正文中真实出现多次，仍按实际次数计算。
4. 返回的 word 保留配置中第一次出现的原始格式。
"""
if isinstance(keywords, str):
keywords = [keywords]
if not isinstance(keywords, (list, tuple, set)):
return []
normalized_text = normalize_for_match(text)
# 保存去重后的原始关键词及标准化关键词
original_keywords = []
normalized_keywords = []
normalized_keyword_set = set()
for keyword in keywords:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
normalized_keyword = normalize_for_match(
original_keyword
)
if not normalized_keyword:
continue
# 标准化后相同的关键词只保留第一个
if normalized_keyword in normalized_keyword_set:
continue
normalized_keyword_set.add(normalized_keyword)
normalized_keywords.append(normalized_keyword)
original_keywords.append({
"word": original_keyword,
"normalized_word": normalized_keyword,
})
# 正文为空时，返回去重后的关键词，排名为空
if not normalized_text:
return [
{
"word": item["word"],
"rank_list": [],
}
for item in original_keywords
]
matches = []
for key in keywords:
# 每个标准化关键词只扫描一次
for normalized_keyword in normalized_keywords:
start = 0
while True:
pos = text.find(key, start)
pos = normalized_text.find(
normalized_keyword,
start,
)
if pos == -1:
break
matches.append({
"word": key,
"pos": pos
"normalized_word": normalized_keyword,
"pos": pos,
"length": len(normalized_keyword),
})
# 允许继续查找后面的相同关键词
# 继续查找正文中后续出现的相同关键词
start = pos + 1
# 按出现位置排序；同位置时短词优先
matches.sort(key=lambda x: (x["pos"], len(x["word"])))
# 按正文出现位置排序；同位置时短词优先
matches.sort(
key=lambda item: (
item["pos"],
item["length"],
)
)
# 生成每个标准化关键词对应的排名列表
rank_map = {}
for rank, item in enumerate(matches, start=1):
normalized_word = item["normalized_word"]
rank_map.setdefault(
normalized_word,
[],
).append(rank)
result = []
for key in keywords:
rank_list = [
i + 1
for i, item in enumerate(matches)
if item["word"] == key
]
# 返回第一次出现的原始关键词格式
for item in original_keywords:
result.append({
"word": key,
"rank_list": rank_list
"word": item["word"],
"rank_list": rank_map.get(
item["normalized_word"],
[],
),
})
return result
def normalize_for_match(value):
"""
将正文和关键词转换成统一格式，仅用于匹配。
不修改原始正文，不修改最终返回的原始关键词。
"""
if value is None:
return ""
if isinstance(value, bytes):
value = value.decode(
"utf-8",
errors="replace",
)
# 统一全角、半角及其他 Unicode 兼容字符
text = unicodedata.normalize(
"NFKC",
str(value),
)
normalized_chars = []
for char in text:
category = unicodedata.category(char)
# 删除零宽空格、BOM、方向控制符等不可见格式字符
if category == "Cf":
continue
# 统一所有 Unicode 横线
# Pd 包括 ‐、‑、‒、–、—、― 等
if category == "Pd" or char == "−":
normalized_chars.append("-")
continue
# 统一换行、制表符、全角空格、不换行空格等
if char.isspace() or category.startswith("Z"):
normalized_chars.append(" ")
continue
# 统一各种单引号
if char in {
"‘",
"’",
"‚",
"‛",
"`",
"´",
}:
normalized_chars.append("'")
continue
# 统一各种双引号
if char in {
"“",
"”",
"„",
"‟",
}:
normalized_chars.append('"')
continue
normalized_chars.append(char)
text = "".join(normalized_chars)
# 把连续空格、换行、制表符合并为一个普通空格
text = re.sub(
r"\s+",
" ",
text,
).strip()
# 比 lower() 更完整，可以统一更多 Unicode 大小写形式
text = text.casefold()
return text
def calculate_rank_min_and_count_sum(data_list):
positive_ranks = [item['rank'] for item in data_list if item['rank'] > 0]
......@@ -124,30 +283,155 @@ def process_com_map(data):
def keyword_map_brand(keyword_list):
def sql_quote(s: str) -> str:
# 最基础的单引号转义
return "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'"
"""
查询关键词对应的品牌。
规则：
1. 查询时忽略英文大小写。
2. 输入中大小写不同的相同关键词只查询一次。
3. 数据库存在多个品牌映射时，使用 any() 任取一个。
4. 返回的 keyword 保留本次传入的原始格式。
5. 数据库不存在映射时才调用 AI，并写入映射表。
"""
if not isinstance(keyword_list, (list, tuple, set)):
return []
# normalized keyword -> 本次输入的原始 keyword
original_keyword_map = {}
for keyword in keyword_list:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
# 与数据库 lowerUTF8() 的处理逻辑保持一致
normalized_keyword = original_keyword.lower()
# 大小写不同的同一个词只保留第一次出现的格式
if normalized_keyword not in original_keyword_map:
original_keyword_map[
normalized_keyword
] = original_keyword
if not original_keyword_map:
return []
normalized_keywords = list(
original_keyword_map.keys()
)
placeholders = ", ".join(
["%s"] * len(normalized_keywords)
)
brand_map = []
in_list = ",".join(sql_quote(x) for x in keyword_list)
query_result = []
if in_list:
query_result = bh_utils.query_data(
f"select brand,keyword from geo_keyword_brand_map where keyword in ({in_list}) group by keyword") or []
f"""
SELECT
lowerUTF8(keyword) AS normalized_keyword,
any(brand) AS brand
FROM geo_keyword_brand_map
WHERE lowerUTF8(keyword) IN ({placeholders})
GROUP BY normalized_keyword
""",
tuple(normalized_keywords),
) or []
# normalized keyword -> brand
database_brand_map = {}
for row in query_result:
normalized_keyword = str(
row.get("normalized_keyword") or ""
).strip().lower()
if not normalized_keyword:
continue
found_keywords = {row.get("keyword") for row in (query_result or []) if row.get("keyword")}
database_brand_map[normalized_keyword] = (
row.get("brand") or "没有品牌"
)
brand_map = []
# 数据库查到的映射重新绑定到本次输入的原始关键词
for normalized_keyword, original_keyword in (
original_keyword_map.items()
):
if normalized_keyword not in database_brand_map:
continue
brand_map.append({
"keyword": original_keyword,
"brand": database_brand_map[
normalized_keyword
],
})
# 只有数据库中不存在的关键词才调用 AI
missing_keywords = [
original_keyword
for normalized_keyword, original_keyword
in original_keyword_map.items()
if normalized_keyword not in database_brand_map
]
missing_keywords = [kw for kw in keyword_list if kw not in found_keywords]
if missing_keywords:
ai_brand = ai_utils.ai_get_brand_name(missing_keywords) or []
if ai_brand:
bh_utils.insert_data("geo_keyword_brand_map", ai_brand)
brand_map.extend(ai_brand)
ai_brand_list = (
ai_utils.ai_get_brand_name(
missing_keywords
)
or []
)
brand_map.extend(query_result)
# AI 返回结果按照忽略大小写方式关联
ai_brand_map = {}
return brand_map
for item in ai_brand_list:
if not isinstance(item, dict):
continue
ai_keyword = str(
item.get("keyword") or ""
).strip()
if not ai_keyword:
continue
ai_brand_map[ai_keyword.lower()] = (
item.get("brand") or "没有品牌"
)
insert_items = []
for original_keyword in missing_keywords:
normalized_keyword = (
original_keyword.lower()
)
if normalized_keyword not in ai_brand_map:
continue
item = {
"keyword": original_keyword,
"brand": ai_brand_map[
normalized_keyword
],
}
insert_items.append(item)
brand_map.append(item)
if insert_items:
bh_utils.insert_data(
"geo_keyword_brand_map",
insert_items,
)
return brand_map
def extract_keywords(data):
if not data:
......@@ -184,6 +468,60 @@ def convert_rank_data(data):
return result
def filter_keywords_in_text(response_content, keyword_list):
"""
只保留真实出现在正文中的关键词。
规则：
1. 使用 normalize_for_match() 标准化正文和关键词。
2. 不区分英文大小写、横线类型、全角半角等。
3. 标准化后相同的关键词只保留第一个。
4. 返回关键词的原始格式。
"""
if not response_content:
return []
if not isinstance(keyword_list, (list, tuple, set)):
return []
normalized_content = normalize_for_match(
response_content
)
if not normalized_content:
return []
result = []
seen_keywords = set()
for keyword in keyword_list:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
normalized_keyword = normalize_for_match(
original_keyword
)
if not normalized_keyword:
continue
# 同一个词的不同字符形式只保留一次
if normalized_keyword in seen_keywords:
continue
# AI 提取词没有真实出现在正文中，直接过滤
if normalized_keyword not in normalized_content:
continue
seen_keywords.add(normalized_keyword)
result.append(original_keyword)
return result
def cache_get_ai_brand_list(taskId, platform, response_content, prompt):
result = bh_utils.query_data(
......@@ -197,16 +535,25 @@ def cache_get_ai_brand_list(taskId, platform, response_content, prompt):
if ai_brand_list:
if isinstance(ai_brand_list, str):
try:
return json.loads(ai_brand_list)
ai_brand_list = json.loads(ai_brand_list)
except Exception:
return ai_brand_list
return ai_brand_list
ai_brand_list = []
filtered_brand_list = filter_keywords_in_text(
response_content=response_content,
keyword_list=ai_brand_list,
)
return filtered_brand_list
ai_result = ai_utils.ai_get_brand_list(response_content, prompt)
bh_utils.insert_data("geo_ai_rank_list", [{"taskId": taskId, "platform": platform, "ai_brand_list": ai_result}])
filtered_ai_result = filter_keywords_in_text(
response_content=response_content,
keyword_list=ai_result,
)
bh_utils.insert_data("geo_ai_rank_list", [{"taskId": taskId, "platform": platform, "ai_brand_list": filtered_ai_result}])
return ai_result
return filtered_ai_result
def cache_get_ai_product_list(taskId, platform, response_content, prompt):
......@@ -1703,6 +2050,7 @@ def platform_process(data):
thinking_enabled = data.get('thinkingEnabled', '0')
reqId = data.get('reqId')
context_path = f'geo/{data["taskId"]}/{platform}/context.txt'
quto_path = f'geo/{data["taskId"]}/{platform}/quote.txt'
response_content = None
url_list= None
# -------------
......@@ -1711,17 +2059,25 @@ def platform_process(data):
if not channel:
if check_file_in_tos(context_path):
response_content = tos_utils.get_string_from_tos(context_path)
url_list = tos_utils.get_string_from_tos(quto_path)
elif process_func:
file_path, search_keyword, url_list, think_content, response_content, suggestions = process_func(data)
# -------------
if response_content:
result_v2(response_content, data)
if str(thinking_enabled) == '1' and platform =='DB' and not url_list:
redis_client.lpush("DB:geo:api:list", json.dumps(data))
if (str(thinking_enabled) == "1" and platform == "DB" and not url_list):
commit_task(data, "ING")
redis_client.lpush(
"DB:geo:api:list",
json.dumps(data)
)
logger.success(f"{reqId} assistant 处理")
return
else:
return True
if not response_content:
scheduler(data)
return True
else:
_, _, _, _, response_content, _ = process_func(data)
......@@ -2016,19 +2372,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
if __name__ == '__main__':
# req_id_sql = ",".join([f"'{req_id}'" for req_id in req_ids])
#
# data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId in ({req_id_sql}) ")
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status !='SUCCESS'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
data_list = bh_utils.query_data("select * from geo_commit_task where status='ING'")
def handle_item(i):
i["comWordsMap"] = safe_json_loads(i.get("comWordsMap"), [])
i["brandWords"] = safe_json_loads(i.get("brandWords"), [])
......@@ -2040,10 +2384,10 @@ if __name__ == '__main__':
# commit_task(i,'ING')
return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i)
return platform_process(i)
# return platform_process(i)
#
if data_list:
with ThreadPoolExecutor(max_workers=50) as executor:
with ThreadPoolExecutor(max_workers=10) as executor:
futures = [executor.submit(handle_item, i) for i in data_list]
for future in as_completed(futures):
......
aidso_geo/utils/ai_utils.py
View file @ b3a35136
......@@ -4,6 +4,8 @@ import time
import requests
import json
from loguru import logger
def ai_get_brand_list(content, prompt):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
......@@ -53,7 +55,8 @@ def ai_get_brand_list(content, prompt):
"thinking": {
"type": "disabled"
},
"temperature": 0
"temperature": 0,
"top_p": 1
})
headers = {
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
......@@ -150,23 +153,31 @@ def ai_get_product_relation_spu(product_list, keyword):
"role": "system",
"content": """
你的核心任务为：基于给定的商品列表与查询关键词，为列表内的每一件商品匹配对应信息，最终输出符合规范要求的JSON结果，执行过程需严格遵循以下规则：
1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下4个字段，各字段的取值规则明确如下：
（1）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改；
（2）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值；
（3）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；需注意同一款商品的spu_name表述必须完全统一，禁止出现同一SPU存在空格差异、格式差异的情况，例如不得同时出现“无线洗地机5”和“无线洗地机 5”这类仅空格/格式有区别的命名，需统一为规范表述；
（4）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。
1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下5个字段，各字段的取值规则明确如下：
（1）row_id：原样返回输入商品中的row_id，禁止修改、遗漏、重新编号或与其他商品交换；
（2）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改；
（3）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值；
（4）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；需注意同一款商品的spu_name表述必须完全统一，禁止出现同一SPU存在空格差异、格式差异的情况，例如不得同时出现“无线洗地机5”和“无线洗地机 5”这类仅空格/格式有区别的命名，需统一为规范表述；
（5）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。
2. 一致性校验特别要求：同批次传入的商品中，若商品标题指向的品牌名相同，brand字段的取值必须保持完全统一，禁止出现同一品牌同时标注“小米”和“xiaomi”这类中英文/不同写法混用的情况，需统一为规范名称；同批次商品的spu_name也需保持表述一致，禁止出现同一SPU同时标注“12 promax”和“12promax”这类格式不统一的情况，需统一为规范表述。
3。row_id关联要求：
（1）输出的product_words数量必须与输入的product_list数量完全一致；
（2）每个输入row_id必须在输出中出现且只能出现一次；
（3）不得输出输入中不存在的row_id；
（4）row_id只用于数据关联，不参与商品分析；
（5）输出顺序可以变化，但row_id与商品信息的对应关系不得变化。
{
"row_id": 0,
"produce_name": "郎酒 红花郎15",
"brand": "郎酒",
"spu_name": "郎酒 红花郎15",
"spu_name": "红花郎15",
"current": 1
},
}
"""
},
{
"role": "user",
"content": f"""需要处理的商品列表：{product_list} 本次查询的关键词为：{keyword}"""
"content": json.dumps({"product_list": product_list,"keyword": keyword,},ensure_ascii=False,)
}
],
"thinking": {
......@@ -188,6 +199,10 @@ def ai_get_product_relation_spu(product_list, keyword):
"items": {
"type": "object",
"properties": {
"row_id": {
"type": "integer",
"description": "原样返回输入商品的row_id",
},
"produce_name": {
"type": "string",
"description": "商品完整原始名称",
......@@ -207,6 +222,7 @@ def ai_get_product_relation_spu(product_list, keyword):
},
},
"required": [
"row_id",
"produce_name",
"brand",
"spu_name",
......@@ -255,6 +271,7 @@ def ai_get_product_relation_spu(product_list, keyword):
return product_words if isinstance(product_words, list) else []
except Exception as e:
print("AI调用失败：", repr(e))
return []
def ai_get_product_list_search(product_list):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment