Skip to content

A
aidso-data
Commit b3a35136 authored by Yaowentong's avatar Yaowentong
Browse files
Options

宝洁品牌映射 

电商spuname修复
parent a58e2b76
Show whitespace changes
Inline Side-by-side
Showing with 538 additions and 120 deletions
aidso_geo/config/base_config.py
View file @ b3a35136
...@@ -280,7 +280,7 @@ if __name__ == '__main__': ...@@ -280,7 +280,7 @@ if __name__ == '__main__':
# mt:snipaste_v3:only_content # mt:snipaste_v3:only_content
init_redis_4 = init_redis4() init_redis_4 = init_redis4()
# print(init_redis_4.delete('mt:snipaste_v3:only_content')) # print(init_redis_4.delete('mt:snipaste_v3:only_content'))
print(init_redis_4.llen('mt:snipaste_v3:only_content')) print(init_redis_4.delete('image_video_ocr2'))
# old_count, new_count = deduplicate_redis_list( # old_count, new_count = deduplicate_redis_list(
# redis_client8, # redis_client8,
# "geo:task_commit:list", # "geo:task_commit:list",
......
aidso_geo/core/down_load_bot.py
View file @ b3a35136
...@@ -1357,5 +1357,18 @@ def qian_report2(req_ids,plan_name): ...@@ -1357,5 +1357,18 @@ def qian_report2(req_ids,plan_name):
# 使用示例 # 使用示例
# ========================= # =========================
if __name__ == "__main__": if __name__ == "__main__":
req_list = ['05e40120-018e-430d-9fb7-c921bde61920','09b179f5-8de3-4e10-a2f4-576e7e1af661','3716b2e7-41f3-45bb-beb2-1329e6c44670','585e35c9-2783-4706-87f5-469085349a3f','eccc4af3-2320-4fee-a415-2c6d26a6b5e8','0320b50f-ebfc-42d5-9050-01071b5205bd','75f9fc76-c27f-4573-8954-cea9e915b618','72e688fa-2d73-438e-81aa-1460290f6a82','cb18a066-8c88-4821-9d22-9e517332cf05','d8b3a651-abec-450f-8052-72e3d17a3503','b0ffffa2-5900-46ba-a049-ead25a895d96','1cc4a4a1-2923-47bb-b392-f4ffdde112c7','e352fd0e-7f70-4e88-9034-547c39267d61','440b52a2-beba-4c84-9cce-c09c7998d14c','91f485f3-1a3d-4ed5-a733-8197254f3f3a','c8990254-ba68-4790-bea9-ba40bf1c9774','24bf1e91-eb2f-4e4a-96fe-d1d91fda7889','90054909-aa64-444e-b935-d59044f9407d','0e5b86ba-ac03-4179-827f-696ff6fba15f','1ef447ef-aa86-44ec-b4fd-4cd20834edd9','4e09725d-228f-4bd7-9547-b484cbd9efdd','3a6e128e-7c77-4fdf-b0f4-0881fe6dfeee','d0fba64d-c2ab-438a-b5f2-47fb7eaffcd1','49288de5-9cf8-4cfe-8c7f-d81c8909ca35','e50d3df5-395b-4c26-a885-66088ba1c8a5','ed79787c-78be-41df-866d-777b29d0fe67','be59190a-452d-4e58-873b-a65dbfd87945','ad827a32-9c04-4b1f-b314-a4f3520f6cd9','91cb4eed-e8d9-4666-975b-6e079d279c72','19781ed3-fee9-4848-b936-0dc1517682f5','c6b962a9-6851-4238-83c6-4be6fc09d299','38f0eb5b-ed46-4ab9-8d57-3b6f5ebee2a3','55769d35-310f-4d81-8685-dbecd928e14a','cdc316ad-c8bb-407b-8587-e8f01deb306c','821a68df-7f6c-491d-9dc9-5b550e307aaf','dc1ee718-6e8c-4a99-8bda-aaa518523705','0ea1f698-d9fb-4085-9334-f72226566a70'] phone = '15100000026'
qian_report2(req_list,'养生酒') begin = '2026-08-05'
\ No newline at end of file end = '2026-08-05'
plan_name = '豆包PC/APP_快速_0805'
req_list = get_req_id(phone, begin, end)
req_ids = []
req_time_map = {}
for item in req_list:
if item.get('plan_name') == plan_name:
req_id = item.get("req_id")
created_at = item.get("created_at")
req_ids.append(req_id)
req_time_map[req_id] = created_at
print(req_ids)
\ No newline at end of file
aidso_geo/core/feishu_snipaste_v4.py
View file @ b3a35136
...@@ -851,15 +851,16 @@ def doubao_process_original_data(file_path, original_content): ...@@ -851,15 +851,16 @@ def doubao_process_original_data(file_path, original_content):
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
if json_content.get('query_list'): if json_content.get('query_list'):
extra = json_content.get('ack_client_meta').get('conversation_info').get('extra') ...
if json_content.get('ack_client_meta').get('conversation_info').get('extra'): # extra = json_content.get('ack_client_meta').get('conversation_info').get('extra')
extra_json = json.loads(extra) # if json_content.get('ack_client_meta').get('conversation_info').get('extra'):
inner_user_ip = extra_json.get('inner_user_ip') # extra_json = json.loads(extra)
inner_pc_version = extra_json.get('inner_pc_version') # inner_user_ip = extra_json.get('inner_user_ip')
if inner_user_ip: # inner_pc_version = extra_json.get('inner_pc_version')
tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip) # if inner_user_ip:
if inner_pc_version: # tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version) # if inner_pc_version:
# tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
if json_content.get('event_type') == 2001: if json_content.get('event_type') == 2001:
even_data = json.loads(json_content.get('event_data')) even_data = json.loads(json_content.get('event_data'))
...@@ -936,7 +937,7 @@ def doubao_process_original_data(file_path, original_content): ...@@ -936,7 +937,7 @@ def doubao_process_original_data(file_path, original_content):
index = next( index = next(
( (
item["text_card"]["index"] item["text_card"]["index"]
for item in url_list for item in (url_list or [])
if item.get("text_card", {}).get("title") == target_title if item.get("text_card", {}).get("title") == target_title
), ),
None None
...@@ -3511,7 +3512,7 @@ def run_daily_pipeline_safely( ...@@ -3511,7 +3512,7 @@ def run_daily_pipeline_safely(
if __name__ == "__main__": if __name__ == "__main__":
# platform = "DB" # platform = "DB"
# try: # try:
# run_daily_pipeline(pt="20260811", platform="DB") # run_daily_pipeline(pt="20260813", platform="DB")
# except Exception: # except Exception:
# logger.exception( # logger.exception(
# f"[每日任务 platform={platform} count=1-99] " # f"[每日任务 platform={platform} count=1-99] "
......
aidso_geo/models/eco_data_process.py
View file @ b3a35136
...@@ -6,17 +6,14 @@ from aidso_geo.utils import bh_utils,ai_utils ...@@ -6,17 +6,14 @@ from aidso_geo.utils import bh_utils,ai_utils
def process_eco_product_relation( def process_eco_product_relation(
eco_result, eco_result,
keyword, keyword,
batch_size=500, batch_size=100,
): ):
""" """
每100条商品调用一次AI，并将识别结果合并到原商品数据中。 每batch_size条商品调用一次AI，并将识别结果合并到原商品数据中。
produce_name只用于匹配，不写入最终结果。
"""
def normalize_title(value): row_id只用于关联AI结果，不写入最终数据库。
return " ".join( produce_name只用于传给AI和调试，不再用于匹配。
str(value or "").strip().split() """
)
final_result = [] final_result = []
...@@ -29,10 +26,18 @@ def process_eco_product_relation( ...@@ -29,10 +26,18 @@ def process_eco_product_relation(
start_index:start_index + batch_size start_index:start_index + batch_size
] ]
# 给每条商品增加临时row_id。
# 使用全局位置，保证整个任务内不重复。
product_list = [ product_list = [
str(item.get("eco_title") or "").strip() {
for item in batch "row_id": start_index + batch_index,
"produce_name": str(
item.get("eco_title") or ""
).strip(),
}
for batch_index, item in enumerate(batch)
] ]
try: try:
ai_result = ( ai_result = (
ai_utils.ai_get_product_relation_spu( ai_utils.ai_get_product_relation_spu(
...@@ -51,38 +56,56 @@ def process_eco_product_relation( ...@@ -51,38 +56,56 @@ def process_eco_product_relation(
if not isinstance(ai_result, list): if not isinstance(ai_result, list):
ai_result = [] ai_result = []
# produce_name只用于建立匹配关系。 expected_row_ids = {
product["row_id"]
for product in product_list
}
# 使用row_id建立AI结果映射。
ai_result_map = {} ai_result_map = {}
for ai_item in ai_result: for ai_item in ai_result:
if not isinstance(ai_item, dict): if not isinstance(ai_item, dict):
continue continue
produce_name = str( raw_row_id = ai_item.get("row_id")
ai_item.get("produce_name") or ""
).strip()
normalized_name = normalize_title( # 兼容AI返回数字字符串的情况。
produce_name if isinstance(raw_row_id, bool):
)
if not normalized_name:
continue continue
if normalized_name not in ai_result_map: try:
ai_result_map[normalized_name] = ai_item row_id = int(raw_row_id)
except (TypeError, ValueError):
continue
# 忽略本批次不存在的row_id。
if row_id not in expected_row_ids:
continue
# 如果AI重复返回同一个row_id，保留第一条。
if row_id not in ai_result_map:
ai_result_map[row_id] = ai_item
returned_row_ids = set(ai_result_map.keys())
missing_row_ids = (
expected_row_ids - returned_row_ids
)
for batch_index, original_item in enumerate(batch):
row_id = start_index + batch_index
for original_item in batch:
eco_title = str( eco_title = str(
original_item.get("eco_title") or "" original_item.get("eco_title") or ""
).strip() ).strip()
matched_item = ai_result_map.get( matched_item = ai_result_map.get(row_id)
normalize_title(eco_title)
)
# 未匹配时的默认值。 # 没有匹配到row_id时使用默认值。
relation_data = { relation_data = {
"brand": eco_title, "brand": eco_title,
"search_goods_word": keyword,
"spu_name": "", "spu_name": "",
"current": 0, "current": 0,
} }
...@@ -93,42 +116,60 @@ def process_eco_product_relation( ...@@ -93,42 +116,60 @@ def process_eco_product_relation(
or eco_title or eco_title
).strip() ).strip()
spu_name = str( ai_spu_name = str(
matched_item.get("spu_name") matched_item.get("spu_name")
or "" or ""
).strip() ).strip()
try: try:
current = int( current = int(
matched_item.get("current", 0) matched_item.get(
"current",
0,
)
) )
except (TypeError, ValueError): except (TypeError, ValueError):
current = 0 current = 0
if brand == spu_name: # 生成“品牌 + SPU名称”，同时避免重复品牌。
if not ai_spu_name:
spu_name = brand spu_name = brand
elif ai_spu_name.startswith(brand):
spu_name = ai_spu_name
else: else:
spu_name = brand+" "+spu_name spu_name = (
f"{brand} {ai_spu_name}"
).strip()
relation_data = { relation_data = {
"brand": brand, "brand": brand,
"search_goods_word":keyword, "search_goods_word": keyword,
"spu_name": spu_name, "spu_name": spu_name,
"current": ( "current": (
1 if current == 1 else 0 1 if current == 1 else 0
), ),
} }
final_result.append( final_result.append({
{
**original_item, **original_item,
**relation_data, **relation_data,
} })
)
print(final_result) # 保留你当前的状态处理方式。
for item in final_result: for item in final_result:
item["isCalculateSearchWord"] = 1 item["isCalculateSearchWord"] = 1
bh_utils.insert_data('geo_eco_data', final_result)
insert_success = bh_utils.insert_data(
"geo_eco_data",
final_result,
)
if not insert_success:
print(
"商品关系结果写入失败: "
f"rows={len(final_result)}"
)
return final_result return final_result
...@@ -398,9 +439,11 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -398,9 +439,11 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
# eco_result = process_eco_product_relation(eco_result, search_goods_word) # eco_result = process_eco_product_relation(eco_result, search_goods_word)
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
if __name__ == '__main__': if __name__ == '__main__':
result = bh_utils.query_data("select * from geo_eco_data where req_id in ('0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','10640b76d87747eaa6230d6b9f594e85','10640b76d87747eaa6230d6b9f594e85','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','3bb66c2799514f52995cd2511d1e3daf','3bb66c2799514f52995cd2511d1e3daf','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b31ac2d7b69248f8bc0253c2e0d27c3b','b31ac2d7b69248f8bc0253c2e0d27c3b','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8')") result = bh_utils.query_data("select * from geo_eco_data where req_id ='c23acb75eb114cb085f01fb744b1337f'")
eco_result = process_eco_product_relation(result, "小米洗地机") eco_result = process_eco_product_relation(result, "小米洗地机")
for item in eco_result: # for item in eco_result:
item["isCalculateSearchWord"] = None # item["isCalculateSearchWord"] = None
print(eco_result)
bh_utils.insert_data('geo_eco_data', eco_result) bh_utils.insert_data('geo_eco_data', eco_result)
aidso_geo/models/process.py
View file @ b3a35136
...@@ -9,7 +9,8 @@ import requests ...@@ -9,7 +9,8 @@ import requests
from copy import deepcopy from copy import deepcopy
from datetime import datetime from datetime import datetime
from enum import Enum from enum import Enum
import re
import unicodedata
import json import json
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
...@@ -49,43 +50,201 @@ def commit_task(data, process): ...@@ -49,43 +50,201 @@ def commit_task(data, process):
def get_keyword_ranks(text, keywords): def get_keyword_ranks(text, keywords):
"""
统计关键词在正文中的出现次数和出现顺序。
规则：
1. 正文和关键词统一标准化后再匹配。
2. 同一关键词的不同字符格式只计算一次。
3. 正文中真实出现多次，仍按实际次数计算。
4. 返回的 word 保留配置中第一次出现的原始格式。
"""
if isinstance(keywords, str): if isinstance(keywords, str):
keywords = [keywords] keywords = [keywords]
if not isinstance(keywords, (list, tuple, set)):
return []
normalized_text = normalize_for_match(text)
# 保存去重后的原始关键词及标准化关键词
original_keywords = []
normalized_keywords = []
normalized_keyword_set = set()
for keyword in keywords:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
normalized_keyword = normalize_for_match(
original_keyword
)
if not normalized_keyword:
continue
# 标准化后相同的关键词只保留第一个
if normalized_keyword in normalized_keyword_set:
continue
normalized_keyword_set.add(normalized_keyword)
normalized_keywords.append(normalized_keyword)
original_keywords.append({
"word": original_keyword,
"normalized_word": normalized_keyword,
})
# 正文为空时，返回去重后的关键词，排名为空
if not normalized_text:
return [
{
"word": item["word"],
"rank_list": [],
}
for item in original_keywords
]
matches = [] matches = []
for key in keywords: # 每个标准化关键词只扫描一次
for normalized_keyword in normalized_keywords:
start = 0 start = 0
while True: while True:
pos = text.find(key, start) pos = normalized_text.find(
normalized_keyword,
start,
)
if pos == -1: if pos == -1:
break break
matches.append({ matches.append({
"word": key, "normalized_word": normalized_keyword,
"pos": pos "pos": pos,
"length": len(normalized_keyword),
}) })
# 允许继续查找后面的相同关键词 # 继续查找正文中后续出现的相同关键词
start = pos + 1 start = pos + 1
# 按出现位置排序；同位置时短词优先 # 按正文出现位置排序；同位置时短词优先
matches.sort(key=lambda x: (x["pos"], len(x["word"]))) matches.sort(
key=lambda item: (
item["pos"],
item["length"],
)
)
# 生成每个标准化关键词对应的排名列表
rank_map = {}
for rank, item in enumerate(matches, start=1):
normalized_word = item["normalized_word"]
rank_map.setdefault(
normalized_word,
[],
).append(rank)
result = [] result = []
for key in keywords:
rank_list = [ # 返回第一次出现的原始关键词格式
i + 1 for item in original_keywords:
for i, item in enumerate(matches)
if item["word"] == key
]
result.append({ result.append({
"word": key, "word": item["word"],
"rank_list": rank_list "rank_list": rank_map.get(
item["normalized_word"],
[],
),
}) })
return result return result
def normalize_for_match(value):
"""
将正文和关键词转换成统一格式，仅用于匹配。
不修改原始正文，不修改最终返回的原始关键词。
"""
if value is None:
return ""
if isinstance(value, bytes):
value = value.decode(
"utf-8",
errors="replace",
)
# 统一全角、半角及其他 Unicode 兼容字符
text = unicodedata.normalize(
"NFKC",
str(value),
)
normalized_chars = []
for char in text:
category = unicodedata.category(char)
# 删除零宽空格、BOM、方向控制符等不可见格式字符
if category == "Cf":
continue
# 统一所有 Unicode 横线
# Pd 包括 ‐、‑、‒、–、—、― 等
if category == "Pd" or char == "−":
normalized_chars.append("-")
continue
# 统一换行、制表符、全角空格、不换行空格等
if char.isspace() or category.startswith("Z"):
normalized_chars.append(" ")
continue
# 统一各种单引号
if char in {
"‘",
"’",
"‚",
"‛",
"`",
"´",
}:
normalized_chars.append("'")
continue
# 统一各种双引号
if char in {
"“",
"”",
"„",
"‟",
}:
normalized_chars.append('"')
continue
normalized_chars.append(char)
text = "".join(normalized_chars)
# 把连续空格、换行、制表符合并为一个普通空格
text = re.sub(
r"\s+",
" ",
text,
).strip()
# 比 lower() 更完整，可以统一更多 Unicode 大小写形式
text = text.casefold()
return text
def calculate_rank_min_and_count_sum(data_list): def calculate_rank_min_and_count_sum(data_list):
positive_ranks = [item['rank'] for item in data_list if item['rank'] > 0] positive_ranks = [item['rank'] for item in data_list if item['rank'] > 0]
...@@ -124,30 +283,155 @@ def process_com_map(data): ...@@ -124,30 +283,155 @@ def process_com_map(data):
def keyword_map_brand(keyword_list): def keyword_map_brand(keyword_list):
def sql_quote(s: str) -> str: """
# 最基础的单引号转义 查询关键词对应的品牌。
return "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'"
规则：
1. 查询时忽略英文大小写。
2. 输入中大小写不同的相同关键词只查询一次。
3. 数据库存在多个品牌映射时，使用 any() 任取一个。
4. 返回的 keyword 保留本次传入的原始格式。
5. 数据库不存在映射时才调用 AI，并写入映射表。
"""
if not isinstance(keyword_list, (list, tuple, set)):
return []
# normalized keyword -> 本次输入的原始 keyword
original_keyword_map = {}
for keyword in keyword_list:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
# 与数据库 lowerUTF8() 的处理逻辑保持一致
normalized_keyword = original_keyword.lower()
# 大小写不同的同一个词只保留第一次出现的格式
if normalized_keyword not in original_keyword_map:
original_keyword_map[
normalized_keyword
] = original_keyword
if not original_keyword_map:
return []
normalized_keywords = list(
original_keyword_map.keys()
)
placeholders = ", ".join(
["%s"] * len(normalized_keywords)
)
brand_map = []
in_list = ",".join(sql_quote(x) for x in keyword_list)
query_result = []
if in_list:
query_result = bh_utils.query_data( query_result = bh_utils.query_data(
f"select brand,keyword from geo_keyword_brand_map where keyword in ({in_list}) group by keyword") or [] f"""
SELECT
lowerUTF8(keyword) AS normalized_keyword,
any(brand) AS brand
FROM geo_keyword_brand_map
WHERE lowerUTF8(keyword) IN ({placeholders})
GROUP BY normalized_keyword
""",
tuple(normalized_keywords),
) or []
# normalized keyword -> brand
database_brand_map = {}
for row in query_result:
normalized_keyword = str(
row.get("normalized_keyword") or ""
).strip().lower()
if not normalized_keyword:
continue
found_keywords = {row.get("keyword") for row in (query_result or []) if row.get("keyword")} database_brand_map[normalized_keyword] = (
row.get("brand") or "没有品牌"
)
brand_map = []
# 数据库查到的映射重新绑定到本次输入的原始关键词
for normalized_keyword, original_keyword in (
original_keyword_map.items()
):
if normalized_keyword not in database_brand_map:
continue
brand_map.append({
"keyword": original_keyword,
"brand": database_brand_map[
normalized_keyword
],
})
# 只有数据库中不存在的关键词才调用 AI
missing_keywords = [
original_keyword
for normalized_keyword, original_keyword
in original_keyword_map.items()
if normalized_keyword not in database_brand_map
]
missing_keywords = [kw for kw in keyword_list if kw not in found_keywords]
if missing_keywords: if missing_keywords:
ai_brand = ai_utils.ai_get_brand_name(missing_keywords) or [] ai_brand_list = (
if ai_brand: ai_utils.ai_get_brand_name(
bh_utils.insert_data("geo_keyword_brand_map", ai_brand) missing_keywords
brand_map.extend(ai_brand) )
or []
)
brand_map.extend(query_result) # AI 返回结果按照忽略大小写方式关联
ai_brand_map = {}
return brand_map for item in ai_brand_list:
if not isinstance(item, dict):
continue
ai_keyword = str(
item.get("keyword") or ""
).strip()
if not ai_keyword:
continue
ai_brand_map[ai_keyword.lower()] = (
item.get("brand") or "没有品牌"
)
insert_items = []
for original_keyword in missing_keywords:
normalized_keyword = (
original_keyword.lower()
)
if normalized_keyword not in ai_brand_map:
continue
item = {
"keyword": original_keyword,
"brand": ai_brand_map[
normalized_keyword
],
}
insert_items.append(item)
brand_map.append(item)
if insert_items:
bh_utils.insert_data(
"geo_keyword_brand_map",
insert_items,
)
return brand_map
def extract_keywords(data): def extract_keywords(data):
if not data: if not data:
...@@ -184,6 +468,60 @@ def convert_rank_data(data): ...@@ -184,6 +468,60 @@ def convert_rank_data(data):
return result return result
def filter_keywords_in_text(response_content, keyword_list):
"""
只保留真实出现在正文中的关键词。
规则：
1. 使用 normalize_for_match() 标准化正文和关键词。
2. 不区分英文大小写、横线类型、全角半角等。
3. 标准化后相同的关键词只保留第一个。
4. 返回关键词的原始格式。
"""
if not response_content:
return []
if not isinstance(keyword_list, (list, tuple, set)):
return []
normalized_content = normalize_for_match(
response_content
)
if not normalized_content:
return []
result = []
seen_keywords = set()
for keyword in keyword_list:
if keyword is None:
continue
original_keyword = str(keyword).strip()
if not original_keyword:
continue
normalized_keyword = normalize_for_match(
original_keyword
)
if not normalized_keyword:
continue
# 同一个词的不同字符形式只保留一次
if normalized_keyword in seen_keywords:
continue
# AI 提取词没有真实出现在正文中，直接过滤
if normalized_keyword not in normalized_content:
continue
seen_keywords.add(normalized_keyword)
result.append(original_keyword)
return result
def cache_get_ai_brand_list(taskId, platform, response_content, prompt): def cache_get_ai_brand_list(taskId, platform, response_content, prompt):
result = bh_utils.query_data( result = bh_utils.query_data(
...@@ -197,16 +535,25 @@ def cache_get_ai_brand_list(taskId, platform, response_content, prompt): ...@@ -197,16 +535,25 @@ def cache_get_ai_brand_list(taskId, platform, response_content, prompt):
if ai_brand_list: if ai_brand_list:
if isinstance(ai_brand_list, str): if isinstance(ai_brand_list, str):
try: try:
return json.loads(ai_brand_list) ai_brand_list = json.loads(ai_brand_list)
except Exception: except Exception:
return ai_brand_list ai_brand_list = []
return ai_brand_list filtered_brand_list = filter_keywords_in_text(
response_content=response_content,
keyword_list=ai_brand_list,
)
return filtered_brand_list
ai_result = ai_utils.ai_get_brand_list(response_content, prompt) ai_result = ai_utils.ai_get_brand_list(response_content, prompt)
bh_utils.insert_data("geo_ai_rank_list", [{"taskId": taskId, "platform": platform, "ai_brand_list": ai_result}]) filtered_ai_result = filter_keywords_in_text(
response_content=response_content,
keyword_list=ai_result,
)
bh_utils.insert_data("geo_ai_rank_list", [{"taskId": taskId, "platform": platform, "ai_brand_list": filtered_ai_result}])
return ai_result return filtered_ai_result
def cache_get_ai_product_list(taskId, platform, response_content, prompt): def cache_get_ai_product_list(taskId, platform, response_content, prompt):
...@@ -1703,6 +2050,7 @@ def platform_process(data): ...@@ -1703,6 +2050,7 @@ def platform_process(data):
thinking_enabled = data.get('thinkingEnabled', '0') thinking_enabled = data.get('thinkingEnabled', '0')
reqId = data.get('reqId') reqId = data.get('reqId')
context_path = f'geo/{data["taskId"]}/{platform}/context.txt' context_path = f'geo/{data["taskId"]}/{platform}/context.txt'
quto_path = f'geo/{data["taskId"]}/{platform}/quote.txt'
response_content = None response_content = None
url_list= None url_list= None
# ------------- # -------------
...@@ -1711,17 +2059,25 @@ def platform_process(data): ...@@ -1711,17 +2059,25 @@ def platform_process(data):
if not channel: if not channel:
if check_file_in_tos(context_path): if check_file_in_tos(context_path):
response_content = tos_utils.get_string_from_tos(context_path) response_content = tos_utils.get_string_from_tos(context_path)
url_list = tos_utils.get_string_from_tos(quto_path)
elif process_func: elif process_func:
file_path, search_keyword, url_list, think_content, response_content, suggestions = process_func(data) file_path, search_keyword, url_list, think_content, response_content, suggestions = process_func(data)
# ------------- # -------------
if response_content: if response_content:
result_v2(response_content, data) result_v2(response_content, data)
if str(thinking_enabled) == '1' and platform =='DB' and not url_list:
redis_client.lpush("DB:geo:api:list", json.dumps(data)) if (str(thinking_enabled) == "1" and platform == "DB" and not url_list):
commit_task(data, "ING")
redis_client.lpush(
"DB:geo:api:list",
json.dumps(data)
)
logger.success(f"{reqId} assistant 处理") logger.success(f"{reqId} assistant 处理")
return return True
else: if not response_content:
scheduler(data) scheduler(data)
return True
else: else:
_, _, _, _, response_content, _ = process_func(data) _, _, _, _, response_content, _ = process_func(data)
...@@ -2016,19 +2372,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS): ...@@ -2016,19 +2372,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
if __name__ == '__main__': if __name__ == '__main__':
data_list = bh_utils.query_data("select * from geo_commit_task where status='ING'")
# req_id_sql = ",".join([f"'{req_id}'" for req_id in req_ids])
#
# data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId in ({req_id_sql}) ")
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status !='SUCCESS'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
def handle_item(i): def handle_item(i):
i["comWordsMap"] = safe_json_loads(i.get("comWordsMap"), []) i["comWordsMap"] = safe_json_loads(i.get("comWordsMap"), [])
i["brandWords"] = safe_json_loads(i.get("brandWords"), []) i["brandWords"] = safe_json_loads(i.get("brandWords"), [])
...@@ -2040,10 +2384,10 @@ if __name__ == '__main__': ...@@ -2040,10 +2384,10 @@ if __name__ == '__main__':
# commit_task(i,'ING') # commit_task(i,'ING')
return task_send_queue(i,type_t) return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
return platform_process(i) # return platform_process(i)
# #
if data_list: if data_list:
with ThreadPoolExecutor(max_workers=50) as executor: with ThreadPoolExecutor(max_workers=10) as executor:
futures = [executor.submit(handle_item, i) for i in data_list] futures = [executor.submit(handle_item, i) for i in data_list]
for future in as_completed(futures): for future in as_completed(futures):
......
aidso_geo/utils/ai_utils.py
View file @ b3a35136
...@@ -4,6 +4,8 @@ import time ...@@ -4,6 +4,8 @@ import time
import requests import requests
import json import json
from loguru import logger
def ai_get_brand_list(content, prompt): def ai_get_brand_list(content, prompt):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions" url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
...@@ -53,7 +55,8 @@ def ai_get_brand_list(content, prompt): ...@@ -53,7 +55,8 @@ def ai_get_brand_list(content, prompt):
"thinking": { "thinking": {
"type": "disabled" "type": "disabled"
}, },
"temperature": 0 "temperature": 0,
"top_p": 1
}) })
headers = { headers = {
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
...@@ -150,23 +153,31 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -150,23 +153,31 @@ def ai_get_product_relation_spu(product_list, keyword):
"role": "system", "role": "system",
"content": """ "content": """
你的核心任务为：基于给定的商品列表与查询关键词，为列表内的每一件商品匹配对应信息，最终输出符合规范要求的JSON结果，执行过程需严格遵循以下规则： 你的核心任务为：基于给定的商品列表与查询关键词，为列表内的每一件商品匹配对应信息，最终输出符合规范要求的JSON结果，执行过程需严格遵循以下规则：
1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下4个字段，各字段的取值规则明确如下： 1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下5个字段，各字段的取值规则明确如下：
（1）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改； （1）row_id：原样返回输入商品中的row_id，禁止修改、遗漏、重新编号或与其他商品交换；
（2）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值； （2）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改；
（3）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；需注意同一款商品的spu_name表述必须完全统一，禁止出现同一SPU存在空格差异、格式差异的情况，例如不得同时出现“无线洗地机5”和“无线洗地机 5”这类仅空格/格式有区别的命名，需统一为规范表述； （3）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值；
（4）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。 （4）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；需注意同一款商品的spu_name表述必须完全统一，禁止出现同一SPU存在空格差异、格式差异的情况，例如不得同时出现“无线洗地机5”和“无线洗地机 5”这类仅空格/格式有区别的命名，需统一为规范表述；
（5）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。
2. 一致性校验特别要求：同批次传入的商品中，若商品标题指向的品牌名相同，brand字段的取值必须保持完全统一，禁止出现同一品牌同时标注“小米”和“xiaomi”这类中英文/不同写法混用的情况，需统一为规范名称；同批次商品的spu_name也需保持表述一致，禁止出现同一SPU同时标注“12 promax”和“12promax”这类格式不统一的情况，需统一为规范表述。 2. 一致性校验特别要求：同批次传入的商品中，若商品标题指向的品牌名相同，brand字段的取值必须保持完全统一，禁止出现同一品牌同时标注“小米”和“xiaomi”这类中英文/不同写法混用的情况，需统一为规范名称；同批次商品的spu_name也需保持表述一致，禁止出现同一SPU同时标注“12 promax”和“12promax”这类格式不统一的情况，需统一为规范表述。
3。row_id关联要求：
（1）输出的product_words数量必须与输入的product_list数量完全一致；
（2）每个输入row_id必须在输出中出现且只能出现一次；
（3）不得输出输入中不存在的row_id；
（4）row_id只用于数据关联，不参与商品分析；
（5）输出顺序可以变化，但row_id与商品信息的对应关系不得变化。
{ {
"row_id": 0,
"produce_name": "郎酒 红花郎15", "produce_name": "郎酒 红花郎15",
"brand": "郎酒", "brand": "郎酒",
"spu_name": "郎酒 红花郎15", "spu_name": "红花郎15",
"current": 1 "current": 1
}, }
""" """
}, },
{ {
"role": "user", "role": "user",
"content": f"""需要处理的商品列表：{product_list} 本次查询的关键词为：{keyword}""" "content": json.dumps({"product_list": product_list,"keyword": keyword,},ensure_ascii=False,)
} }
], ],
"thinking": { "thinking": {
...@@ -188,6 +199,10 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -188,6 +199,10 @@ def ai_get_product_relation_spu(product_list, keyword):
"items": { "items": {
"type": "object", "type": "object",
"properties": { "properties": {
"row_id": {
"type": "integer",
"description": "原样返回输入商品的row_id",
},
"produce_name": { "produce_name": {
"type": "string", "type": "string",
"description": "商品完整原始名称", "description": "商品完整原始名称",
...@@ -207,6 +222,7 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -207,6 +222,7 @@ def ai_get_product_relation_spu(product_list, keyword):
}, },
}, },
"required": [ "required": [
"row_id",
"produce_name", "produce_name",
"brand", "brand",
"spu_name", "spu_name",
...@@ -255,6 +271,7 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -255,6 +271,7 @@ def ai_get_product_relation_spu(product_list, keyword):
return product_words if isinstance(product_words, list) else [] return product_words if isinstance(product_words, list) else []
except Exception as e: except Exception as e:
print("AI调用失败：", repr(e))
return [] return []
def ai_get_product_list_search(product_list): def ai_get_product_list_search(product_list):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment