Skip to content

A
aidso-data
Commit 4b1f9b3a authored by Yaowentong's avatar Yaowentong
Browse files
Options

美团截图 

电商词搜索
parent 27ac512a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1431 additions and 103 deletions
aidso_geo/core/feishu_snipaste_v3.py
View file @ 4b1f9b3a
This diff is collapsed.
aidso_geo/core/routes/interface.py
View file @ 4b1f9b3a
......@@ -4,12 +4,16 @@ import json
import queue
from concurrent.futures import ThreadPoolExecutor
from loguru import logger
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from datetime import datetime
sys.path.append(BASE_DIR)
from aidso_geo.models.process import process_call_back, commit_task, main_process
from aidso_geo.config.base_config import init_redis, init_redis8
from aidso_geo.utils import bh_utils, tos_utils, url_utils
from aidso_geo.models.eco_data_process import process_eco_product_relation
line_app = Blueprint("line_app", __name__)
redis_client = init_redis()
redis_client8 = init_redis8()
......@@ -323,3 +327,95 @@ def check_quto():
"reqId": req_id
})
@line_app.route('/api/geo/check_eco_keyword', methods=['POST'])
def check_eco():
try:
body = request.get_json(silent=True) or {}
reqids = body.get("req_ids")
keyword = body.get("keyword")
if not isinstance(reqids, list):
return jsonify({
"code": 400,
"message": "reqids必须是数组",
"data": None,
})
# 过滤空值并去重，同时保持原顺序。
reqids = list(dict.fromkeys(
str(req_id).strip()
for req_id in reqids
if str(req_id or "").strip()
))
if not reqids:
return jsonify({
"code": 400,
"message": "reqids不能为空",
"data": None,
})
if not isinstance(keyword, str) or not keyword.strip():
return jsonify({
"code": 400,
"message": "keyword必须是非空字符串",
"data": None,
})
keyword = keyword.strip()
placeholders = ", ".join(
["%s"] * len(reqids)
)
result = bh_utils.query_data(
f"""
SELECT *
FROM geo_eco_data
WHERE req_id IN ({placeholders})
""",
tuple(reqids),
)
if not result:
return jsonify({
"code": 200,
"message": "处理成功",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": len(result),
},
})
eco_result = process_eco_product_relation(
result,
keyword,
)
bh_utils.insert_data("geo_eco_data",eco_result)
return jsonify({
"code": 200,
"message": "处理成功",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": len(eco_result),
},
})
except Exception as exc:
return jsonify({
"code": 500,
"message": f"商品关系处理失败: {exc}",
"data": None,
})
aidso_geo/models/eco_data_process.py
View file @ 4b1f9b3a
import re
from aidso_geo.utils import bh_utils
from aidso_geo.utils import bh_utils,ai_utils
def process_eco_product_relation(
eco_result,
keyword,
batch_size=500,
):
"""
每100条商品调用一次AI，并将识别结果合并到原商品数据中。
produce_name只用于匹配，不写入最终结果。
"""
def normalize_title(value):
return " ".join(
str(value or "").strip().split()
)
final_result = []
for start_index in range(
0,
len(eco_result),
batch_size,
):
batch = eco_result[
start_index:start_index + batch_size
]
product_list = [
str(item.get("eco_title") or "").strip()
for item in batch
]
try:
ai_result = (
ai_utils.ai_get_product_relation_spu(
product_list,
keyword,
)
)
except Exception as exc:
print(
f"AI商品识别失败: "
f"batch={start_index // batch_size + 1}, "
f"error={exc}"
)
ai_result = []
if not isinstance(ai_result, list):
ai_result = []
# produce_name只用于建立匹配关系。
ai_result_map = {}
for ai_item in ai_result:
if not isinstance(ai_item, dict):
continue
produce_name = str(
ai_item.get("produce_name") or ""
).strip()
normalized_name = normalize_title(
produce_name
)
if not normalized_name:
continue
if normalized_name not in ai_result_map:
ai_result_map[normalized_name] = ai_item
for original_item in batch:
eco_title = str(
original_item.get("eco_title") or ""
).strip()
matched_item = ai_result_map.get(
normalize_title(eco_title)
)
# 未匹配时的默认值。
relation_data = {
"brand": eco_title,
"spu_name": "",
"current": 0,
}
if isinstance(matched_item, dict):
brand = str(
matched_item.get("brand")
or eco_title
).strip()
spu_name = str(
matched_item.get("spu_name")
or ""
).strip()
try:
current = int(
matched_item.get("current", 0)
)
except (TypeError, ValueError):
current = 0
relation_data = {
"brand": brand,
"search_goods_word":keyword,
"spu_name": brand+" "+spu_name,
"current": (
1 if current == 1 else 0
),
}
final_result.append(
{
**original_item,
**relation_data,
}
)
return final_result
def extract_price(price):
if not price:
......@@ -234,6 +357,7 @@ def yuanbao_android_process_txmap_eco(data,eco_list):
def save_eco_data_to_bh(data,eco_type,eco_list):
platform = data.get('platform')
search_goods_word = data.get('search_goods_word')
eco_result = []
if platform == 'TYQWA':
......@@ -259,6 +383,10 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
if eco_type == 'txmap':
eco_result = yuanbao_process_txmap_eco(data,eco_list)
if search_goods_word:
eco_result = process_eco_product_relation(eco_result, search_goods_word)
bh_utils.insert_data('geo_eco_data',eco_result)
aidso_geo/models/process.py
View file @ 4b1f9b3a
......@@ -2032,9 +2032,9 @@ if __name__ == '__main__':
type_t = i.get('type')
type_t = 'batch'
# commit_task(i,'ING')
return task_send_queue(i,type_t)
# return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i)
# return platform_process(i)
return platform_process(i)
#
if data_list:
with ThreadPoolExecutor(max_workers=50) as executor:
......
aidso_geo/models/qianwen_data_process.py
View file @ 4b1f9b3a
......@@ -33,7 +33,6 @@ def qianwen_process_original_data(data):
data_str = i.split("data:")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue
......@@ -146,6 +145,19 @@ def qianwen_process_original_data(data):
if paas:
for pa in ms.get('meta_data').get('paas'):
suggestions.append(pa.get('show_text'))
if ms.get('mime_type') == 'bar/workflow' and ms.get('status') == 'complete':
ms_meta_data_multi_load = ms.get('meta_data').get('multi_load')
if isinstance(ms_meta_data_multi_load,list):
for i in ms_meta_data_multi_load:
ms_meta_data_multi_load_type =i.get('type')
if ms_meta_data_multi_load_type == 'bar_thinking':
think_content+=i.get('content').get('body')
if ms_meta_data_multi_load_type == 'bar_ref_source_inline':
if i.get('content'):
if i.get('content').get('query_list'):
search_keyword.extend(i.get('content').get('query_list'))
if i.get('content').get('docs'):
url_list_batch.extend(i.get('content').get('docs'))
if url_list_batch:
for url in url_list_batch:
......@@ -163,7 +175,6 @@ def qianwen_process_original_data(data):
suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e:
traceback.print_exc()
parts = file_path.split('/')
platform = parts[2]
task_id = parts[1]
......@@ -195,7 +206,7 @@ if __name__ == '__main__':
# # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
# qianwen_process_original_data(file_path2)
# for i in task_id:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '13e740b9-955c-4f9b-a03e-92e656ffaf43' and platform = 'TYQW'")
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '9fd93209-01db-4dd2-84e3-02eeccbedb16' and platform = 'TYQW'")
# # #
# # #
......
aidso_geo/utils/ai_utils.py
View file @ 4b1f9b3a
......@@ -3,9 +3,6 @@ import time
import requests
import json
from aidso_geo.core.down_load_bot import get_req_id
from aidso_geo.utils import bh_utils
from aidso_geo.utils.tos_utils import get_string_from_tos
......@@ -144,6 +141,126 @@ def ai_get_product_list(content, prompt):
except Exception as e:
return []
def ai_get_product_relation_spu(product_list, keyword):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
payload = {
"model": "doubao-seed-2-0-mini-260428",
"messages": [
{
"role": "system",
"content": """
你的核心任务为：基于给定的商品列表与查询关键词，为列表内的每一件商品匹配对应信息，最终输出符合规范要求的JSON结果，执行过程需严格遵循以下规则：
1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下4个字段，各字段的取值规则明确如下：
（1）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改；
（2）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值；
（3）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；
（4）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。
2. 一致性校验特别要求：同批次传入的商品中，若商品标题指向的品牌名相同，brand字段的取值必须保持完全统一，禁止出现同一品牌同时标注“小米”和“xiaomi”这类中英文/不同写法混用的情况，需统一为规范名称；同批次商品的spu_name也需保持表述一致，禁止出现同一SPU同时标注“12 promax”和“12promax”这类格式不统一的情况，需统一为规范表述。
{
"produce_name": "郎酒 红花郎15",
"brand": "郎酒",
"spu_name": "红花郎15",
"current": 0
},
"""
},
{
"role": "user",
"content": f"""需要处理的商品列表：{product_list} 本次查询的关键词为：{keyword}"""
}
],
"thinking": {
"type": "disabled"
},
"temperature": 0,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "product_relation_result",
"strict": True,
"schema": {
"type": "object",
"properties": {
"product_words": {
"type": "array",
"description": "商品识别和关联判断结果",
"items": {
"type": "object",
"properties": {
"produce_name": {
"type": "string",
"description": "商品完整原始名称",
},
"brand": {
"type": "string",
"description": "商品所属品牌",
},
"spu_name": {
"type": "string",
"description": "去除品牌后的标准SPU名称",
},
"current": {
"type": "integer",
"enum": [0, 1],
"description": "与查询关键词有关为1，否则为0",
},
},
"required": [
"produce_name",
"brand",
"spu_name",
"current",
],
"additionalProperties": False,
},
},
},
"required": ["product_words"],
"additionalProperties": False,
},
},
},
}
headers = {
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
try:
response = requests.post(
url,
headers=headers,
json=payload,
timeout=120,
)
if response.status_code != 200:
print("status_code:", response.status_code)
print("response_text:", response.text)
response.raise_for_status()
response_data = response.json()
content = (
response_data
.get("choices", [{}])[0]
.get("message", {})
.get("content")
)
if not content:
return []
if isinstance(content, str):
content = json.loads(content)
product_words = content.get("product_words", [])
return product_words if isinstance(product_words, list) else []
except Exception as e:
print(e)
return []
def ai_get_product_list_search(product_list):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
payload = json.dumps({
......@@ -989,16 +1106,16 @@ render_ecom_card_widget_jd_start:
render_ecom_card_widget_jd_end:
"""
brand_list =[
product_list =[
"瓜子二手车","二手车之家"
# fcc424e5-58af-494d-9683-5787413a26c9
]
promp = """
空调选购
keyword = """
瓜子
"""
print(ai_get_product_list(content,promp))
print(ai_get_product_relation_spu(product_list,keyword))
# print(ai_result)
# pro =
# ai_get_product_list()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment