Skip to content

A
aidso-data
Commit 4b1f9b3a authored by Yaowentong's avatar Yaowentong
Browse files
Options

美团截图 

电商词搜索
parent 27ac512a
Hide whitespace changes
Inline Side-by-side
Showing with 1431 additions and 103 deletions
aidso_geo/core/feishu_snipaste_v3.py
View file @ 4b1f9b3a
"""每日把飞书 Wiki 文档表格转换为截图任务并写入数据库。""" """每日把飞书 Wiki 文档表格转换为截图任务并写入数据库。"""
from __future__ import annotations from __future__ import annotations
import re
import json import json
import os import os
import sys import sys
import tempfile
import uuid import uuid
from datetime import datetime from datetime import datetime
import time import time
...@@ -12,7 +13,16 @@ import redis ...@@ -12,7 +13,16 @@ import redis
import requests import requests
from loguru import logger from loguru import logger
from openpyxl import Workbook from openpyxl import Workbook
from openpyxl.styles import (
Alignment,
Border,
Font,
PatternFill,
Side,
)
from openpyxl.utils import get_column_letter
from aidso_geo.clients.tos_client import tos_client
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
POLL_SECONDS = 120 POLL_SECONDS = 120
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
...@@ -27,12 +37,34 @@ FEISHU_CHAT_ID = "oc_41bc8e496a0d9bb93498a6a6cf6f30c1" ...@@ -27,12 +37,34 @@ FEISHU_CHAT_ID = "oc_41bc8e496a0d9bb93498a6a6cf6f30c1"
WIKI_TOKEN = "RndAwboqZiuCAFkTGkScaWHPnmg" WIKI_TOKEN = "RndAwboqZiuCAFkTGkScaWHPnmg"
TABLE_NAME = "douchacha_data.geo_feishu_snipaste_v3" TABLE_NAME = "douchacha_data.geo_feishu_snipaste_v3"
LOG_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_log_v3" LOG_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_log_v3"
RESULT_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_result_v3"
ROUND_COUNT = 30 ROUND_COUNT = 30
MAX_ROUND_COUNT = 30 MAX_ROUND_COUNT = 30
REDIS_QUEUE_KEY = "mt:snipaste_v3:only_content" REDIS_QUEUE_KEY = "mt:snipaste_v3:only_content"
REDIS_QUEUE_KEY_SHARE = "mt:snipaste_v3:with_share" REDIS_QUEUE_KEY_SHARE = "mt:snipaste_v3:with_share"
SYNC_HOUR = 0 SYNC_HOUR = 0
SYNC_MINUTE = 1 SYNC_MINUTE = 1
PLATFORM_CONFIGS = {
"DB": {
"name": "豆包网页版",
"sheet_name": "豆包网页版",
"tos_directory": "doubao",
"content_redis_key": REDIS_QUEUE_KEY,
"share_redis_key": REDIS_QUEUE_KEY_SHARE,
},
"DP": {
"name": "DeepSeek网页版",
"sheet_name": "deepseek网页版",
"tos_directory": "DP",
"content_redis_key": "mt:DP:snipaste_v3:only_content",
"share_redis_key": "mt:DP:snipaste_v3:with_share",
},
}
THIRTY_ROUND_EXPORT_WORKERS = 20
EXCEL_CELL_MAX_LENGTH = 32767
ILLEGAL_EXCEL_CHARACTERS = re.compile(
r"[\x00-\x08\x0B-\x0C\x0E-\x1F]"
)
PROMPT_HEADERS = {"问题", "prompt", "问题词", "问题内容"} PROMPT_HEADERS = {"问题", "prompt", "问题词", "问题内容"}
...@@ -45,6 +77,16 @@ class TaskResultUnavailableError(RuntimeError): ...@@ -45,6 +77,16 @@ class TaskResultUnavailableError(RuntimeError):
pass pass
def normalize_platform(platform: str = "DB") -> str:
normalized_platform = str(platform or "").strip().upper()
if normalized_platform not in PLATFORM_CONFIGS:
raise ValueError(
f"不支持的平台: {platform!r}，"
f"可选值={list(PLATFORM_CONFIGS)}"
)
return normalized_platform
def normalize_pt(pt=None) -> str: def normalize_pt(pt=None) -> str:
""" """
规范化运行批次。 规范化运行批次。
...@@ -157,7 +199,11 @@ def get_wiki_node(access_token: str) -> dict: ...@@ -157,7 +199,11 @@ def get_wiki_node(access_token: str) -> dict:
return node return node
def get_first_sheet(spreadsheet_token: str, access_token: str) -> dict: def get_sheet_by_name(
spreadsheet_token: str,
access_token: str,
sheet_name: str,
) -> dict:
data = request_json( data = request_json(
"GET", "GET",
( (
...@@ -170,9 +216,27 @@ def get_first_sheet(spreadsheet_token: str, access_token: str) -> dict: ...@@ -170,9 +216,27 @@ def get_first_sheet(spreadsheet_token: str, access_token: str) -> dict:
if not sheets: if not sheets:
raise RuntimeError(f"飞书电子表格中没有工作表: {data}") raise RuntimeError(f"飞书电子表格中没有工作表: {data}")
sheet = sheets[0] normalized_sheet_name = str(sheet_name or "").strip().lower()
sheet = next(
(
item
for item in sheets
if str(item.get("title") or "").strip().lower()
== normalized_sheet_name
),
None,
)
if sheet is None:
available_sheet_names = [
str(item.get("title") or "")
for item in sheets
]
raise RuntimeError(
f"飞书电子表格中没有名为 {sheet_name!r} 的工作表，"
f"现有工作表={available_sheet_names}"
)
if not sheet.get("sheet_id"): if not sheet.get("sheet_id"):
raise RuntimeError(f"飞书返回中没有 sheet_id: {sheets[0]}") raise RuntimeError(f"飞书返回中没有 sheet_id: {sheet}")
return sheet return sheet
...@@ -187,8 +251,16 @@ def column_name(column_count: int) -> str: ...@@ -187,8 +251,16 @@ def column_name(column_count: int) -> str:
return "".join(reversed(result)) return "".join(reversed(result))
def get_sheet_rows(spreadsheet_token: str, access_token: str) -> list[list]: def get_sheet_rows(
sheet = get_first_sheet(spreadsheet_token, access_token) spreadsheet_token: str,
access_token: str,
sheet_name: str,
) -> list[list]:
sheet = get_sheet_by_name(
spreadsheet_token,
access_token,
sheet_name,
)
sheet_id = sheet["sheet_id"] sheet_id = sheet["sheet_id"]
grid_properties = sheet.get("grid_properties") or {} grid_properties = sheet.get("grid_properties") or {}
row_count = int(grid_properties.get("row_count") or 1000) row_count = int(grid_properties.get("row_count") or 1000)
...@@ -228,7 +300,21 @@ def cell_text(value) -> str: ...@@ -228,7 +300,21 @@ def cell_text(value) -> str:
return str(value).strip() return str(value).strip()
def build_tasks(rows: list[list], pt: str) -> list[dict]: def build_tasks(
rows: list[list],
pt: str,
platform: str,
) -> list[dict]:
platform = normalize_platform(platform)
business_line_column_index = None
for row in rows:
for column_index, cell in enumerate(row):
if cell_text(cell) == "业务线":
business_line_column_index = column_index
break
if business_line_column_index is not None:
break
tasks = [] tasks = []
for row in rows: for row in rows:
if not row: if not row:
...@@ -239,10 +325,23 @@ def build_tasks(rows: list[list], pt: str) -> list[dict]: ...@@ -239,10 +325,23 @@ def build_tasks(rows: list[list], pt: str) -> list[dict]:
continue continue
keywords = [] keywords = []
for cell in row[1:]: for column_index, cell in enumerate(
row[1:],
start=1,
):
if column_index == business_line_column_index:
continue
keyword = cell_text(cell) keyword = cell_text(cell)
if keyword: if keyword:
keywords.append(keyword) keywords.append(keyword)
business_line = ""
if (
business_line_column_index is not None
and business_line_column_index < len(row)
):
business_line = cell_text(
row[business_line_column_index]
)
tasks.append( tasks.append(
{ {
"taskId": str(uuid.uuid4()), "taskId": str(uuid.uuid4()),
...@@ -250,31 +349,43 @@ def build_tasks(rows: list[list], pt: str) -> list[dict]: ...@@ -250,31 +349,43 @@ def build_tasks(rows: list[list], pt: str) -> list[dict]:
"keywords": json.dumps(keywords, ensure_ascii=False), "keywords": json.dumps(keywords, ensure_ascii=False),
"pt": pt, "pt": pt,
"sheet_order": len(tasks) + 1, "sheet_order": len(tasks) + 1,
"platform": platform,
"business_line": business_line,
} }
) )
return tasks return tasks
def get_tasks(pt: str) -> list[dict]: def get_tasks(pt: str, platform: str = "DB") -> list[dict]:
platform = normalize_platform(platform)
rows = bh_utils.query_data( rows = bh_utils.query_data(
( (
f"SELECT taskId, prompt, keywords, pt, sheet_order " f"SELECT taskId, prompt, keywords, pt, sheet_order, "
f"FROM {TABLE_NAME} WHERE pt = %s" f"business_line, "
f"ifNull(platform, 'DB') AS platform "
f"FROM {TABLE_NAME} WHERE pt = %s "
f"AND ifNull(platform, 'DB') = %s"
), ),
(pt,), (pt, platform),
) )
if rows is None: if rows is None:
raise RuntimeError(f"查询 {TABLE_NAME} 失败") raise RuntimeError(f"查询 {TABLE_NAME} 失败")
return rows return rows
def create_round_logs(tasks: list[dict], pt: str) -> int: def create_round_logs(
tasks: list[dict],
pt: str,
platform: str = "DB",
) -> int:
platform = normalize_platform(platform)
existing_rows = bh_utils.query_data( existing_rows = bh_utils.query_data(
( (
f"SELECT taskId, `count` FROM {LOG_TABLE_NAME} " f"SELECT taskId, `count` FROM {LOG_TABLE_NAME} "
f"WHERE pt = %s" f"WHERE pt = %s "
f"AND ifNull(platform, 'DB') = %s"
), ),
(pt,), (pt, platform),
) )
if existing_rows is None: if existing_rows is None:
raise RuntimeError(f"查询 {LOG_TABLE_NAME} 失败") raise RuntimeError(f"查询 {LOG_TABLE_NAME} 失败")
...@@ -298,6 +409,7 @@ def create_round_logs(tasks: list[dict], pt: str) -> int: ...@@ -298,6 +409,7 @@ def create_round_logs(tasks: list[dict], pt: str) -> int:
"keywords": task.get("keywords"), "keywords": task.get("keywords"),
"`count`": count, "`count`": count,
"pt": pt, "pt": pt,
"platform": platform,
} }
) )
...@@ -318,7 +430,9 @@ def push_rounds_to_redis( ...@@ -318,7 +430,9 @@ def push_rounds_to_redis(
redis_key: str, redis_key: str,
pt: str = None, pt: str = None,
task_ids: list = None, task_ids: list = None,
platform: str = "DB",
) -> int: ) -> int:
platform = normalize_platform(platform)
is_share_round = ( is_share_round = (
start_count == end_count start_count == end_count
and start_count in (0, 99) and start_count in (0, 99)
...@@ -367,11 +481,12 @@ def push_rounds_to_redis( ...@@ -367,11 +481,12 @@ def push_rounds_to_redis(
( (
f"SELECT reqId, prompt, pt FROM {LOG_TABLE_NAME} " f"SELECT reqId, prompt, pt FROM {LOG_TABLE_NAME} "
f"WHERE pt = %s " f"WHERE pt = %s "
f"AND ifNull(platform, 'DB') = %s "
f"AND toInt32(`count`) BETWEEN %s AND %s " f"AND toInt32(`count`) BETWEEN %s AND %s "
f"{task_filter_sql}" f"{task_filter_sql}"
f"ORDER BY toInt32(`count`), taskId" f"ORDER BY toInt32(`count`), taskId"
), ),
tuple(query_params), tuple([query_params[0], platform, *query_params[1:]]),
) )
if rows is None: if rows is None:
raise RuntimeError( raise RuntimeError(
...@@ -416,10 +531,15 @@ def push_rounds_to_redis( ...@@ -416,10 +531,15 @@ def push_rounds_to_redis(
return len(values) return len(values)
def sync_feishu_to_database(pt: str = None) -> None: def sync_feishu_to_database(
pt: str = None,
platform: str = "DB",
) -> None:
validate_config() validate_config()
platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform]
pt = normalize_pt(pt) pt = normalize_pt(pt)
tasks = get_tasks(pt) tasks = get_tasks(pt, platform=platform)
title = "" title = ""
if not tasks: if not tasks:
...@@ -431,9 +551,13 @@ def sync_feishu_to_database(pt: str = None) -> None: ...@@ -431,9 +551,13 @@ def sync_feishu_to_database(pt: str = None) -> None:
raise RuntimeError( raise RuntimeError(
f"当前 Wiki 节点类型是 {obj_type!r}，仅支持 sheet" f"当前 Wiki 节点类型是 {obj_type!r}，仅支持 sheet"
) )
table_rows = get_sheet_rows(node["obj_token"], access_token) table_rows = get_sheet_rows(
node["obj_token"],
access_token,
platform_config["sheet_name"],
)
tasks = build_tasks(table_rows, pt) tasks = build_tasks(table_rows, pt, platform)
if not tasks: if not tasks:
raise RuntimeError("飞书文档中没有解析到可写入的表格任务") raise RuntimeError("飞书文档中没有解析到可写入的表格任务")
if not bh_utils.insert_data(TABLE_NAME, tasks): if not bh_utils.insert_data(TABLE_NAME, tasks):
...@@ -445,10 +569,14 @@ def sync_feishu_to_database(pt: str = None) -> None: ...@@ -445,10 +569,14 @@ def sync_feishu_to_database(pt: str = None) -> None:
f"rows={len(tasks)}" f"rows={len(tasks)}"
) )
inserted_log_count = create_round_logs(tasks, pt) inserted_log_count = create_round_logs(
tasks,
pt,
platform=platform,
)
logger.success( logger.success(
f"[第一阶段 pt={pt} count=1-30] " f"[第一阶段 platform={platform} pt={pt} count=1-30] "
f"飞书数据库同步完成: title={title}, " f"飞书数据库同步完成: title={title}, "
f"tasks={len(tasks)}, inserted_logs={inserted_log_count}" f"tasks={len(tasks)}, inserted_logs={inserted_log_count}"
) )
...@@ -458,21 +586,25 @@ def start_scheduler() -> None: ...@@ -458,21 +586,25 @@ def start_scheduler() -> None:
from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.schedulers.blocking import BlockingScheduler
scheduler = BlockingScheduler(timezone="Asia/Shanghai") scheduler = BlockingScheduler(timezone="Asia/Shanghai")
scheduler.add_job( for platform in PLATFORM_CONFIGS:
run_daily_pipeline_safely, scheduler.add_job(
trigger="cron", run_daily_pipeline_safely,
hour=SYNC_HOUR, trigger="cron",
minute=SYNC_MINUTE, hour=SYNC_HOUR,
second=0, minute=SYNC_MINUTE,
id="feishu_snipaste_v3_daily_pipeline", second=0,
replace_existing=True, kwargs={"platform": platform},
max_instances=1, id=f"feishu_snipaste_v3_daily_pipeline_{platform}",
coalesce=True, replace_existing=True,
misfire_grace_time=3600, max_instances=1,
) coalesce=True,
misfire_grace_time=3600,
)
logger.success( logger.success(
f"[调度 count=1-99] 七阶段每日任务已注册: " f"[调度 platforms={list(PLATFORM_CONFIGS)} count=1-99] "
f"{SYNC_HOUR:02d}:{SYNC_MINUTE:02d}" f"七阶段每日任务已注册: "
f"{SYNC_HOUR:02d}:{SYNC_MINUTE:02d}, "
f"jobs={len(PLATFORM_CONFIGS)}"
) )
scheduler.start() scheduler.start()
...@@ -636,6 +768,214 @@ def doubao_process_original_data(file_path, original_content): ...@@ -636,6 +768,214 @@ def doubao_process_original_data(file_path, original_content):
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
def deepseek_process_original_data(file_path, original_content):
url_list = ""
think_content = ""
response_content = ""
search_keyword = []
is_think = False
think_bool = False
suggestions = []
response_bool = False
url_id_map = {}
# 按空行分割内容，过滤空字符串
content_list = [item for item in original_content.split("\n\n") if item]
for item in content_list:
if item.startswith("event: "):
# 可根据需要补充event数据处理逻辑
continue
# 处理data类型数据
if item.startswith("data: "):
try:
# 提取并解析JSON数据
data_str = item.split("data: ")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue # 跳过格式错误的数据
if isinstance(json_data.get('v'),dict):
if json_data.get('v').get('response').get('thinking_enabled') == True:
is_think=True
fragments = json_data.get('v').get('response').get('fragments')
if fragments:
if isinstance(fragments, list):
if fragments[0].get('type') == 'RESPONSE':
response_content += fragments[0].get('content')
if fragments[0].get('type') == 'THINK':
think_content += fragments[0].get('content')
think_bool = True
if fragments[0].get('type') == 'SEARCH':
if isinstance(fragments[0].get('queries'), list):
for sea in fragments[0].get('queries'):
search_keyword.append(sea.get('query'))
# if json_data.get('v').get('response').get('search_enabled') == True:
# if len(json_data.get('v').get('response').get('fragments'))>0:
# query_list = json_data.get('v').get('response').get('fragments')[0].get('queries')
# if query_list:
# result = [item.get('query', '') for item in query_list]
# search_keyword.extend(result)
if is_think:
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'SEARCH':
search_keyword.append(json_data.get('v')[0].get('queries')[0].get('query'))
if json_data.get('p') == 'response/fragments/0/results' or json_data.get(
'p') == 'response/fragments/-1/results':
url_list = json_data.get('v')
if json_data.get('p') =='response/fragments' and json_data.get('v')[0].get('type') == 'THINK':
think_bool = True
think_content+=json_data.get('v')[0].get('content')
continue
if json_data.get('p') == 'response' :
if isinstance(json_data.get('v')[0].get('v'),list):
if json_data.get('v')[0].get('v')[0].get('type') == 'TOOL_SEARCH':
for q in json_data.get('v')[0].get('v')[0].get('queries'):
search_keyword.append(q.get('query'))
if json_data.get('v')[1].get('p') == 'fragments' or json_data.get('v')[1].get('p') == 'response/fragments':
if json_data.get('v')[1].get('v')[0].get('type') == 'THINK':
think_content+=json_data.get('v')[1].get('v')[0].get('content')
think_bool = True
continue
if json_data.get('v')[0].get('p') == 'fragments' or json_data.get('v')[0].get('p') == 'response/fragments':
if json_data.get('v')[0].get('v')[0].get('type') == 'THINK':
think_content+=json_data.get('v')[0].get('v')[0].get('content')
think_bool = True
continue
if json_data.get('v')[0].get('v') == 'FINISHED':
response_bool = False
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'RESPONSE':
think_bool = False
response_bool = True
response_content += json_data.get('v')[0].get('content')
continue
if json_data.get('p') == 'response/fragments/1/elapsed_secs':
think_bool = False
if isinstance(json_data.get('v'),list):
v_json_data = json_data.get('v')
has_tool_search = False
tool_search_id = None
for item in v_json_data:
if item.get('p') == 'references':
references = item.get('v') or []
if isinstance(references, list):
for reference in references:
if (
isinstance(reference, dict)
and reference.get('type') == 'TOOL_OPEN'
):
has_tool_search = True
tool_search_id = reference.get('id')
break
for v in v_json_data:
if v.get('p') == 'content' and has_tool_search:
p_content = v.get('v') or ''
p_content = re.sub(
r'\[reference:\d+\]',
f'[reference:{tool_search_id}]',
p_content
)
response_content+=p_content
if v.get('p') == 'fragments':
if v.get('p') == 'fragments':
fragments = v.get('v') or []
if isinstance(fragments, list):
for fragment in fragments:
if (
isinstance(fragment, dict)
and fragment.get('type') == 'TOOL_OPEN'
and fragment.get('result') is not None
):
url_id_map[fragment.get('result').get('url')] = fragment.get('id')
if json_data.get('p') == 'response/fragments/-1/elapsed_secs':
think_bool = False
if json_data.get('p') == 'response/fragments':
response_bool = False
if response_bool:
if isinstance(json_data.get('v'), str):
response_content += json_data.get('v')
if think_bool:
if isinstance(json_data.get('v'), str):
think_content+=json_data.get('v')
else:
if json_data.get('p') == 'response/fragments/-1/content':
response_bool = True
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'SEARCH':
search_keyword.append(json_data.get('v')[0].get('queries')[0].get('query'))
if json_data.get('p') == 'response/fragments/0/results' or json_data.get('p') == 'response/fragments/-1/results':
url_list = json_data.get('v')
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'RESPONSE':
response_content+=json_data.get('v')[0].get('content')
response_bool = True
continue
if json_data.get('p') == 'response':
if json_data.get('v')[1].get('p') == 'fragments':
if json_data.get('v')[1].get('v')[0].get('type') == 'RESPONSE':
response_content+=json_data.get('v')[1].get('v')[0].get('content')
response_bool = True
continue
if json_data.get('v')[0].get('p') == 'fragments':
if json_data.get('v')[0].get('v')[0].get('type') == 'RESPONSE':
response_content+=json_data.get('v')[0].get('v')[0].get('content')
response_bool = True
continue
if json_data.get('v')[0].get('v') == 'FINISHED':
response_bool = False
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'TIP':
response_bool= False
continue
if json_data.get('v') == 'FINISHED':
response_bool = False
if response_bool:
if isinstance(json_data.get('v'),str):
response_content+=json_data.get('v')
used_ids = set(url_id_map.values())
next_id = 1
for item in url_list:
url = item.get('url')
if url in url_id_map:
item['cite_index'] = url_id_map[url]
elif item.get('cite_index') is None:
while next_id in used_ids:
next_id += 1
item['cite_index'] = next_id
used_ids.add(next_id)
next_id += 1
think_content = think_content.replace('DEEP_SEARCH','')
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
PLATFORM_PROCESS_MAP = {
"DB": doubao_process_original_data,
"DP": deepseek_process_original_data,
}
def get_platform_tos_directory(platform: str) -> str:
platform = normalize_platform(platform)
return PLATFORM_CONFIGS[platform]["tos_directory"]
def wait_mt_tasks_done( def wait_mt_tasks_done(
redis_key: str, redis_key: str,
pt: str = None, pt: str = None,
...@@ -704,13 +1044,17 @@ def process_rounds_to_result( ...@@ -704,13 +1044,17 @@ def process_rounds_to_result(
end_count: int, end_count: int,
redis_key: str, redis_key: str,
task_ids: list = None, task_ids: list = None,
platform: str = "DB",
) -> int: ) -> int:
""" """
发送指定轮次，等待并补发未完成任务， 发送指定轮次，等待并补发未完成任务，
最后解析 TOS 正文、统计关键词并写入结果表。 最后解析 TOS 正文、统计关键词并写入结果表。
""" """
platform = normalize_platform(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
redis_pt = pt[:8] redis_pt = pt[:8]
tos_directory = get_platform_tos_directory(platform)
process_original_data = PLATFORM_PROCESS_MAP[platform]
normalized_task_ids = None normalized_task_ids = None
task_filter_sql = "" task_filter_sql = ""
task_filter_params = [] task_filter_params = []
...@@ -740,6 +1084,7 @@ def process_rounds_to_result( ...@@ -740,6 +1084,7 @@ def process_rounds_to_result(
redis_key, redis_key,
pt=pt, pt=pt,
task_ids=normalized_task_ids, task_ids=normalized_task_ids,
platform=platform,
) )
wait_mt_tasks_done( wait_mt_tasks_done(
redis_key, redis_key,
...@@ -756,6 +1101,7 @@ def process_rounds_to_result( ...@@ -756,6 +1101,7 @@ def process_rounds_to_result(
SELECT reqId, prompt, pt SELECT reqId, prompt, pt
FROM {LOG_TABLE_NAME} FROM {LOG_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) BETWEEN %s AND %s AND toInt32(`count`) BETWEEN %s AND %s
{task_filter_sql} {task_filter_sql}
) AS a1 ) AS a1
...@@ -764,6 +1110,7 @@ def process_rounds_to_result( ...@@ -764,6 +1110,7 @@ def process_rounds_to_result(
""", """,
( (
pt, pt,
platform,
start_count, start_count,
end_count, end_count,
*task_filter_params, *task_filter_params,
...@@ -826,11 +1173,13 @@ def process_rounds_to_result( ...@@ -826,11 +1173,13 @@ def process_rounds_to_result(
SELECT * SELECT *
FROM {LOG_TABLE_NAME} FROM {LOG_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) BETWEEN %s AND %s AND toInt32(`count`) BETWEEN %s AND %s
{task_filter_sql} {task_filter_sql}
""", """,
( (
pt, pt,
platform,
start_count, start_count,
end_count, end_count,
*task_filter_params, *task_filter_params,
...@@ -850,7 +1199,7 @@ def process_rounds_to_result( ...@@ -850,7 +1199,7 @@ def process_rounds_to_result(
return None return None
file = ( file = (
f"geo_snipaste/{redis_pt}/doubao/" f"geo_snipaste/{redis_pt}/{tos_directory}/"
f"{req_id}/text.json" f"{req_id}/text.json"
) )
try: try:
...@@ -893,7 +1242,7 @@ def process_rounds_to_result( ...@@ -893,7 +1242,7 @@ def process_rounds_to_result(
_, _,
response_content, response_content,
_, _,
) = doubao_process_original_data( ) = process_original_data(
file, file,
content, content,
) )
...@@ -931,6 +1280,7 @@ def process_rounds_to_result( ...@@ -931,6 +1280,7 @@ def process_rounds_to_result(
"pt": int(row.get("pt")), "pt": int(row.get("pt")),
"has_keyword": has_keyword, "has_keyword": has_keyword,
"keywords_count": keywords_count, "keywords_count": keywords_count,
"platform": platform,
} }
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
...@@ -1081,8 +1431,7 @@ def process_rounds_to_result( ...@@ -1081,8 +1431,7 @@ def process_rounds_to_result(
for attempt in range(1, 4): for attempt in range(1, 4):
try: try:
success = bh_utils.insert_data( success = bh_utils.insert_data(
"douchacha_data." RESULT_TABLE_NAME,
"geo_feishu_snipaste_result_v3",
batch_items, batch_items,
retry_times=0, retry_times=0,
) )
...@@ -1233,16 +1582,19 @@ def create_and_process_share_rounds( ...@@ -1233,16 +1582,19 @@ def create_and_process_share_rounds(
hit_start_count: int = 1, hit_start_count: int = 1,
hit_end_count: int = 30, hit_end_count: int = 30,
hit_threshold: int = 24, hit_threshold: int = 24,
platform: str = "DB",
) -> int: ) -> int:
""" """
第 1-30 轮中关键词命中轮数达到阈值的 taskId， 第 1-30 轮中关键词命中轮数达到阈值的 taskId，
生成 count=0 的分享任务并完成 Redis、TOS 和结果入库流程。 生成 count=0 的分享任务并完成 Redis、TOS 和结果入库流程。
""" """
platform = normalize_platform(platform)
qualified_rows = bh_utils.query_data( qualified_rows = bh_utils.query_data(
""" f"""
SELECT taskId SELECT taskId
FROM douchacha_data.geo_feishu_snipaste_result_v3 FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN %s AND %s AND `count` BETWEEN %s AND %s
AND has_keyword = 1 AND has_keyword = 1
GROUP BY taskId GROUP BY taskId
...@@ -1250,6 +1602,7 @@ def create_and_process_share_rounds( ...@@ -1250,6 +1602,7 @@ def create_and_process_share_rounds(
""", """,
( (
int(pt), int(pt),
platform,
hit_start_count, hit_start_count,
hit_end_count, hit_end_count,
hit_threshold, hit_threshold,
...@@ -1276,10 +1629,12 @@ def create_and_process_share_rounds( ...@@ -1276,10 +1629,12 @@ def create_and_process_share_rounds(
SELECT taskId, prompt, keywords, pt SELECT taskId, prompt, keywords, pt
FROM {TABLE_NAME} FROM {TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND taskId IN ({task_id_placeholders}) AND taskId IN ({task_id_placeholders})
""", """,
( (
pt, pt,
platform,
*task_ids, *task_ids,
), ),
) )
...@@ -1291,11 +1646,13 @@ def create_and_process_share_rounds( ...@@ -1291,11 +1646,13 @@ def create_and_process_share_rounds(
SELECT taskId SELECT taskId
FROM {LOG_TABLE_NAME} FROM {LOG_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) = 0 AND toInt32(`count`) = 0
AND taskId IN ({task_id_placeholders}) AND taskId IN ({task_id_placeholders})
""", """,
( (
pt, pt,
platform,
*task_ids, *task_ids,
), ),
) )
...@@ -1320,6 +1677,7 @@ def create_and_process_share_rounds( ...@@ -1320,6 +1677,7 @@ def create_and_process_share_rounds(
"keywords": task.get("keywords"), "keywords": task.get("keywords"),
"`count`": "0", "`count`": "0",
"pt": str(task.get("pt") or pt), "pt": str(task.get("pt") or pt),
"platform": platform,
} }
) )
...@@ -1342,6 +1700,7 @@ def create_and_process_share_rounds( ...@@ -1342,6 +1700,7 @@ def create_and_process_share_rounds(
start_count=0, start_count=0,
end_count=0, end_count=0,
redis_key=redis_key, redis_key=redis_key,
platform=platform,
) )
return inserted_log_count return inserted_log_count
...@@ -1352,16 +1711,19 @@ def create_and_process_unqualified_share_rounds( ...@@ -1352,16 +1711,19 @@ def create_and_process_unqualified_share_rounds(
hit_start_count: int = 1, hit_start_count: int = 1,
hit_end_count: int = 30, hit_end_count: int = 30,
hit_threshold: int = 24, hit_threshold: int = 24,
platform: str = "DB",
) -> int: ) -> int:
""" """
第 1-30 轮中关键词命中轮数不足阈值的 taskId， 第 1-30 轮中关键词命中轮数不足阈值的 taskId，
生成 count=99 的分享任务并完成 Redis、TOS 和结果入库流程。 生成 count=99 的分享任务并完成 Redis、TOS 和结果入库流程。
""" """
platform = normalize_platform(platform)
qualified_rows = bh_utils.query_data( qualified_rows = bh_utils.query_data(
""" f"""
SELECT taskId SELECT taskId
FROM douchacha_data.geo_feishu_snipaste_result_v3 FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN %s AND %s AND `count` BETWEEN %s AND %s
AND has_keyword = 1 AND has_keyword = 1
GROUP BY taskId GROUP BY taskId
...@@ -1369,6 +1731,7 @@ def create_and_process_unqualified_share_rounds( ...@@ -1369,6 +1731,7 @@ def create_and_process_unqualified_share_rounds(
""", """,
( (
int(pt), int(pt),
platform,
hit_start_count, hit_start_count,
hit_end_count, hit_end_count,
hit_threshold, hit_threshold,
...@@ -1387,8 +1750,9 @@ def create_and_process_unqualified_share_rounds( ...@@ -1387,8 +1750,9 @@ def create_and_process_unqualified_share_rounds(
SELECT taskId, prompt, keywords, pt SELECT taskId, prompt, keywords, pt
FROM {TABLE_NAME} FROM {TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
""", """,
(pt,), (pt, platform),
) )
if tasks is None: if tasks is None:
raise RuntimeError("第五阶段从主表查询未达标任务失败") raise RuntimeError("第五阶段从主表查询未达标任务失败")
...@@ -1411,9 +1775,10 @@ def create_and_process_unqualified_share_rounds( ...@@ -1411,9 +1775,10 @@ def create_and_process_unqualified_share_rounds(
SELECT taskId SELECT taskId
FROM {LOG_TABLE_NAME} FROM {LOG_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) = 99 AND toInt32(`count`) = 99
""", """,
(pt,), (pt, platform),
) )
if existing_rows is None: if existing_rows is None:
raise RuntimeError("第五阶段查询已生成 count=99 任务失败") raise RuntimeError("第五阶段查询已生成 count=99 任务失败")
...@@ -1436,6 +1801,7 @@ def create_and_process_unqualified_share_rounds( ...@@ -1436,6 +1801,7 @@ def create_and_process_unqualified_share_rounds(
"keywords": task.get("keywords"), "keywords": task.get("keywords"),
"`count`": "99", "`count`": "99",
"pt": str(task.get("pt") or pt), "pt": str(task.get("pt") or pt),
"platform": platform,
} }
) )
...@@ -1460,6 +1826,7 @@ def create_and_process_unqualified_share_rounds( ...@@ -1460,6 +1826,7 @@ def create_and_process_unqualified_share_rounds(
start_count=99, start_count=99,
end_count=99, end_count=99,
redis_key=redis_key, redis_key=redis_key,
platform=platform,
) )
return inserted_log_count return inserted_log_count
...@@ -1468,14 +1835,18 @@ def export_share_rounds_to_excel( ...@@ -1468,14 +1835,18 @@ def export_share_rounds_to_excel(
pt: str, pt: str,
output_path: str = None, output_path: str = None,
share_count: int = 0, share_count: int = 0,
platform: str = "DB",
) -> str: ) -> str:
""" """
导出当天指定分享轮次的任务及其 TOS 结果到 Excel。 导出当天指定分享轮次的任务及其 TOS 结果到 Excel。
""" """
if share_count not in (0, 99): if share_count not in (0, 99):
raise ValueError("Excel 导出仅支持分享轮次 count=0 或 count=99") raise ValueError("Excel 导出仅支持分享轮次 count=0 或 count=99")
platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform]
pt = normalize_pt(pt) pt = normalize_pt(pt)
tos_pt = pt[:8] tos_pt = pt[:8]
tos_directory = get_platform_tos_directory(platform)
stage_name = "第四阶段" if share_count == 0 else "第六阶段" stage_name = "第四阶段" if share_count == 0 else "第六阶段"
file_label = "达标数据" if share_count == 0 else "未达标" file_label = "达标数据" if share_count == 0 else "未达标"
qualification_filter_sql = ( qualification_filter_sql = (
...@@ -1501,6 +1872,8 @@ def export_share_rounds_to_excel( ...@@ -1501,6 +1872,8 @@ def export_share_rounds_to_excel(
LEFT JOIN {TABLE_NAME} AS a5 LEFT JOIN {TABLE_NAME} AS a5
ON a1.taskId = a5.taskId ON a1.taskId = a5.taskId
AND a1.pt = a5.pt AND a1.pt = a5.pt
AND ifNull(a1.platform, 'DB')
= ifNull(a5.platform, 'DB')
LEFT JOIN ( LEFT JOIN (
SELECT SELECT
taskId, taskId,
...@@ -1511,22 +1884,32 @@ def export_share_rounds_to_excel( ...@@ -1511,22 +1884,32 @@ def export_share_rounds_to_excel(
taskId, taskId,
`count`, `count`,
max(has_keyword) AS round_has_keyword max(has_keyword) AS round_has_keyword
FROM douchacha_data.geo_feishu_snipaste_result_v3 FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND 30
GROUP BY taskId, `count` GROUP BY taskId, `count`
) AS round_result ) AS round_result
GROUP BY taskId GROUP BY taskId
) AS a3 ON a1.taskId = a3.taskId ) AS a3 ON a1.taskId = a3.taskId
LEFT JOIN douchacha_data.geo_feishu_snipaste_result_v3 AS a4 LEFT JOIN {RESULT_TABLE_NAME} AS a4
ON a1.reqId = a4.reqId ON a1.reqId = a4.reqId
AND toInt32(a1.`count`) = a4.`count` AND toInt32(a1.`count`) = a4.`count`
AND ifNull(a1.platform, 'DB')
= ifNull(a4.platform, 'DB')
WHERE a1.pt = %s WHERE a1.pt = %s
AND ifNull(a1.platform, 'DB') = %s
AND toInt32(a1.`count`) = %s AND toInt32(a1.`count`) = %s
{qualification_filter_sql} {qualification_filter_sql}
ORDER BY a5.sheet_order, a1.reqId ORDER BY a5.sheet_order, a1.reqId
""", """,
(int(pt), pt, share_count), (
int(pt),
platform,
pt,
platform,
share_count,
),
) )
if rows is None: if rows is None:
raise RuntimeError( raise RuntimeError(
...@@ -1538,9 +1921,10 @@ def export_share_rounds_to_excel( ...@@ -1538,9 +1921,10 @@ def export_share_rounds_to_excel(
SELECT taskId, keywords SELECT taskId, keywords
FROM {TABLE_NAME} FROM {TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
ORDER BY sheet_order, taskId ORDER BY sheet_order, taskId
""", """,
(pt,), (pt, platform),
) )
if task_keyword_rows is None: if task_keyword_rows is None:
raise RuntimeError( raise RuntimeError(
...@@ -1548,13 +1932,14 @@ def export_share_rounds_to_excel( ...@@ -1548,13 +1932,14 @@ def export_share_rounds_to_excel(
) )
round_keyword_rows = bh_utils.query_data( round_keyword_rows = bh_utils.query_data(
""" f"""
SELECT taskId, `count`, keywords_count SELECT taskId, `count`, keywords_count
FROM douchacha_data.geo_feishu_snipaste_result_v3 FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND 30
""", """,
(int(pt),), (int(pt), platform),
) )
if round_keyword_rows is None: if round_keyword_rows is None:
raise RuntimeError( raise RuntimeError(
...@@ -1636,7 +2021,10 @@ def export_share_rounds_to_excel( ...@@ -1636,7 +2021,10 @@ def export_share_rounds_to_excel(
) )
output_path = os.path.join( output_path = os.path.join(
"/Users/yaowentong/Desktop", "/Users/yaowentong/Desktop",
f"{formatted_date}_{file_label}.xlsx", (
f"{formatted_date}_"
f"{platform_config['name']}_{file_label}.xlsx"
),
) )
output_dir = os.path.dirname(os.path.abspath(output_path)) output_dir = os.path.dirname(os.path.abspath(output_path))
...@@ -1653,6 +2041,8 @@ def export_share_rounds_to_excel( ...@@ -1653,6 +2041,8 @@ def export_share_rounds_to_excel(
[ [
"问题", "问题",
"是否达标", "是否达标",
"达标轮次",
"达标率",
"本次是否达标", "本次是否达标",
*all_keywords, *all_keywords,
"平台", "平台",
...@@ -1668,10 +2058,6 @@ def export_share_rounds_to_excel( ...@@ -1668,10 +2058,6 @@ def export_share_rounds_to_excel(
] ]
) )
plat_form_map = {
"DB": "豆包网页版",
}
def timestamp_to_datetime(timestamp): def timestamp_to_datetime(timestamp):
if timestamp in (None, ""): if timestamp in (None, ""):
return "" return ""
...@@ -1686,14 +2072,16 @@ def export_share_rounds_to_excel( ...@@ -1686,14 +2072,16 @@ def export_share_rounds_to_excel(
task_id = str(row.get("taskId") or "") task_id = str(row.get("taskId") or "")
prompt = row.get("prompt") prompt = row.get("prompt")
platform = row.get("platform") row_platform = str(row.get("platform") or platform)
insertime = row.get("insertime") insertime = row.get("insertime")
sheet_order = row.get("sheet_order") sheet_order = row.get("sheet_order")
processed_round_count = row.get("processed_round_count") processed_round_count = row.get("processed_round_count")
hit_round_count = row.get("hit_round_count") hit_round_count = row.get("hit_round_count")
hit_round_count_value = int(hit_round_count or 0)
hit_rate = hit_round_count_value / 30
if not processed_round_count: if not processed_round_count:
qualified_text = "没处理" qualified_text = "没处理"
elif int(hit_round_count or 0) >= 24: elif hit_round_count_value >= 24:
qualified_text = "是" qualified_text = "是"
else: else:
qualified_text = "否" qualified_text = "否"
...@@ -1715,7 +2103,10 @@ def export_share_rounds_to_excel( ...@@ -1715,7 +2103,10 @@ def export_share_rounds_to_excel(
for keyword in all_keywords for keyword in all_keywords
] ]
base_path = f"geo_snipaste/{tos_pt}/doubao/{req_id}" base_path = (
f"geo_snipaste/{tos_pt}/"
f"{tos_directory}/{req_id}"
)
text_json_path = f"{base_path}/text.json" text_json_path = f"{base_path}/text.json"
context_path = f"{base_path}/context.txt" context_path = f"{base_path}/context.txt"
think_path = f"{base_path}/think.txt" think_path = f"{base_path}/think.txt"
...@@ -1726,7 +2117,8 @@ def export_share_rounds_to_excel( ...@@ -1726,7 +2117,8 @@ def export_share_rounds_to_excel(
) )
png_url = ( png_url = (
f"https://tcdn.aidso.com/" f"https://tcdn.aidso.com/"
f"geo_snipaste/{tos_pt}/doubao/{req_id}/png.png" f"geo_snipaste/{tos_pt}/"
f"{tos_directory}/{req_id}/png.png"
) )
text_json_str = ( text_json_str = (
...@@ -1744,9 +2136,14 @@ def export_share_rounds_to_excel( ...@@ -1744,9 +2136,14 @@ def export_share_rounds_to_excel(
[ [
prompt, prompt,
qualified_text, qualified_text,
hit_round_count_value,
hit_rate,
current_qualified_text, current_qualified_text,
*keyword_total_values, *keyword_total_values,
plat_form_map.get(platform, "豆包网页版"), PLATFORM_CONFIGS.get(
row_platform,
platform_config,
).get("name", row_platform),
tos_utils.get_string_from_tos(context_path), tos_utils.get_string_from_tos(context_path),
tos_utils.get_string_from_tos(think_path), tos_utils.get_string_from_tos(think_path),
tos_utils.get_string_from_tos(quote_path), tos_utils.get_string_from_tos(quote_path),
...@@ -1763,6 +2160,9 @@ def export_share_rounds_to_excel( ...@@ -1763,6 +2160,9 @@ def export_share_rounds_to_excel(
] ]
) )
for cell in worksheet["D"][1:]:
cell.number_format = "0.00%"
workbook.save(output_path) workbook.save(output_path)
workbook.close() workbook.close()
logger.success( logger.success(
...@@ -1776,6 +2176,7 @@ def export_share_rounds_to_excel( ...@@ -1776,6 +2176,7 @@ def export_share_rounds_to_excel(
def export_unqualified_share_rounds_to_excel( def export_unqualified_share_rounds_to_excel(
pt: str, pt: str,
output_path: str = None, output_path: str = None,
platform: str = "DB",
) -> str: ) -> str:
""" """
第六阶段导出当天 count=99 的未达标分享结果。 第六阶段导出当天 count=99 的未达标分享结果。
...@@ -1784,6 +2185,7 @@ def export_unqualified_share_rounds_to_excel( ...@@ -1784,6 +2185,7 @@ def export_unqualified_share_rounds_to_excel(
pt=pt, pt=pt,
output_path=output_path, output_path=output_path,
share_count=99, share_count=99,
platform=platform,
) )
...@@ -1867,30 +2269,549 @@ def upload_excel_to_feishu_chat( ...@@ -1867,30 +2269,549 @@ def upload_excel_to_feishu_chat(
return message_id return message_id
def build_daily_run_report(pt: str) -> str: def _parse_json_list(value) -> list:
if isinstance(value, list):
return value
if not isinstance(value, str) or not value.strip():
return []
try:
parsed = json.loads(value)
except json.JSONDecodeError:
return [value]
return parsed if isinstance(parsed, list) else []
def _parse_task_keywords(value) -> list[str]:
keywords = []
seen_keywords = set()
for item in _parse_json_list(value):
keyword = str(item or "").strip()
if not keyword or keyword in seen_keywords:
continue
seen_keywords.add(keyword)
keywords.append(keyword)
return keywords
def _parse_round_keywords_count(value) -> dict:
keyword_counts = {}
for item in _parse_json_list(value):
if not isinstance(item, dict):
continue
keyword = str(item.get("keyword") or "").strip()
if not keyword:
continue
try:
word_count = int(item.get("word_count") or 0)
except (TypeError, ValueError):
word_count = 0
keyword_counts[keyword] = max(
keyword_counts.get(keyword, 0),
word_count,
)
return keyword_counts
def _excel_safe_text(value) -> str:
text = str(value or "")
text = ILLEGAL_EXCEL_CHARACTERS.sub("", text)
return text[:EXCEL_CELL_MAX_LENGTH]
def _safe_get_tos_text(path: str) -> str:
try:
return tos_utils.get_string_from_tos(path) or ""
except Exception as exc:
logger.warning(
f"30轮明细读取 TOS 失败: path={path}, error={exc}"
)
return ""
def _timestamp_to_text(timestamp) -> str:
if timestamp in (None, ""):
return ""
try:
timestamp_number = int(timestamp)
if timestamp_number > 10 ** 12:
timestamp_number //= 1000
return datetime.fromtimestamp(timestamp_number).strftime(
"%Y-%m-%d %H:%M:%S"
)
except (TypeError, ValueError, OSError, OverflowError):
return str(timestamp)
def _sheet_order_sort_key(row: dict) -> tuple:
try:
sheet_order = int(row.get("sheet_order"))
return (
0,
sheet_order,
str(row.get("prompt") or ""),
)
except (TypeError, ValueError):
return (
1,
0,
str(row.get("prompt") or ""),
)
def _chinese_round_number(number: int) -> str:
digits = "零一二三四五六七八九"
if number < 10:
return digits[number]
if number == 10:
return "十"
if number < 20:
return f"十{digits[number % 10]}"
if number % 10 == 0:
return f"{digits[number // 10]}十"
return f"{digits[number // 10]}十{digits[number % 10]}"
def _safe_file_name_part(value: str) -> str:
normalized_value = str(value or "").strip() or "未分类"
normalized_value = re.sub(
r'[\\/:*?"<>|]',
"_",
normalized_value,
)
return normalized_value.strip(". ") or "未分类"
def _style_thirty_round_worksheet(
worksheet,
total_columns: int,
) -> None:
worksheet.freeze_panes = "A2"
worksheet.auto_filter.ref = worksheet.dimensions
worksheet.sheet_view.showGridLines = False
worksheet.row_dimensions[1].height = 32
header_fill = PatternFill("solid", fgColor="6D28D9")
header_font = Font(color="FFFFFF", bold=True)
header_alignment = Alignment(
horizontal="center",
vertical="center",
wrap_text=True,
)
row_border = Border(
bottom=Side(style="thin", color="D8DCE6")
)
for cell in worksheet[1]:
cell.fill = header_fill
cell.font = header_font
cell.alignment = header_alignment
for row in worksheet.iter_rows(
min_row=2,
max_row=worksheet.max_row,
max_col=total_columns,
):
for cell in row:
cell.alignment = Alignment(
vertical="top",
wrap_text=True,
)
cell.border = row_border
column_widths = {
"问题": 42,
"批次": 16,
"30次是否达标": 16,
"本次是否达标": 16,
"平台": 18,
"回答": 70,
"思考过程": 55,
"引用来源": 55,
"查询时间": 22,
"IP": 18,
"版本号": 20,
}
for column_index, cell in enumerate(
worksheet[1],
start=1,
):
worksheet.column_dimensions[
get_column_letter(column_index)
].width = column_widths.get(str(cell.value), 16)
def _upload_thirty_round_excel_to_tos(
file_path: str,
object_key: str,
retry_times: int = 3,
) -> str:
client = tos_client.get_client()
bucket = tos_client.bucket_name
with open(file_path, "rb") as file_object:
file_content = file_object.read()
last_error = None
for attempt in range(1, retry_times + 1):
try:
client.put_object(
bucket,
object_key,
content=file_content,
content_type=(
"application/vnd.openxmlformats-officedocument."
"spreadsheetml.sheet"
),
)
logger.success(
f"[第七阶段 count=1-30] 30轮明细已上传 TOS: "
f"tos://{bucket}/{object_key}, "
f"attempt={attempt}/{retry_times}"
)
return f"tos://{bucket}/{object_key}"
except Exception as exc:
last_error = exc
logger.warning(
f"[第七阶段 count=1-30] 30轮明细上传失败: "
f"object_key={object_key}, "
f"attempt={attempt}/{retry_times}, error={exc}"
)
raise RuntimeError(
f"30轮明细上传 TOS 连续失败 {retry_times} 次: "
f"object_key={object_key}"
) from last_error
def export_thirty_round_details(
pt: str,
platform: str = "DB",
) -> list[dict]:
"""
按业务线导出当前 pt、当前平台的 1-30 轮明细并上传到 TOS。
返回可直接写入飞书日报的业务线及 CDN 下载地址列表。
"""
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
platform = normalize_platform(platform)
pt = normalize_pt(pt)
platform_config = PLATFORM_CONFIGS[platform]
tos_directory = get_platform_tos_directory(platform)
log_rows = bh_utils.query_data(
f"""
SELECT
a1.reqId,
a1.taskId,
a1.prompt,
a1.pt,
toInt32(a1.`count`) AS round_count,
a2.sheet_order,
a2.business_line,
a3.insertime
FROM {LOG_TABLE_NAME} AS a1
LEFT JOIN {TABLE_NAME} AS a2
ON a1.taskId = a2.taskId
AND a1.pt = a2.pt
AND ifNull(a1.platform, 'DB')
= ifNull(a2.platform, 'DB')
LEFT JOIN geo_third_task AS a3
ON a1.reqId = a3.reqId
WHERE toString(a1.pt) = %s
AND ifNull(a1.platform, 'DB') = %s
AND toInt32(a1.`count`) BETWEEN 1 AND 30
""",
(pt, platform),
)
if log_rows is None:
raise RuntimeError("第七阶段查询 1-30 轮日志数据失败")
result_rows = bh_utils.query_data(
f"""
SELECT
reqId,
taskId,
pt,
toInt32(`count`) AS round_count,
has_keyword
FROM {RESULT_TABLE_NAME}
WHERE toString(pt) = %s
AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) BETWEEN 1 AND 30
""",
(pt, platform),
)
if result_rows is None:
raise RuntimeError("第七阶段查询 1-30 轮结果数据失败")
result_by_req_id = {}
task_round_hits = defaultdict(dict)
for result_row in result_rows:
req_id = str(result_row.get("reqId") or "")
task_id = str(result_row.get("taskId") or "")
result_pt = str(result_row.get("pt") or "")
try:
round_count = int(
result_row.get("round_count") or 0
)
has_keyword = int(
result_row.get("has_keyword") or 0
)
except (TypeError, ValueError):
continue
if (
not req_id
or not task_id
or not 1 <= round_count <= 30
):
continue
current_result = result_by_req_id.setdefault(
req_id,
{
"has_keyword": 0,
},
)
current_result["has_keyword"] = max(
current_result["has_keyword"],
has_keyword,
)
task_key = (result_pt, task_id)
task_round_hits[task_key][round_count] = max(
task_round_hits[task_key].get(round_count, 0),
has_keyword,
)
rows_by_business_line = defaultdict(list)
for log_row in log_rows:
business_line = str(
log_row.get("business_line") or ""
).strip() or "未分类"
rows_by_business_line[business_line].append(log_row)
downloads = []
for business_line in sorted(rows_by_business_line):
business_line_rows = rows_by_business_line[business_line]
rows_by_round = defaultdict(list)
for log_row in business_line_rows:
try:
round_count = int(
log_row.get("round_count") or 0
)
except (TypeError, ValueError):
continue
if 1 <= round_count <= 30:
rows_by_round[round_count].append(log_row)
safe_business_line = _safe_file_name_part(business_line)
file_name = (
f"{pt}_{platform_config['name']}_"
f"{safe_business_line}_30轮明细.xlsx"
)
local_path = os.path.join(
tempfile.gettempdir(),
file_name,
)
headers = [
"问题",
"批次",
"30次是否达标",
"本次是否达标",
"平台",
"回答",
"思考过程",
"引用来源",
"查询时间",
"IP",
"版本号",
]
def build_excel_row(log_row: dict) -> list:
req_id = str(log_row.get("reqId") or "")
task_id = str(log_row.get("taskId") or "")
row_pt = str(log_row.get("pt") or "")
result_item = result_by_req_id.get(req_id)
round_hits = task_round_hits.get(
(row_pt, task_id),
{},
)
if not round_hits:
thirty_round_qualified = "没处理"
else:
hit_round_count = sum(
1
for value in round_hits.values()
if value == 1
)
thirty_round_qualified = (
"是" if hit_round_count >= 24 else "否"
)
if result_item is None:
current_qualified = "没处理"
else:
current_qualified = (
"是"
if int(
result_item.get("has_keyword") or 0
) == 1
else "否"
)
base_path = (
f"geo_snipaste/{row_pt[:8]}/"
f"{tos_directory}/{req_id}"
)
return [
_excel_safe_text(log_row.get("prompt")),
row_pt,
thirty_round_qualified,
current_qualified,
platform_config["name"],
_excel_safe_text(
_safe_get_tos_text(
f"{base_path}/context.txt"
)
),
_excel_safe_text(
_safe_get_tos_text(
f"{base_path}/think.txt"
)
),
_excel_safe_text(
_safe_get_tos_text(
f"{base_path}/quote.txt"
)
),
_timestamp_to_text(log_row.get("insertime")),
_excel_safe_text(
_safe_get_tos_text(
f"{base_path}/inner_user_ip.txt"
)
),
_excel_safe_text(
_safe_get_tos_text(
f"{base_path}/inner_pc_version.txt"
)
),
]
workbook = Workbook()
workbook.remove(workbook.active)
total_rows = 0
with ThreadPoolExecutor(
max_workers=THIRTY_ROUND_EXPORT_WORKERS
) as executor:
for round_count in range(1, 31):
worksheet = workbook.create_sheet(
title=(
f"第"
f"{_chinese_round_number(round_count)}轮"
)
)
worksheet.append(headers)
round_rows = [
row
for row in sorted(
rows_by_round.get(round_count, []),
key=_sheet_order_sort_key,
)
if row.get("reqId")
]
for excel_row in executor.map(
build_excel_row,
round_rows,
):
worksheet.append(excel_row)
total_rows += 1
_style_thirty_round_worksheet(
worksheet,
len(headers),
)
logger.info(
f"[第七阶段 platform={platform} pt={pt} "
f"business_line={business_line} "
f"count={round_count}] 30轮明细 Sheet 完成: "
f"rows={len(round_rows)}"
)
workbook.save(local_path)
workbook.close()
object_key = (
f"geo_snipaste/{pt[:8]}/"
f"{tos_directory}/{file_name}"
)
try:
_upload_thirty_round_excel_to_tos(
file_path=local_path,
object_key=object_key,
)
finally:
try:
os.remove(local_path)
except OSError:
pass
cdn_url = (
f"https://tcdn.aidso.com/"
f"geo_snipaste/{pt[:8]}/"
f"{tos_directory}/{file_name}"
)
downloads.append(
{
"business_line": business_line,
"url": cdn_url,
}
)
logger.success(
f"[第七阶段 platform={platform} pt={pt} "
f"business_line={business_line} count=1-30] "
f"30轮明细导出完成: rows={total_rows}, "
f"url={cdn_url}"
)
if not downloads:
logger.warning(
f"[第七阶段 platform={platform} pt={pt} count=1-30] "
f"没有可导出的30轮业务线数据"
)
return downloads
def build_daily_run_report(
pt: str,
platform: str = "DB",
thirty_round_downloads: list = None,
) -> str:
""" """
汇总当天各阶段运行结果，生成 Markdown 报告。 汇总当天各阶段运行结果，生成 Markdown 报告。
""" """
platform = normalize_platform(platform)
platform_name = PLATFORM_CONFIGS[platform]["name"]
pt = normalize_pt(pt) pt = normalize_pt(pt)
tasks = bh_utils.query_data( tasks = bh_utils.query_data(
f""" f"""
SELECT taskId, prompt, sheet_order SELECT taskId, prompt, sheet_order
FROM {TABLE_NAME} FROM {TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
""", """,
(pt,), (pt, platform),
) )
if tasks is None: if tasks is None:
raise RuntimeError("最后阶段查询当天主任务失败") raise RuntimeError("最后阶段查询当天主任务失败")
result_rows = bh_utils.query_data( result_rows = bh_utils.query_data(
""" f"""
SELECT taskId, `count`, has_keyword SELECT taskId, `count`, has_keyword
FROM douchacha_data.geo_feishu_snipaste_result_v3 FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND 30
""", """,
(int(pt),), (int(pt), platform),
) )
if result_rows is None: if result_rows is None:
raise RuntimeError("最后阶段查询关键词统计结果失败") raise RuntimeError("最后阶段查询关键词统计结果失败")
...@@ -1928,7 +2849,7 @@ def build_daily_run_report(pt: str) -> str: ...@@ -1928,7 +2849,7 @@ def build_daily_run_report(pt: str) -> str:
thirty_round_failed = set(task_map) - thirty_round_qualified thirty_round_failed = set(task_map) - thirty_round_qualified
markdown_lines = [ markdown_lines = [
"## 美团 GEO 今日任务运行总结", f"## {platform_name} GEO 今日任务运行总结",
"", "",
f"**统计日期：** {format_pt_display(pt)}", f"**统计日期：** {format_pt_display(pt)}",
"", "",
...@@ -1940,10 +2861,33 @@ def build_daily_run_report(pt: str) -> str: ...@@ -1940,10 +2861,33 @@ def build_daily_run_report(pt: str) -> str:
f"- 今日问题任务总数：{len(task_map)} 个", f"- 今日问题任务总数：{len(task_map)} 个",
f"- 达标问题：{len(thirty_round_qualified)} 个", f"- 达标问题：{len(thirty_round_qualified)} 个",
f"- 未达标问题：{len(thirty_round_failed)} 个", f"- 未达标问题：{len(thirty_round_failed)} 个",
"",
"### 未达标问题 Top 10",
"",
] ]
if thirty_round_downloads:
markdown_lines.extend(
[
"",
"**30轮数据如下：**",
"",
]
)
for download in thirty_round_downloads:
business_line = str(
download.get("business_line") or "未分类"
)
download_url = str(download.get("url") or "")
if not download_url:
continue
markdown_lines.append(
f"- {business_line}："
f"[点击下载30轮明细]({download_url})"
)
markdown_lines.extend(
[
"",
"### 未达标问题 Top 10",
"",
]
)
def failed_task_sort_key(task): def failed_task_sort_key(task):
task_id = str(task.get("taskId") or "") task_id = str(task.get("taskId") or "")
...@@ -1985,8 +2929,13 @@ def build_daily_run_report(pt: str) -> str: ...@@ -1985,8 +2929,13 @@ def build_daily_run_report(pt: str) -> str:
def send_markdown_report_to_feishu_chat( def send_markdown_report_to_feishu_chat(
markdown_text: str, markdown_text: str,
chat_id: str, chat_id: str,
platform: str = "DB",
) -> str: ) -> str:
card_title = "美团 GEO 今日任务运行总结" platform = normalize_platform(platform)
card_title = (
f"{PLATFORM_CONFIGS[platform]['name']} "
f"GEO 今日任务运行总结"
)
card_markdown_lines = [] card_markdown_lines = []
for line in markdown_text.splitlines(): for line in markdown_text.splitlines():
stripped_line = line.strip() stripped_line = line.strip()
...@@ -2050,40 +2999,51 @@ def send_markdown_report_to_feishu_chat( ...@@ -2050,40 +2999,51 @@ def send_markdown_report_to_feishu_chat(
return message_id return message_id
def run_daily_pipeline(pt: str = None) -> None: def run_daily_pipeline(
pt: str = None,
platform: str = "DB",
) -> None:
""" """
按顺序执行飞书截图任务的第一至第七阶段。 按顺序执行飞书截图任务的第一至第七阶段。
""" """
platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform]
pt = normalize_pt(pt) pt = normalize_pt(pt)
start_count = 1 start_count = 1
end_count = 30 end_count = 30
logger.success( logger.success(
f"[每日任务 pt={pt} count=1-99] 七阶段任务开始执行" f"[每日任务 platform={platform} pt={pt} count=1-99] "
f"七阶段任务开始执行"
) )
# 第一阶段：飞书同步到主表和 30 轮日志表。 # 第一阶段：飞书同步到主表和 30 轮日志表。
sync_feishu_to_database(pt=pt) sync_feishu_to_database(pt=pt, platform=platform)
# 第二阶段：所有任务发送 1-30 轮，完成三轮补发、解析和结果入库。 # 第二阶段：所有任务发送 1-30 轮，完成三轮补发、解析和结果入库。
process_rounds_to_result( process_rounds_to_result(
pt=pt, pt=pt,
start_count=start_count, start_count=start_count,
end_count=end_count, end_count=end_count,
redis_key=REDIS_QUEUE_KEY, redis_key=platform_config["content_redis_key"],
platform=platform,
) )
# 第三阶段：1-30 轮命中不少于 24 轮的任务生成 count=0 分享任务。 # 第三阶段：1-30 轮命中不少于 24 轮的任务生成 count=0 分享任务。
create_and_process_share_rounds( create_and_process_share_rounds(
pt=pt, pt=pt,
redis_key=REDIS_QUEUE_KEY_SHARE, redis_key=platform_config["share_redis_key"],
hit_start_count=1, hit_start_count=1,
hit_end_count=30, hit_end_count=30,
hit_threshold=24, hit_threshold=24,
platform=platform,
) )
# 第四阶段：导出 count=0 的达标分享结果。 # 第四阶段：导出 count=0 的达标分享结果。
excel_path = export_share_rounds_to_excel(pt=pt) excel_path = export_share_rounds_to_excel(
pt=pt,
platform=platform,
)
upload_excel_to_feishu_chat( upload_excel_to_feishu_chat(
file_path=excel_path, file_path=excel_path,
chat_id=FEISHU_CHAT_ID, chat_id=FEISHU_CHAT_ID,
...@@ -2094,15 +3054,19 @@ def run_daily_pipeline(pt: str = None) -> None: ...@@ -2094,15 +3054,19 @@ def run_daily_pipeline(pt: str = None) -> None:
# 第五阶段：1-30 轮命中不足 24 轮的任务生成 count=99 分享任务。 # 第五阶段：1-30 轮命中不足 24 轮的任务生成 count=99 分享任务。
create_and_process_unqualified_share_rounds( create_and_process_unqualified_share_rounds(
pt=pt, pt=pt,
redis_key=REDIS_QUEUE_KEY_SHARE, redis_key=platform_config["share_redis_key"],
hit_start_count=1, hit_start_count=1,
hit_end_count=30, hit_end_count=30,
hit_threshold=24, hit_threshold=24,
platform=platform,
) )
# 第六阶段：导出 count=99 的未达标分享结果。 # 第六阶段：导出 count=99 的未达标分享结果。
unqualified_excel_path = ( unqualified_excel_path = (
export_unqualified_share_rounds_to_excel(pt=pt) export_unqualified_share_rounds_to_excel(
pt=pt,
platform=platform,
)
) )
upload_excel_to_feishu_chat( upload_excel_to_feishu_chat(
file_path=unqualified_excel_path, file_path=unqualified_excel_path,
...@@ -2111,28 +3075,40 @@ def run_daily_pipeline(pt: str = None) -> None: ...@@ -2111,28 +3075,40 @@ def run_daily_pipeline(pt: str = None) -> None:
count_label="99", count_label="99",
) )
# 第七阶段：生成并发送今日运行总结。 # 第七阶段：导出30轮明细，并在今日运行总结中发送下载链接。
markdown_report = build_daily_run_report(pt=pt) thirty_round_downloads = export_thirty_round_details(
pt=pt,
platform=platform,
)
markdown_report = build_daily_run_report(
pt=pt,
platform=platform,
thirty_round_downloads=thirty_round_downloads,
)
send_markdown_report_to_feishu_chat( send_markdown_report_to_feishu_chat(
markdown_text=markdown_report, markdown_text=markdown_report,
chat_id=FEISHU_CHAT_ID, chat_id=FEISHU_CHAT_ID,
platform=platform,
) )
logger.success( logger.success(
f"[每日任务 pt={pt} count=1-99] 七阶段任务执行完成" f"[每日任务 platform={platform} pt={pt} count=1-99] "
f"七阶段任务执行完成"
) )
def run_daily_pipeline_safely(pt: str = None) -> None: def run_daily_pipeline_safely(
pt: str = None,
platform: str = "DB",
) -> None:
try: try:
run_daily_pipeline(pt=pt) run_daily_pipeline(pt=pt, platform=platform)
except Exception: except Exception:
logger.exception( logger.exception(
"[每日任务 count=1-99] 七阶段任务执行失败" f"[每日任务 platform={platform} count=1-99] "
f"七阶段任务执行失败"
) )
if __name__ == "__main__": if __name__ == "__main__":
# run_daily_pipeline('20260729')
start_scheduler() start_scheduler()
# run_daily_pipeline(pt="2026072702")
aidso_geo/core/routes/interface.py
View file @ 4b1f9b3a
...@@ -4,12 +4,16 @@ import json ...@@ -4,12 +4,16 @@ import json
import queue import queue
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from loguru import logger from loguru import logger
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from datetime import datetime from datetime import datetime
sys.path.append(BASE_DIR) sys.path.append(BASE_DIR)
from aidso_geo.models.process import process_call_back, commit_task, main_process from aidso_geo.models.process import process_call_back, commit_task, main_process
from aidso_geo.config.base_config import init_redis, init_redis8 from aidso_geo.config.base_config import init_redis, init_redis8
from aidso_geo.utils import bh_utils, tos_utils, url_utils from aidso_geo.utils import bh_utils, tos_utils, url_utils
from aidso_geo.models.eco_data_process import process_eco_product_relation
line_app = Blueprint("line_app", __name__) line_app = Blueprint("line_app", __name__)
redis_client = init_redis() redis_client = init_redis()
redis_client8 = init_redis8() redis_client8 = init_redis8()
...@@ -323,3 +327,95 @@ def check_quto(): ...@@ -323,3 +327,95 @@ def check_quto():
"reqId": req_id "reqId": req_id
}) })
@line_app.route('/api/geo/check_eco_keyword', methods=['POST'])
def check_eco():
try:
body = request.get_json(silent=True) or {}
reqids = body.get("req_ids")
keyword = body.get("keyword")
if not isinstance(reqids, list):
return jsonify({
"code": 400,
"message": "reqids必须是数组",
"data": None,
})
# 过滤空值并去重，同时保持原顺序。
reqids = list(dict.fromkeys(
str(req_id).strip()
for req_id in reqids
if str(req_id or "").strip()
))
if not reqids:
return jsonify({
"code": 400,
"message": "reqids不能为空",
"data": None,
})
if not isinstance(keyword, str) or not keyword.strip():
return jsonify({
"code": 400,
"message": "keyword必须是非空字符串",
"data": None,
})
keyword = keyword.strip()
placeholders = ", ".join(
["%s"] * len(reqids)
)
result = bh_utils.query_data(
f"""
SELECT *
FROM geo_eco_data
WHERE req_id IN ({placeholders})
""",
tuple(reqids),
)
if not result:
return jsonify({
"code": 200,
"message": "处理成功",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": len(result),
},
})
eco_result = process_eco_product_relation(
result,
keyword,
)
bh_utils.insert_data("geo_eco_data",eco_result)
return jsonify({
"code": 200,
"message": "处理成功",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": len(eco_result),
},
})
except Exception as exc:
return jsonify({
"code": 500,
"message": f"商品关系处理失败: {exc}",
"data": None,
})
aidso_geo/models/eco_data_process.py
View file @ 4b1f9b3a
import re import re
from aidso_geo.utils import bh_utils from aidso_geo.utils import bh_utils,ai_utils
def process_eco_product_relation(
eco_result,
keyword,
batch_size=500,
):
"""
每100条商品调用一次AI，并将识别结果合并到原商品数据中。
produce_name只用于匹配，不写入最终结果。
"""
def normalize_title(value):
return " ".join(
str(value or "").strip().split()
)
final_result = []
for start_index in range(
0,
len(eco_result),
batch_size,
):
batch = eco_result[
start_index:start_index + batch_size
]
product_list = [
str(item.get("eco_title") or "").strip()
for item in batch
]
try:
ai_result = (
ai_utils.ai_get_product_relation_spu(
product_list,
keyword,
)
)
except Exception as exc:
print(
f"AI商品识别失败: "
f"batch={start_index // batch_size + 1}, "
f"error={exc}"
)
ai_result = []
if not isinstance(ai_result, list):
ai_result = []
# produce_name只用于建立匹配关系。
ai_result_map = {}
for ai_item in ai_result:
if not isinstance(ai_item, dict):
continue
produce_name = str(
ai_item.get("produce_name") or ""
).strip()
normalized_name = normalize_title(
produce_name
)
if not normalized_name:
continue
if normalized_name not in ai_result_map:
ai_result_map[normalized_name] = ai_item
for original_item in batch:
eco_title = str(
original_item.get("eco_title") or ""
).strip()
matched_item = ai_result_map.get(
normalize_title(eco_title)
)
# 未匹配时的默认值。
relation_data = {
"brand": eco_title,
"spu_name": "",
"current": 0,
}
if isinstance(matched_item, dict):
brand = str(
matched_item.get("brand")
or eco_title
).strip()
spu_name = str(
matched_item.get("spu_name")
or ""
).strip()
try:
current = int(
matched_item.get("current", 0)
)
except (TypeError, ValueError):
current = 0
relation_data = {
"brand": brand,
"search_goods_word":keyword,
"spu_name": brand+" "+spu_name,
"current": (
1 if current == 1 else 0
),
}
final_result.append(
{
**original_item,
**relation_data,
}
)
return final_result
def extract_price(price): def extract_price(price):
if not price: if not price:
...@@ -234,6 +357,7 @@ def yuanbao_android_process_txmap_eco(data,eco_list): ...@@ -234,6 +357,7 @@ def yuanbao_android_process_txmap_eco(data,eco_list):
def save_eco_data_to_bh(data,eco_type,eco_list): def save_eco_data_to_bh(data,eco_type,eco_list):
platform = data.get('platform') platform = data.get('platform')
search_goods_word = data.get('search_goods_word')
eco_result = [] eco_result = []
if platform == 'TYQWA': if platform == 'TYQWA':
...@@ -259,6 +383,10 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -259,6 +383,10 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
if eco_type == 'txmap': if eco_type == 'txmap':
eco_result = yuanbao_process_txmap_eco(data,eco_list) eco_result = yuanbao_process_txmap_eco(data,eco_list)
if search_goods_word:
eco_result = process_eco_product_relation(eco_result, search_goods_word)
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
aidso_geo/models/process.py
View file @ 4b1f9b3a
...@@ -2032,9 +2032,9 @@ if __name__ == '__main__': ...@@ -2032,9 +2032,9 @@ if __name__ == '__main__':
type_t = i.get('type') type_t = i.get('type')
type_t = 'batch' type_t = 'batch'
# commit_task(i,'ING') # commit_task(i,'ING')
return task_send_queue(i,type_t) # return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
# return platform_process(i) return platform_process(i)
# #
if data_list: if data_list:
with ThreadPoolExecutor(max_workers=50) as executor: with ThreadPoolExecutor(max_workers=50) as executor:
......
aidso_geo/models/qianwen_data_process.py
View file @ 4b1f9b3a
...@@ -33,7 +33,6 @@ def qianwen_process_original_data(data): ...@@ -33,7 +33,6 @@ def qianwen_process_original_data(data):
data_str = i.split("data:")[1] data_str = i.split("data:")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
...@@ -146,6 +145,19 @@ def qianwen_process_original_data(data): ...@@ -146,6 +145,19 @@ def qianwen_process_original_data(data):
if paas: if paas:
for pa in ms.get('meta_data').get('paas'): for pa in ms.get('meta_data').get('paas'):
suggestions.append(pa.get('show_text')) suggestions.append(pa.get('show_text'))
if ms.get('mime_type') == 'bar/workflow' and ms.get('status') == 'complete':
ms_meta_data_multi_load = ms.get('meta_data').get('multi_load')
if isinstance(ms_meta_data_multi_load,list):
for i in ms_meta_data_multi_load:
ms_meta_data_multi_load_type =i.get('type')
if ms_meta_data_multi_load_type == 'bar_thinking':
think_content+=i.get('content').get('body')
if ms_meta_data_multi_load_type == 'bar_ref_source_inline':
if i.get('content'):
if i.get('content').get('query_list'):
search_keyword.extend(i.get('content').get('query_list'))
if i.get('content').get('docs'):
url_list_batch.extend(i.get('content').get('docs'))
if url_list_batch: if url_list_batch:
for url in url_list_batch: for url in url_list_batch:
...@@ -163,7 +175,6 @@ def qianwen_process_original_data(data): ...@@ -163,7 +175,6 @@ def qianwen_process_original_data(data):
suggestions,rich_media_block) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e: except Exception as e:
traceback.print_exc()
parts = file_path.split('/') parts = file_path.split('/')
platform = parts[2] platform = parts[2]
task_id = parts[1] task_id = parts[1]
...@@ -195,7 +206,7 @@ if __name__ == '__main__': ...@@ -195,7 +206,7 @@ if __name__ == '__main__':
# # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text' # # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
# qianwen_process_original_data(file_path2) # qianwen_process_original_data(file_path2)
# for i in task_id: # for i in task_id:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '13e740b9-955c-4f9b-a03e-92e656ffaf43' and platform = 'TYQW'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '9fd93209-01db-4dd2-84e3-02eeccbedb16' and platform = 'TYQW'")
# # # # # #
# # # # # #
......
aidso_geo/utils/ai_utils.py
View file @ 4b1f9b3a
...@@ -3,9 +3,6 @@ import time ...@@ -3,9 +3,6 @@ import time
import requests import requests
import json import json
from aidso_geo.core.down_load_bot import get_req_id
from aidso_geo.utils import bh_utils
from aidso_geo.utils.tos_utils import get_string_from_tos from aidso_geo.utils.tos_utils import get_string_from_tos
...@@ -144,6 +141,126 @@ def ai_get_product_list(content, prompt): ...@@ -144,6 +141,126 @@ def ai_get_product_list(content, prompt):
except Exception as e: except Exception as e:
return [] return []
def ai_get_product_relation_spu(product_list, keyword):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
payload = {
"model": "doubao-seed-2-0-mini-260428",
"messages": [
{
"role": "system",
"content": """
你的核心任务为：基于给定的商品列表与查询关键词，为列表内的每一件商品匹配对应信息，最终输出符合规范要求的JSON结果，执行过程需严格遵循以下规则：
1. 需为商品列表中的每一件商品单独生成一条匹配记录，每条记录必须固定包含以下4个字段，各字段的取值规则明确如下：
（1）produce_name：填写对应商品的完整原始名称，即商品列表中给出的该商品全称，不得做任何增删修改；
（2）brand：填写该商品对应的SPU品牌名称，需精准识别商品所属品牌，参考示例：“北京同仁堂陈皮茯苓茶”的品牌取值为“北京同仁堂”，“红魔11 Pro+”的品牌取值为“红魔”；若商品无明确可识别的品牌信息，则直接返回该商品的完整名称作为brand字段值；
（3）spu_name：填写该商品的标准SPU名称，即去除品牌前缀后的商品核心名称，参考示例：“北京同仁堂陈皮茯苓茶”的spu_name取值为“陈皮茯苓茶”，“一加 Ace 6 至尊版”的spu_name取值为“Ace 6 至尊版”；
（4）current：判断该商品与给定查询关键词是否存在关联，关联判定范围包括但不限于：关键词为该商品的品牌名、关键词为该品牌旗下的子品牌/系列名称、商品属于该关键词对应的品牌产品线；只要满足上述任意一种关联情形，该字段取值为1，若不存在任何关联则取值为0。
2. 一致性校验特别要求：同批次传入的商品中，若商品标题指向的品牌名相同，brand字段的取值必须保持完全统一，禁止出现同一品牌同时标注“小米”和“xiaomi”这类中英文/不同写法混用的情况，需统一为规范名称；同批次商品的spu_name也需保持表述一致，禁止出现同一SPU同时标注“12 promax”和“12promax”这类格式不统一的情况，需统一为规范表述。
{
"produce_name": "郎酒 红花郎15",
"brand": "郎酒",
"spu_name": "红花郎15",
"current": 0
},
"""
},
{
"role": "user",
"content": f"""需要处理的商品列表：{product_list} 本次查询的关键词为：{keyword}"""
}
],
"thinking": {
"type": "disabled"
},
"temperature": 0,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "product_relation_result",
"strict": True,
"schema": {
"type": "object",
"properties": {
"product_words": {
"type": "array",
"description": "商品识别和关联判断结果",
"items": {
"type": "object",
"properties": {
"produce_name": {
"type": "string",
"description": "商品完整原始名称",
},
"brand": {
"type": "string",
"description": "商品所属品牌",
},
"spu_name": {
"type": "string",
"description": "去除品牌后的标准SPU名称",
},
"current": {
"type": "integer",
"enum": [0, 1],
"description": "与查询关键词有关为1，否则为0",
},
},
"required": [
"produce_name",
"brand",
"spu_name",
"current",
],
"additionalProperties": False,
},
},
},
"required": ["product_words"],
"additionalProperties": False,
},
},
},
}
headers = {
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
try:
response = requests.post(
url,
headers=headers,
json=payload,
timeout=120,
)
if response.status_code != 200:
print("status_code:", response.status_code)
print("response_text:", response.text)
response.raise_for_status()
response_data = response.json()
content = (
response_data
.get("choices", [{}])[0]
.get("message", {})
.get("content")
)
if not content:
return []
if isinstance(content, str):
content = json.loads(content)
product_words = content.get("product_words", [])
return product_words if isinstance(product_words, list) else []
except Exception as e:
print(e)
return []
def ai_get_product_list_search(product_list): def ai_get_product_list_search(product_list):
url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions" url = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
payload = json.dumps({ payload = json.dumps({
...@@ -989,16 +1106,16 @@ render_ecom_card_widget_jd_start: ...@@ -989,16 +1106,16 @@ render_ecom_card_widget_jd_start:
render_ecom_card_widget_jd_end: render_ecom_card_widget_jd_end:
""" """
brand_list =[ product_list =[
"瓜子二手车","二手车之家" "瓜子二手车","二手车之家"
# fcc424e5-58af-494d-9683-5787413a26c9 # fcc424e5-58af-494d-9683-5787413a26c9
] ]
promp = """ keyword = """
空调选购 瓜子
""" """
print(ai_get_product_list(content,promp)) print(ai_get_product_relation_spu(product_list,keyword))
# print(ai_result) # print(ai_result)
# pro = # pro =
# ai_get_product_list() # ai_get_product_list()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment