Skip to content

A
aidso-data
Commit 5c5131e5 authored by Yaowentong's avatar Yaowentong
Browse files
Options

豆包deepseek双端增加 引用来源区分 引用和 搜索

parent 49195aab
Show whitespace changes
Inline Side-by-side
Showing with 188 additions and 22 deletions
aidso_geo/models/deepseek_android_data_process.py
View file @ 5c5131e5
...@@ -214,7 +214,7 @@ def deepseek_android_process_original_data(data): ...@@ -214,7 +214,7 @@ def deepseek_android_process_original_data(data):
used_ids.add(next_id) used_ids.add(next_id)
next_id += 1 next_id += 1
think_content = think_content.replace('DEEP_SEARCH','')
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions) suggestions)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -242,7 +242,7 @@ def deepseek_android_process_original_data(data): ...@@ -242,7 +242,7 @@ def deepseek_android_process_original_data(data):
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
if __name__ == '__main__': if __name__ == '__main__':
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'dd37fa86e6dd49c4b0617ceb47cea25c' and platform = 'DPA'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'b188155e-7c55-44ed-955b-d369431fe22f' and platform = 'DP'")
# # # # # #
# # # # # #
......
aidso_geo/models/deepseek_data_process.py
View file @ 5c5131e5
import json import json
import traceback import traceback
import re
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos, put_string_to_tos from aidso_geo.utils.tos_utils import get_string_from_tos, put_string_to_tos
...@@ -26,7 +26,7 @@ def deepseek_process_original_data(data): ...@@ -26,7 +26,7 @@ def deepseek_process_original_data(data):
think_bool = False think_bool = False
suggestions = [] suggestions = []
response_bool = False response_bool = False
url_id_map = {}
# 按空行分割内容，过滤空字符串 # 按空行分割内容，过滤空字符串
try: try:
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
...@@ -44,6 +44,11 @@ def deepseek_process_original_data(data): ...@@ -44,6 +44,11 @@ def deepseek_process_original_data(data):
# 提取并解析JSON数据 # 提取并解析JSON数据
data_str = item.split("data: ")[1] data_str = item.split("data: ")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
print(json_data)
print('----')
print('----')
print('----')
print('----')
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue # 跳过格式错误的数据 continue # 跳过格式错误的数据
...@@ -51,12 +56,25 @@ def deepseek_process_original_data(data): ...@@ -51,12 +56,25 @@ def deepseek_process_original_data(data):
if json_data.get('v').get('response').get('thinking_enabled') == True: if json_data.get('v').get('response').get('thinking_enabled') == True:
is_think=True is_think=True
if json_data.get('v').get('response').get('search_enabled') == True: fragments = json_data.get('v').get('response').get('fragments')
if len(json_data.get('v').get('response').get('fragments'))>0: if fragments:
query_list = json_data.get('v').get('response').get('fragments')[0].get('queries') if isinstance(fragments, list):
if query_list: if fragments[0].get('type') == 'RESPONSE':
result = [item.get('query', '') for item in query_list] response_content += fragments[0].get('content')
search_keyword.extend(result) if fragments[0].get('type') == 'THINK':
think_content += fragments[0].get('content')
think_bool = True
if fragments[0].get('type') == 'SEARCH':
if isinstance(fragments[0].get('queries'), list):
for sea in fragments[0].get('queries'):
search_keyword.append(sea.get('query'))
# if json_data.get('v').get('response').get('search_enabled') == True:
# if len(json_data.get('v').get('response').get('fragments'))>0:
# query_list = json_data.get('v').get('response').get('fragments')[0].get('queries')
# if query_list:
# result = [item.get('query', '') for item in query_list]
# search_keyword.extend(result)
if is_think: if is_think:
if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'SEARCH': if json_data.get('p') == 'response/fragments' and json_data.get('v')[0].get('type') == 'SEARCH':
...@@ -75,6 +93,7 @@ def deepseek_process_original_data(data): ...@@ -75,6 +93,7 @@ def deepseek_process_original_data(data):
if json_data.get('v')[0].get('v')[0].get('type') == 'TOOL_SEARCH': if json_data.get('v')[0].get('v')[0].get('type') == 'TOOL_SEARCH':
for q in json_data.get('v')[0].get('v')[0].get('queries'): for q in json_data.get('v')[0].get('v')[0].get('queries'):
search_keyword.append(q.get('query')) search_keyword.append(q.get('query'))
if json_data.get('v')[1].get('p') == 'fragments' or json_data.get('v')[1].get('p') == 'response/fragments': if json_data.get('v')[1].get('p') == 'fragments' or json_data.get('v')[1].get('p') == 'response/fragments':
if json_data.get('v')[1].get('v')[0].get('type') == 'THINK': if json_data.get('v')[1].get('v')[0].get('type') == 'THINK':
think_content+=json_data.get('v')[1].get('v')[0].get('content') think_content+=json_data.get('v')[1].get('v')[0].get('content')
...@@ -94,6 +113,47 @@ def deepseek_process_original_data(data): ...@@ -94,6 +113,47 @@ def deepseek_process_original_data(data):
continue continue
if json_data.get('p') == 'response/fragments/1/elapsed_secs': if json_data.get('p') == 'response/fragments/1/elapsed_secs':
think_bool = False think_bool = False
if isinstance(json_data.get('v'),list):
v_json_data = json_data.get('v')
has_tool_search = False
tool_search_id = None
for item in v_json_data:
if item.get('p') == 'references':
references = item.get('v') or []
if isinstance(references, list):
for reference in references:
if (
isinstance(reference, dict)
and reference.get('type') == 'TOOL_OPEN'
):
has_tool_search = True
tool_search_id = reference.get('id')
break
for v in v_json_data:
if v.get('p') == 'content' and has_tool_search:
p_content = v.get('v') or ''
p_content = re.sub(
r'\[reference:\d+\]',
f'[reference:{tool_search_id}]',
p_content
)
response_content+=p_content
if v.get('p') == 'fragments':
if v.get('p') == 'fragments':
fragments = v.get('v') or []
if isinstance(fragments, list):
for fragment in fragments:
if (
isinstance(fragment, dict)
and fragment.get('type') == 'TOOL_OPEN'
and fragment.get('result') is not None
):
url_id_map[fragment.get('result').get('url')] = fragment.get('id')
if json_data.get('p') == 'response/fragments/-1/elapsed_secs': if json_data.get('p') == 'response/fragments/-1/elapsed_secs':
think_bool = False think_bool = False
...@@ -137,6 +197,23 @@ def deepseek_process_original_data(data): ...@@ -137,6 +197,23 @@ def deepseek_process_original_data(data):
if response_bool: if response_bool:
if isinstance(json_data.get('v'),str): if isinstance(json_data.get('v'),str):
response_content+=json_data.get('v') response_content+=json_data.get('v')
used_ids = set(url_id_map.values())
next_id = 1
for item in url_list:
url = item.get('url')
if url in url_id_map:
item['cite_index'] = url_id_map[url]
elif item.get('cite_index') is None:
while next_id in used_ids:
next_id += 1
item['cite_index'] = next_id
used_ids.add(next_id)
next_id += 1
think_content = think_content.replace('DEEP_SEARCH','')
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions) suggestions)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -165,5 +242,29 @@ def deepseek_process_original_data(data): ...@@ -165,5 +242,29 @@ def deepseek_process_original_data(data):
# #
if __name__ == '__main__': if __name__ == '__main__':
file_path = "geo/a374ca2e-eb45-4c2d-badd-9219af98ff38/DP/original.text" data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'b188155e-7c55-44ed-955b-d369431fe22f' and platform = 'DP'")
deepseek_process_original_data(file_path)
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return deepseek_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/models/doubao_android_data_process.py
View file @ 5c5131e5
...@@ -81,6 +81,23 @@ def doubao_mobile_process_original_data(task_data): ...@@ -81,6 +81,23 @@ def doubao_mobile_process_original_data(task_data):
# if content_block[0].get('is_finish') and content_block[0].get('parent_id'): # if content_block[0].get('is_finish') and content_block[0].get('parent_id'):
# think_bool = True # think_bool = True
# response_bool = False # response_bool = False
if content_block[0].get('block_type') == 10000 and content_block[0].get('meta_info'):
meta_info = json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"meta_info")
for meta in meta_info:
target_title = json.loads(meta.get('info')).get('title')
index = next(
(
item["index"]
for item in url_list
if item.get("title") == target_title
),
None
)
if index:
response_content+=f"[reference:{index}]"
if content_block[0].get('block_type') == 10025: if content_block[0].get('block_type') == 10025:
if think_bool: if think_bool:
...@@ -251,6 +268,7 @@ def doubao_mobile_process_original_data(task_data): ...@@ -251,6 +268,7 @@ def doubao_mobile_process_original_data(task_data):
if content_block[0].get('block_type') == 10000 and content_block[0].get( if content_block[0].get('block_type') == 10000 and content_block[0].get(
'patch_type') == 1: 'patch_type') == 1:
text_block = content_block[0].get('content').get('text_block').get('text', '') text_block = content_block[0].get('content').get('text_block').get('text', '')
response_content += text_block response_content += text_block
...@@ -306,7 +324,6 @@ def doubao_mobile_process_original_data(task_data): ...@@ -306,7 +324,6 @@ def doubao_mobile_process_original_data(task_data):
if dy_eco_list: if dy_eco_list:
save_eco_data_to_bh(task_data,'dyeco',dy_eco_list) save_eco_data_to_bh(task_data,'dyeco',dy_eco_list)
url_list = new_url_list url_list = new_url_list
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -343,7 +360,7 @@ if __name__ == '__main__': ...@@ -343,7 +360,7 @@ if __name__ == '__main__':
# file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt' # file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '3f53b599-3710-4312-9b50-73f07bdba631' and platform = 'DOUBA'") f"select * from geo_commit_task where taskId = '019cd67a-7cef-4390-934e-500719a16ec8' and platform = 'DOUBA'")
......
aidso_geo/models/doubao_data_process.py
View file @ 5c5131e5
...@@ -104,6 +104,26 @@ def doubao_process_original_data(data): ...@@ -104,6 +104,26 @@ def doubao_process_original_data(data):
"content").get("search_query_result_block").get("summary") "content").get("search_query_result_block").get("summary")
think_content += "**" think_content += "**"
think_content += "\n\n" think_content += "\n\n"
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") == 10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"meta_info"):
meta_info = json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"meta_info")
for meta in meta_info:
target_title = json.loads(meta.get('info')).get('title')
index = next(
(
item["text_card"]["index"]
for item in url_list
if item.get("text_card", {}).get("title") == target_title
),
None
)
if index:
response_content+=f"[reference:{index}]"
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( "block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"parent_id") and len(json_content.get('patch_op'))>1: "parent_id") and len(json_content.get('patch_op'))>1:
...@@ -132,6 +152,7 @@ def doubao_process_original_data(data): ...@@ -132,6 +152,7 @@ def doubao_process_original_data(data):
for i in content_block[0].get('content').get('rich_media_block').get('creations'): for i in content_block[0].get('content').get('rich_media_block').get('creations'):
rich_media_block.append(i.get('video')) rich_media_block.append(i.get('video'))
if content_block[0].get('block_type') ==10000: if content_block[0].get('block_type') ==10000:
if content_block[0].get("content").get("text_block").get("text"): if content_block[0].get("content").get("text_block").get("text"):
response_content = content_block[0].get("content").get("text_block").get("text") response_content = content_block[0].get("content").get("text_block").get("text")
...@@ -183,9 +204,32 @@ if __name__ == '__main__': ...@@ -183,9 +204,32 @@ if __name__ == '__main__':
# 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35' # 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35'
# ] # ]
# for task in task_id_list: # for task in task_id_list:
task = 'a090ebc2-8f74-4533-915c-f7352658c42e' data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'e5b63f95-f713-4c51-b282-78c8c1c130ad' and platform = 'DB'")
file_path = f'geo/{task}/DB/original.text'
doubao_process_original_data(file_path) # # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return doubao_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/models/process.py
View file @ 5c5131e5
...@@ -2009,20 +2009,24 @@ if __name__ == '__main__': ...@@ -2009,20 +2009,24 @@ if __name__ == '__main__':
# #
# data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId in ({req_id_sql}) ") # data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId in ({req_id_sql}) ")
data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DPA' and thinking_enabled = 1 and insertime > 1782021666 order by insertime desc") # data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status = 'ING'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ") # data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
def handle_item(i): def handle_item(i):
logger.success(f"{i.get('taskId')} 处理成功")
i["comWordsMap"] = safe_json_loads(i.get("comWordsMap"), []) i["comWordsMap"] = safe_json_loads(i.get("comWordsMap"), [])
i["brandWords"] = safe_json_loads(i.get("brandWords"), []) i["brandWords"] = safe_json_loads(i.get("brandWords"), [])
i["comWords"] = safe_json_loads(i.get("comWords"), []) i["comWords"] = safe_json_loads(i.get("comWords"), [])
i["keywords"] = safe_json_loads(i.get("keywords"), []) i["keywords"] = safe_json_loads(i.get("keywords"), [])
i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), []) i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), [])
type_t = i.get('type') type_t = i.get('type')
# type_t = 'batch' type_t = 'batch'
# commit_task(i,'ING') # commit_task(i,'ING')
# return task_send_queue(i,type_t) return task_send_queue(i,type_t)
return deepseek_android_data_process.deepseek_android_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
# return platform_process(i) # return platform_process(i)
# #
if data_list: if data_list:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment