Skip to content

A
aidso-data
Commit 916b7b37 authored by Yaowentong's avatar Yaowentong
Browse files
Options

修复

parent a385cade
Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 24 deletions
aidso_geo/models/doubao_android_data_process.py
View file @ 916b7b37
......@@ -2,7 +2,7 @@ import json
import traceback
from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils
from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
......@@ -299,12 +299,10 @@ def doubao_mobile_process_original_data(data):
url_list = new_url_list
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e:
# traceback.print_exc()
parts = file_path.split('/')
platform = parts[2]
task_id = parts[1]
......@@ -332,12 +330,34 @@ if __name__ == '__main__':
# poi 21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6 0提及
# 电商+poi 21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6 0提及
aa = ['21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6']
# aa = ['667b7528-5ff4-4ccb-bb98-7d0b5972014d']
# file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
for a in aa:
file_path = f'geo/{a}/DOUBA/original.text'
doubao_mobile_process_original_data(file_path)
# print(file_path)
# file_path = 'geo/da4ee3sssa2-4c8e-ccc4776-aaaaaaa587-40asas7543ssssssaf6b9d//original.text'
# file_path = 'geo/da4ee3sssa2aaaa-4c8e-ccc4776-aaaaaaa587-40asas7543ssssssaf6b9d/DOUBA/original.text'
data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '48d5b7e0-49e8-491b-8719-bcb82daee835' and platform = 'DOUBA'")
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return doubao_mobile_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/models/qianwen_data_process.py
View file @ 916b7b37
import json
import traceback
from aidso_geo.utils import robot_utils
from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
from aidso_geo.models import spider_save_tos
......@@ -32,7 +32,6 @@ def qianwen_process_original_data(data):
data_str = i.split("data:")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue
......@@ -62,8 +61,7 @@ def qianwen_process_original_data(data):
think_content = json.loads(cn.get('content')).get('content')
if cn.get('contentType') == 'text':
response_content = cn.get('content')
if json_data.get("data"):
if json_data.get("data"):
if json_data.get("data").get('status'):
if json_data.get("data").get('status') == 'complete':
......@@ -76,10 +74,11 @@ def qianwen_process_original_data(data):
if cn.get('mime_type') == 'multi_load/iframe' and cn.get('status') == 'complete':
response_content = cn.get('content')
response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
multi_load = cn.get('meta_data').get('multi_load')
if multi_load:
for mu in multi_load:
if mu.get('type') == 'deep_think':
if mu.get('type') == 'deep_think' or mu.get('type') == 'multimodal_chat_think':
if mu.get('content').get('status') =='complete':
think_content = mu.get('content').get('think_content')
......@@ -88,16 +87,27 @@ def qianwen_process_original_data(data):
for ms in messages:
if ms.get('mime_type') =='multi_load/iframe' and ms.get('status') == 'complete':
response_content = ms.get('content')
response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
multi_load = ms.get('meta_data').get('multi_load')
if multi_load:
for mu in multi_load:
if mu.get('type') == 'deep_think':
if mu.get('type') == 'deep_think' or mu.get('type') == 'multimodal_chat_think':
if mu.get('content').get('status') == 'complete':
think_content = mu.get('content').get('think_content')
if mu.get('type') == 'ref_source_inline':
mu_content = mu.get('content')
if mu_content:
if mu_content.get('query_list'):
search_keyword.extend(mu_content.get('query_list'))
if mu_content.get('list'):
url_list_batch.extend(mu_content.get('list'))
if ms.get('mime_type') =='bar/iframe' and ms.get('status') == 'complete':
url_list_batch = ms.get('meta_data').get('sources')[0].get('content').get('list')
if ms.get('meta_data').get('sources'):
url_list_batch = ms.get('meta_data').get('sources')[0].get('content').get('list')
if ms.get('mime_type') =='paa/iframe' and ms.get('status') == 'complete':
paas = ms.get('meta_data').get('paas')
if paas:
......@@ -116,6 +126,7 @@ def qianwen_process_original_data(data):
"time": url.get('publish_time', ''),
}
)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
......@@ -141,17 +152,42 @@ def qianwen_process_original_data(data):
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
#
if __name__ == '__main__':
file_path2 = 'geo/21fd5xxx37f-131b-469axxxxx-9d5c-6ea00da2c0d6/TYQW/original.text'
# file_path2 = 'geo/jqk/TYQW/2.txt'
# /geo/900f0ea6e9a34c95b6b57aa7519a4820/
# file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
qianwen_process_original_data(file_path2)
# file_path2 = 'geo/21fd5xxx37f-131b-469axxxxx-9d5c-6ea00da2c0d6/TYQW/original.text'
# # file_path2 = 'geo/jqk/TYQW/2.txt'
# # /geo/900f0ea6e9a34c95b6b57aa7519a4820/
# # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
# qianwen_process_original_data(file_path2)
# for i in task_id:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '16e53643-b1ac-4566-a0aa-68340576a3ff' and platform = 'TYQW'")
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return qianwen_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
# plan_cot/post and status =='complete' 思考过程前
# bar/progress and type = cot 搜索关键词
# bar/progress and type = sources 引用来源
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment