Skip to content

A
aidso-data
Commit cf03ab12 authored by Yaowentong's avatar Yaowentong
Browse files
Options

元宝千问修复

parent d521b79d
Show whitespace changes
Inline Side-by-side
Showing with 410 additions and 268 deletions
aidso_geo/core/hua.py
View file @ cf03ab12
from venv import logger
import tldextract as tldextract
from openpyxl import load_workbook
......
aidso_geo/models/process.py
View file @ cf03ab12
......@@ -1910,43 +1910,38 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
if __name__ == '__main__':
# data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId = '84db7b12-88e7-4aaa-bc58-8d61f1051934'")
# # data_list = bh_utils.query_data(query_sql)
# # print(data_list)
# # # #
# # # #
# # # # # #
# def handle_item(i):
# if i.get('comWordsMap'):
# i['comWordsMap'] = json.loads(i.get('comWordsMap'))
# if i.get('brandWords'):
# i['brandWords'] = json.loads(i.get('brandWords'))
# if i.get('comWords'):
# i['comWords'] = json.loads(i.get('comWords'))
# if i.get('keywords'):
# i['keywords'] = json.loads(i.get('keywords'))
# if i.get('productWordsMap'):
# i['productWordsMap'] = json.loads(i.get('productWordsMap'))
# type_t = i.get('type')
# # type_t = 'batch'
#
# # return task_send_queue(i,type_t)
# return platform_process(i)
#
# if data_list:
# with ThreadPoolExecutor(max_workers=30) as executor:
# futures = [executor.submit(handle_item, i) for i in data_list]
#
# for future in as_completed(futures):
# try:
# future.result()
# except Exception as e:
# logger.exception(f"platform_process 执行异常: {e}")
PAGE_SIZE =1000
MAX_WORKERS =50
run_data(PAGE_SIZE,MAX_WORKERS)
text = """
data_list = bh_utils.query_data(f"select * from geo_commit_task where status ='ING' ")
# data_list = bh_utils.query_data(query_sql)
# print(data_list)
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
type_t = i.get('type')
# type_t = 'batch'
# return task_send_queue(i,type_t)
return platform_process(i)
if data_list:
with ThreadPoolExecutor(max_workers=30) as executor:
futures = [executor.submit(handle_item, i) for i in data_list]
for future in as_completed(futures):
try:
future.result()
except Exception as e:
logger.exception(f"platform_process 执行异常: {e}")
"""
# keywords = []
# get_keyword_ranks()
aidso_geo/models/qianwen_android_data_process.py
View file @ cf03ab12
......@@ -10,6 +10,7 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
def qianwen_android_process_original_data(data):
# def qianwen_android_process_original_data(file_path):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = []
url_list_batch = []
......@@ -23,8 +24,71 @@ def qianwen_android_process_original_data(data):
response_bool = False
result = []
try:
json_original = ""
original_content = get_string_from_tos(file_path)
try:
json_original = json.loads(original_content)
sse_json_data = json.loads(json_original.get('sse'))
response_content = sse_json_data.get('reply_text','')
think_content = sse_json_data.get('deepthink_text','')
paa_answer = sse_json_data.get('paa_answer','')
bar_sources = sse_json_data.get('bar_sources')
source_content =next(
(item.get('content') for item in bar_sources if item.get('type') == 'source'),
{}
)
source_list = source_content.get('list', [])
video_note_list = sse_json_data.get('video_note_list',{}).get('list')
video_list = []
if video_note_list:
for i in video_note_list:
rich_media = {
'zhidaye_id':i.get('zhidaye_id'),
'title':i.get('title'),
'cover':i.get('cover'),
'url':i.get('url'),
'author':i.get('author'),
'publishTime':i.get('publishTime'),
'duration':i.get('duration'),
}
video_list.append(rich_media)
rich_media_block.append({
"utl":video_list,
"source_seq":''
})
source_count_list = []
if source_list:
for s in source_list:
source_url = {
'url':s.get('url'),
'title':s.get('title'),
'summary':s.get('summary'),
'publish_time':s.get('publish_time'),
'name':s.get('name'),
'icon':s.get('icon'),
'source_seq':s.get('source_seq',''),
}
source_count_list.append(source_url)
url_list.append({
"url":[
{
"type":"source",
"content":{
"list":source_count_list
},
}
],
"source_seq":''
})
if paa_answer:
paa_json = json.loads(paa_answer)
for paa in paa_json:
suggestions.append(paa.get('paa_query'))
except Exception as e:
content_list = original_content.split("\n")
for i in content_list:
......@@ -35,7 +99,7 @@ def qianwen_android_process_original_data(data):
data_str = i.split("data:")[1]
json_data = json.loads(data_str)
#
except (IndexError, json.JSONDecodeError):
continue
......@@ -208,6 +272,9 @@ def qianwen_android_process_original_data(data):
if url_list_batch:
url_list = url_list_batch
response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
......@@ -240,17 +307,13 @@ if __name__ == '__main__':
#https://tcdn.aidso.com/geo/6fede9d0-ce6c-4c97-bcb8-43c499fc24e5/TYQWA/think.txt?secret=1104ed37df6abefbb2b454530b06515f0f59505d5751504e524855
# 6fede9d0-ce6c-4c97-bcb8-43c499fc24e5
file_path2 = 'geo/0a2ed990b43a4f389ecdc14d185a3866/TYQWA/original.text'
# file_path2 = 'geo/ef586bf6-55d8-4e2a-8fb0-5fd1beb5bf7c/TYQWA/original.text'
# qianwen_android_process_original_data(file_path2)
file_path2 = 'geo/alskdjhfasldjkfh/TYQWA/original.text'
# file_path2 = 'geo/ef66d5f9-cadb-43f3-b8cf-824d75a81e7c/TYQWA/original.text'
qianwen_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'd652c3d0-484b-46b6-aefc-1ccd58c46a51' and platform = 'TYQWA'")
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0a2ed990b43a4f389ecdc14d185a3866' and platform = 'TYQWA'")
# data_list = bh_utils.query_data(query_sql)
# print(data_list)
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
......
aidso_geo/models/qianwen_data_process.py
View file @ cf03ab12
......@@ -15,6 +15,7 @@ def qianwen_process_original_data(data):
response_content = ""
search_keyword = []
suggestions = []
rich_media_block =[]
is_think = False
think_bool = False
response_bool = False
......@@ -32,6 +33,7 @@ def qianwen_process_original_data(data):
data_str = i.split("data:")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue
......@@ -85,7 +87,9 @@ def qianwen_process_original_data(data):
if json_data.get("data").get('messages'):
messages = json_data.get("data").get('messages')
for ms in messages:
if ms.get('mime_type') =='multi_load/iframe' and ms.get('status') == 'complete':
response_content = ms.get('content')
response_content = response_content.replace("[(deep_think)]", "")
......@@ -97,13 +101,20 @@ def qianwen_process_original_data(data):
if mu.get('content').get('status') == 'complete':
think_content = mu.get('content').get('think_content')
if mu.get('type') == 'ref_source_inline':
mu_content = mu.get('content')
if mu_content:
if mu_content.get('query_list'):
search_keyword.extend(mu_content.get('query_list'))
if mu_content.get('list'):
url_list_batch.extend(mu_content.get('list'))
if mu.get('type') == 'video_note_list':
rich_media_block.append(
{
"url": mu.get('content').get('list'),
"source_seq": mu.get('source_seq')
}
)
if ms.get('mime_type') =='bar/iframe' and ms.get('status') == 'complete':
if ms.get('meta_data').get('sources'):
......@@ -128,7 +139,7 @@ def qianwen_process_original_data(data):
)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions)
suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e:
parts = file_path.split('/')
......@@ -162,7 +173,7 @@ if __name__ == '__main__':
# # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
# qianwen_process_original_data(file_path2)
# for i in task_id:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '16e53643-b1ac-4566-a0aa-68340576a3ff' and platform = 'TYQW'")
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '3e317dd8-2512-476c-a31a-104c0bcaca75' and platform = 'TYQW'")
# # #
# # #
......
aidso_geo/models/spider_save_tos.py
View file @ cf03ab12
......@@ -73,6 +73,22 @@ def doubao_android_process_rich_media(url_list):
quto_list.append(raw_data)
return quto_list
def yuanbao_android_process_rich_media(url_list):
quto_list = []
for item in url_list:
video_id = item.get('mediaId', '')
raw_data = {
"video_id": video_id,
"video_captions": item.get('title', ''),
"video_first_frame_image": item.get('url', '') or item.get('resourceUrl') or '',
"main_site_url": '',
"source_app_name": '视频号'
}
quto_list.append(raw_data)
return quto_list
def doubao_process_rich_media(url_list):
quto_list = []
......@@ -121,6 +137,26 @@ def qianwen_android_process_rich_media(url_list):
quto_list.append(raw_data)
return quto_list
def qianwen_process_rich_media(url_list):
quto_list = []
for item in url_list:
source_seq= item.get('source_seq', '')
if isinstance(item.get('url'),list):
for url in item.get('url'):
raw_data = {
"video_id": url.get('zhidaye_id', ''),
"video_captions": url.get('title', ''),
"video_first_frame_image": url.get('cover',''),
"main_site_url": url.get('norm_url', '') or url.get('url', ''),
"author": url.get('author', ''),
"publishTime": url.get('publishTime', ''),
"duration": url.get('duration', ''),
"source_seq": source_seq
}
quto_list.append(raw_data)
return quto_list
def baidu_ai_rich_media(url_list):
quto_list = []
for item in url_list:
......@@ -343,6 +379,10 @@ def save_data_to_tos(target_dir, content, file_name):
content = qianwen_android_process_rich_media(content)
if platform =='BDAI':
content = baidu_ai_rich_media(content)
if platform =='TYQW':
content = qianwen_process_rich_media(content)
if platform == 'TXYBA':
content = yuanbao_android_process_rich_media(content)
if file_name == "quote.txt":
if platform == 'DB':
......
aidso_geo/models/yuanbao_android_data_process.py
View file @ cf03ab12
import json
from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils
from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
......@@ -17,6 +17,8 @@ def yuanbao_android_process_original_data(data):
is_think = False
think_bool = False
suggestions = []
rich_media_block = []
mini_program = []
response_bool = False
try:
original_content = get_string_from_tos(file_path)
......@@ -29,6 +31,7 @@ def yuanbao_android_process_original_data(data):
# 提取并解析JSON数据
data_str = i.split("data: ")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue # 跳过格式错误的数据
if json_data.get('type') == 'searchGuid':
......@@ -48,10 +51,35 @@ def yuanbao_android_process_original_data(data):
url_list.extend(json_data.get('contents')[0].get('docs'))
if json_data.get('type') == 'image':
response_content = "生成了图片"
if json_data.get('type') == 'replace':
if json_data.get('replace').get('display') =='videoBoxV2':
rich_media_block.extend(json_data.get('replace').get('multimedias'))
if json_data.get('replace').get('display') =='miniProgramBox':
response_content+=json_data.get('replace').get('title')
response_content += '\n\nrender_ecom_card_widget_miniprogram_start:\n\n'
print(json_data.get('replace').get('multimedias'))
for pro in json_data.get('replace').get('multimedias'):
title = pro.get('title','')
icon = pro.get('linkInfo', {}).get('icon','') or pro.get('linkInfo', {}).get('iconDark','')
wx_app_id = pro.get('linkInfo', {}).get('wxAppId','')
response_mini = {
"title":title,
"icon":icon,
"wx_app_id":wx_app_id,
}
mini_program.append(response_mini)
mini_program_str = json.dumps(mini_program, ensure_ascii=False)
response_content += mini_program_str
response_content += '\n\nrender_ecom_card_widget_miniprogram_end:'
print(response_content)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions)
suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e:
parts = file_path.split('/')
platform = parts[2]
task_id = parts[1]
......@@ -76,5 +104,33 @@ def yuanbao_android_process_original_data(data):
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
if __name__ == '__main__':
file_path2 = 'geo/5582d220-f032-4095-ad39-1dbc33a536c3/TXYBA/original.text'
yuanbao_android_process_original_data(file_path2)
# file_path2 = 'geo/6f49084e-f191-447d-849b-bc6c60fd5d20/TXYBA/original.text'
# yuanbao_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '6de35e20-4a67-4761-8a16-b91dd1f2fb7f' and platform = 'TXYBA'")
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return yuanbao_android_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/utils/ai_utils.py
View file @ cf03ab12
......@@ -60,7 +60,7 @@ def ai_get_brand_list(content, prompt):
"temperature": 0
})
headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9',
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
......@@ -122,36 +122,13 @@ def ai_get_product_list(content, prompt):
"content": f"""请从下面的文本中抽取所有产品词。 文本如下{content}"""
}
],
# "response_format": {
# "type": "json_schema",
# "json_schema": {
# "name": "brand_list",
# "strict": True,
# "schema": {
# "type": "object",
# "properties": {
# "brands": {
# "type": "array",
# "items": {
# "type": "string"
# },
# "description": "原文中的产品词"
# }
# },
# "required": [
# "brands"
# ],
# "additionalProperties": False
# }
# }
# },
"thinking": {
"type": "disabled"
},
"temperature": 0
})
headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9',
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
......@@ -218,7 +195,7 @@ def ai_get_product_list_search(product_list):
"temperature": 0
})
headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9',
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
......@@ -325,7 +302,7 @@ def ai_get_brand_name(content):
"temperature": 0
})
headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9',
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
......@@ -441,7 +418,7 @@ def ai_get_product_sentiment_and_mentions(brand_list, ai_answer_text):
}
headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9",
"Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json"
}
......@@ -573,7 +550,7 @@ def ai_get_brand_sentiment_and_mentions(brand_list, ai_answer_text):
}
headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9",
"Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json"
}
......@@ -717,7 +694,7 @@ def ai_get_product_and_brands(content, prompt):
"temperature": 0
})
headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9',
'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json'
}
......@@ -918,7 +895,7 @@ def _call_single_batch(brand_name, batch_articles):
}
headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9",
"Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json"
}
......@@ -1025,8 +1002,9 @@ if __name__ == '__main__':
'崔字牌',
'海南金永丰',
'金永丰',
# fcc424e5-58af-494d-9683-5787413a26c9
]
print(ai_get_brand_sentiment_and_mentions(brand_list,content))
print(ai_get_brand_list(brand_list,content))
# print(ai_result)
# pro =
# ai_get_product_list()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment