Skip to content

A
aidso-data
Commit cf03ab12 authored by Yaowentong's avatar Yaowentong
Browse files
Options

元宝千问修复

parent d521b79d
Hide whitespace changes
Inline Side-by-side
Showing with 410 additions and 268 deletions
aidso_geo/core/down_load_bot.py
View file @ cf03ab12
...@@ -1047,7 +1047,7 @@ if __name__ == "__main__": ...@@ -1047,7 +1047,7 @@ if __name__ == "__main__":
# for b in brand_name: # for b in brand_name:
# zhou_report(phone, begin, end, b) # zhou_report(phone, begin, end, b)
# # qian_report(phone,begin,end,b) # # qian_report(phone,begin,end,b)
# qian_report(phone,begin,end,b) # qian_report(phone,begin,end,b)
brand_library_id = 2063180776253702144 brand_library_id = 2063180776253702144
user_id = 2063177785609949184 user_id = 2063177785609949184
begin = '2026-06-08' begin = '2026-06-08'
......
aidso_geo/core/hua.py
View file @ cf03ab12
from venv import logger
import tldextract as tldextract import tldextract as tldextract
from openpyxl import load_workbook from openpyxl import load_workbook
......
aidso_geo/models/process.py
View file @ cf03ab12
...@@ -1910,43 +1910,38 @@ def run_data(PAGE_SIZE,MAX_WORKERS): ...@@ -1910,43 +1910,38 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
if __name__ == '__main__': if __name__ == '__main__':
# data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId = '84db7b12-88e7-4aaa-bc58-8d61f1051934'") data_list = bh_utils.query_data(f"select * from geo_commit_task where status ='ING' ")
# # data_list = bh_utils.query_data(query_sql) # data_list = bh_utils.query_data(query_sql)
# # print(data_list) # print(data_list)
# # # # # # #
# # # # # # #
# # # # # # # # # # #
# def handle_item(i): def handle_item(i):
# if i.get('comWordsMap'): if i.get('comWordsMap'):
# i['comWordsMap'] = json.loads(i.get('comWordsMap')) i['comWordsMap'] = json.loads(i.get('comWordsMap'))
# if i.get('brandWords'): if i.get('brandWords'):
# i['brandWords'] = json.loads(i.get('brandWords')) i['brandWords'] = json.loads(i.get('brandWords'))
# if i.get('comWords'): if i.get('comWords'):
# i['comWords'] = json.loads(i.get('comWords')) i['comWords'] = json.loads(i.get('comWords'))
# if i.get('keywords'): if i.get('keywords'):
# i['keywords'] = json.loads(i.get('keywords')) i['keywords'] = json.loads(i.get('keywords'))
# if i.get('productWordsMap'): if i.get('productWordsMap'):
# i['productWordsMap'] = json.loads(i.get('productWordsMap')) i['productWordsMap'] = json.loads(i.get('productWordsMap'))
# type_t = i.get('type') type_t = i.get('type')
# # type_t = 'batch' # type_t = 'batch'
#
# # return task_send_queue(i,type_t) # return task_send_queue(i,type_t)
# return platform_process(i) return platform_process(i)
#
# if data_list: if data_list:
# with ThreadPoolExecutor(max_workers=30) as executor: with ThreadPoolExecutor(max_workers=30) as executor:
# futures = [executor.submit(handle_item, i) for i in data_list] futures = [executor.submit(handle_item, i) for i in data_list]
#
# for future in as_completed(futures): for future in as_completed(futures):
# try: try:
# future.result() future.result()
# except Exception as e: except Exception as e:
# logger.exception(f"platform_process 执行异常: {e}") logger.exception(f"platform_process 执行异常: {e}")
PAGE_SIZE =1000
MAX_WORKERS =50
run_data(PAGE_SIZE,MAX_WORKERS)
text = """
"""
# keywords = [] # keywords = []
# get_keyword_ranks() # get_keyword_ranks()
aidso_geo/models/qianwen_android_data_process.py
View file @ cf03ab12
...@@ -10,6 +10,7 @@ from aidso_geo.utils.tos_utils import get_string_from_tos ...@@ -10,6 +10,7 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
def qianwen_android_process_original_data(data): def qianwen_android_process_original_data(data):
# def qianwen_android_process_original_data(file_path):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text' file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = [] url_list = []
url_list_batch = [] url_list_batch = []
...@@ -23,190 +24,256 @@ def qianwen_android_process_original_data(data): ...@@ -23,190 +24,256 @@ def qianwen_android_process_original_data(data):
response_bool = False response_bool = False
result = [] result = []
try: try:
json_original = ""
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
try:
json_original = json.loads(original_content)
sse_json_data = json.loads(json_original.get('sse'))
response_content = sse_json_data.get('reply_text','')
think_content = sse_json_data.get('deepthink_text','')
paa_answer = sse_json_data.get('paa_answer','')
bar_sources = sse_json_data.get('bar_sources')
content_list = original_content.split("\n") source_content =next(
(item.get('content') for item in bar_sources if item.get('type') == 'source'),
for i in content_list: {}
)
if i.startswith("data:"): source_list = source_content.get('list', [])
try:
# 提取并解析JSON数据 video_note_list = sse_json_data.get('video_note_list',{}).get('list')
data_str = i.split("data:")[1] video_list = []
if video_note_list:
json_data = json.loads(data_str) for i in video_note_list:
# rich_media = {
except (IndexError, json.JSONDecodeError): 'zhidaye_id':i.get('zhidaye_id'),
'title':i.get('title'),
continue 'cover':i.get('cover'),
data = json_data.get('data') 'url':i.get('url'),
'author':i.get('author'),
if data: 'publishTime':i.get('publishTime'),
messages = data.get('messages') 'duration':i.get('duration'),
if isinstance(messages, list): }
if len(messages) > 0: video_list.append(rich_media)
mime_type = messages[0].get('mime_type')
status = messages[0].get('status') rich_media_block.append({
type = messages[0].get('type') "utl":video_list,
content = messages[0].get('content') "source_seq":''
meta_data = messages[0].get('meta_data') })
meta_data_type = "" source_count_list = []
multi_load = "" if source_list:
multi_load_type = "" for s in source_list:
multi_load_content = "" source_url = {
multi_load_status = "" 'url':s.get('url'),
if meta_data: 'title':s.get('title'),
meta_data_type = messages[0].get('meta_data').get('type') 'summary':s.get('summary'),
multi_load = messages[0].get('meta_data').get('multi_load') 'publish_time':s.get('publish_time'),
if multi_load: 'name':s.get('name'),
if isinstance(multi_load, list): 'icon':s.get('icon'),
multi_load_type = multi_load[0].get('type') 'source_seq':s.get('source_seq',''),
multi_load_content = multi_load[0].get('content') }
if multi_load_content: source_count_list.append(source_url)
multi_load_status = multi_load[0].get('content').get('status') url_list.append({
# 搜索前 "url":[
if mime_type == 'plan_cot/post' and status == 'complete': {
think_content += content "type":"source",
# 搜索词 "content":{
if mime_type == 'bar/progress' and meta_data_type == 'cot': "list":source_count_list
if messages[0].get('meta_data').get('content'): },
search_list = messages[0].get('meta_data').get('content').get('list') }
if isinstance(search_list, list): ],
for search in search_list: "source_seq":''
search_keyword.append(search.get('query')) })
if mime_type == 'bar/progress' and meta_data_type == 'bar_update':
if messages[0].get('meta_data'): if paa_answer:
if messages[0].get('meta_data').get('elements'): paa_json = json.loads(paa_answer)
response_content+=messages[0].get('meta_data').get('elements')[0].get('content') for paa in paa_json:
# 引用来源-无下标 suggestions.append(paa.get('paa_query'))
if mime_type == 'bar/iframe' and status == 'complete': except Exception as e:
content_list = original_content.split("\n")
for i in content_list:
if i.startswith("data:"):
try:
# 提取并解析JSON数据
data_str = i.split("data:")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue
data = json_data.get('data')
if data:
messages = data.get('messages')
if isinstance(messages, list):
if len(messages) > 0:
mime_type = messages[0].get('mime_type')
status = messages[0].get('status')
type = messages[0].get('type')
content = messages[0].get('content')
meta_data = messages[0].get('meta_data')
meta_data_type = ""
multi_load = ""
multi_load_type = ""
multi_load_content = ""
multi_load_status = ""
if meta_data: if meta_data:
sources = meta_data.get('sources') meta_data_type = messages[0].get('meta_data').get('type')
for sou in sources: multi_load = messages[0].get('meta_data').get('multi_load')
url_list.append({ if multi_load:
"url": [sou], if isinstance(multi_load, list):
"source_seq": '', multi_load_type = multi_load[0].get('type')
}) multi_load_content = multi_load[0].get('content')
taobao_list = [] if multi_load_content:
if mime_type == 'multi_load/iframe' and multi_load_type == 'taoassistant_fold_product_feeds' and status == 'complete': multi_load_status = multi_load[0].get('content').get('status')
# 搜索前
for mu in multi_load: if mime_type == 'plan_cot/post' and status == 'complete':
think_content += content
if mu.get('type') == 'taoassistant_fold_product_feeds': # 搜索词
source_seq = mu.get('source_seq') if mime_type == 'bar/progress' and meta_data_type == 'cot':
jump_url = next( if messages[0].get('meta_data').get('content'):
( search_list = messages[0].get('meta_data').get('content').get('list')
item.get('params', {}).get('popupLink', '') if isinstance(search_list, list):
for item in for search in search_list:
mu.get('content', {}).get('cardData', {}).get('clientActions', []) search_keyword.append(search.get('query'))
if item.get('type') == 'popupLink' if mime_type == 'bar/progress' and meta_data_type == 'bar_update':
), if messages[0].get('meta_data'):
'' if messages[0].get('meta_data').get('elements'):
) response_content+=messages[0].get('meta_data').get('elements')[0].get('content')
# 引用来源-无下标
query = next( if mime_type == 'bar/iframe' and status == 'complete':
( if meta_data:
item.get('params', {}).get('params', {}).get('queryList',[]) sources = meta_data.get('sources')
for item in for sou in sources:
mu.get('content', {}).get('cardData', {}).get('clientActions', []) url_list.append({
if item.get('type') == 'loadmore' "url": [sou],
), "source_seq": '',
'' })
) taobao_list = []
if isinstance(query,str): if mime_type == 'multi_load/iframe' and multi_load_type == 'taoassistant_fold_product_feeds' and status == 'complete':
if query:
query = json.loads(query) for mu in multi_load:
if isinstance(query,list):
search_keyword.extend(query) if mu.get('type') == 'taoassistant_fold_product_feeds':
if isinstance(mu.get('content').get('cardData').get('data').get('items'), list): source_seq = mu.get('source_seq')
for pro in mu.get('content').get('cardData').get('data').get('items'): jump_url = next(
(
price = (pro.get('priceShowWithIcon') or {}).get('price') or pro.get('itemPrice') or '' item.get('params', {}).get('popupLink', '')
taobao_list.append({ for item in
"title": pro.get('title'), mu.get('content', {}).get('cardData', {}).get('clientActions', [])
"shop_name": next( if item.get('type') == 'popupLink'
( ),
item.get('text', '') ''
for item in )
pro.get('structuredShopInfo', {}).get('infoList', [])
if item.get('sourceType') == 'shop_name' query = next(
(
item.get('params', {}).get('params', {}).get('queryList',[])
for item in
mu.get('content', {}).get('cardData', {}).get('clientActions', [])
if item.get('type') == 'loadmore'
),
''
)
if isinstance(query,str):
if query:
query = json.loads(query)
if isinstance(query,list):
search_keyword.extend(query)
if isinstance(mu.get('content').get('cardData').get('data').get('items'), list):
for pro in mu.get('content').get('cardData').get('data').get('items'):
price = (pro.get('priceShowWithIcon') or {}).get('price') or pro.get('itemPrice') or ''
taobao_list.append({
"title": pro.get('title'),
"shop_name": next(
(
item.get('text', '')
for item in
pro.get('structuredShopInfo', {}).get('infoList', [])
if item.get('sourceType') == 'shop_name'
),
''
), ),
'' "pic_path": pro.get('pic_path', ''),
), "price": price,
"pic_path": pro.get('pic_path', ''), "jump_url": jump_url,
"price": price, "source_seq": source_seq,
"jump_url": jump_url, "auctionURL": pro.get('auctionURL'),
"source_seq": source_seq, })
"auctionURL": pro.get('auctionURL'),
}) if taobao_list:
response_content += 'render_ecom_card_widget_taobao_start:'
if taobao_list: damai_str = json.dumps(taobao_list, ensure_ascii=False)
response_content += 'render_ecom_card_widget_taobao_start:' response_content += damai_str
damai_str = json.dumps(taobao_list, ensure_ascii=False) response_content += 'render_ecom_card_widget_taobao_end:\n'
response_content += damai_str
response_content += 'render_ecom_card_widget_taobao_end:\n' damai_list = []
if mime_type == 'multi_load/iframe' and multi_load_type == 'damai_shows_list' and multi_load_status == 'complete':
damai_list = []
if mime_type == 'multi_load/iframe' and multi_load_type == 'damai_shows_list' and multi_load_status == 'complete': for mu in multi_load:
if mu.get('type') == 'damai_shows_list':
for mu in multi_load: source_seq = mu.get('source_seq')
if mu.get('type') == 'damai_shows_list': if isinstance(mu.get('content').get('list'), list):
source_seq = mu.get('source_seq') for damai in mu.get('content').get('list'):
if isinstance(mu.get('content').get('list'), list): venueName = damai.get('data').get('venueName', '')
for damai in mu.get('content').get('list'): cityName = damai.get('data').get('cityName', '')
venueName = damai.get('data').get('venueName', '') damai_list.append({
cityName = damai.get('data').get('cityName', '') 'name': damai.get('data').get('name', ''),
damai_list.append({ 'venueName': venueName,
'name': damai.get('data').get('name', ''), 'showTime': damai.get('data').get('showTime', ''),
'venueName': venueName, "cityName": cityName,
'showTime': damai.get('data').get('showTime', ''), 'verticalPic': damai.get('data').get('verticalPic', ''),
"cityName": cityName, 'priceShowText': damai.get('data').get('priceShowText', ''),
'verticalPic': damai.get('data').get('verticalPic', ''), 'webURL': damai.get('jumpConfig').get('detailUrl').get('webURL',
'priceShowText': damai.get('data').get('priceShowText', ''), '') or '',
'webURL': damai.get('jumpConfig').get('detailUrl').get('webURL', 'priceLow': damai.get('data').get('priceLow', ''),
'') or '', 'minPrice': damai.get('data').get('minPrice', ''),
'priceLow': damai.get('data').get('priceLow', ''), 'maxPrice': damai.get('data').get('maxPrice', ''),
'minPrice': damai.get('data').get('minPrice', ''), 'priceStr': damai.get('data').get('priceStr', ''),
'maxPrice': damai.get('data').get('maxPrice', ''), 'showVenueName': cityName + venueName,
'priceStr': damai.get('data').get('priceStr', ''), 'source_seq_id': source_seq
'showVenueName': cityName + venueName, })
'source_seq_id': source_seq if damai_list:
}) response_content += 'render_ecom_card_widget_damai_start:'
if damai_list: damai_str = json.dumps(damai_list, ensure_ascii=False)
response_content += 'render_ecom_card_widget_damai_start:' response_content += damai_str
damai_str = json.dumps(damai_list, ensure_ascii=False) response_content += 'render_ecom_card_widget_damai_end:\n'
response_content += damai_str
response_content += 'render_ecom_card_widget_damai_end:\n' if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete':
think_content += multi_load_content.get('think_content')
if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete': # 引用来源 回答 视频列表
think_content += multi_load_content.get('think_content') if mime_type == 'multi_load/iframe' and status == 'complete':
# 引用来源 回答 视频列表 response_content += content
if mime_type == 'multi_load/iframe' and status == 'complete': for mu in multi_load:
response_content += content if mu.get('type') == 'video_note_list':
for mu in multi_load: rich_media_block.append(
if mu.get('type') == 'video_note_list': {
rich_media_block.append( "url": mu.get('content').get('list'),
{ "source_seq": mu.get('source_seq')
"url": mu.get('content').get('list'), }
"source_seq": mu.get('source_seq') )
}
) if mu.get('type') == 'source_group_web':
url_list_batch.append(
if mu.get('type') == 'source_group_web': {
url_list_batch.append( "url": mu.get('content').get('list'),
{ "source_seq": mu.get('source_seq')
"url": mu.get('content').get('list'), }
"source_seq": mu.get('source_seq') )
}
) if mime_type == 'paa/iframe' and status == 'complete':
paas = meta_data.get('paas')
if mime_type == 'paa/iframe' and status == 'complete': if isinstance(paas, list):
paas = meta_data.get('paas') for pa in paas:
if isinstance(paas, list): suggestions.append(pa.get('show_text'))
for pa in paas: if url_list_batch:
suggestions.append(pa.get('show_text')) url_list = url_list_batch
if url_list_batch:
url_list = url_list_batch response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
...@@ -240,17 +307,13 @@ if __name__ == '__main__': ...@@ -240,17 +307,13 @@ if __name__ == '__main__':
#https://tcdn.aidso.com/geo/6fede9d0-ce6c-4c97-bcb8-43c499fc24e5/TYQWA/think.txt?secret=1104ed37df6abefbb2b454530b06515f0f59505d5751504e524855 #https://tcdn.aidso.com/geo/6fede9d0-ce6c-4c97-bcb8-43c499fc24e5/TYQWA/think.txt?secret=1104ed37df6abefbb2b454530b06515f0f59505d5751504e524855
# 6fede9d0-ce6c-4c97-bcb8-43c499fc24e5 # 6fede9d0-ce6c-4c97-bcb8-43c499fc24e5
file_path2 = 'geo/0a2ed990b43a4f389ecdc14d185a3866/TYQWA/original.text' file_path2 = 'geo/alskdjhfasldjkfh/TYQWA/original.text'
# file_path2 = 'geo/ef586bf6-55d8-4e2a-8fb0-5fd1beb5bf7c/TYQWA/original.text' # file_path2 = 'geo/ef66d5f9-cadb-43f3-b8cf-824d75a81e7c/TYQWA/original.text'
# qianwen_android_process_original_data(file_path2) qianwen_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'd652c3d0-484b-46b6-aefc-1ccd58c46a51' and platform = 'TYQWA'")
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0a2ed990b43a4f389ecdc14d185a3866' and platform = 'TYQWA'")
# data_list = bh_utils.query_data(query_sql)
# print(data_list)
# # #
# # #
# # # # #
def handle_item(i): def handle_item(i):
if i.get('comWordsMap'): if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap')) i['comWordsMap'] = json.loads(i.get('comWordsMap'))
......
aidso_geo/models/qianwen_data_process.py
View file @ cf03ab12
...@@ -15,6 +15,7 @@ def qianwen_process_original_data(data): ...@@ -15,6 +15,7 @@ def qianwen_process_original_data(data):
response_content = "" response_content = ""
search_keyword = [] search_keyword = []
suggestions = [] suggestions = []
rich_media_block =[]
is_think = False is_think = False
think_bool = False think_bool = False
response_bool = False response_bool = False
...@@ -32,6 +33,7 @@ def qianwen_process_original_data(data): ...@@ -32,6 +33,7 @@ def qianwen_process_original_data(data):
data_str = i.split("data:")[1] data_str = i.split("data:")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
...@@ -85,7 +87,9 @@ def qianwen_process_original_data(data): ...@@ -85,7 +87,9 @@ def qianwen_process_original_data(data):
if json_data.get("data").get('messages'): if json_data.get("data").get('messages'):
messages = json_data.get("data").get('messages') messages = json_data.get("data").get('messages')
for ms in messages: for ms in messages:
if ms.get('mime_type') =='multi_load/iframe' and ms.get('status') == 'complete': if ms.get('mime_type') =='multi_load/iframe' and ms.get('status') == 'complete':
response_content = ms.get('content') response_content = ms.get('content')
response_content = response_content.replace("[(deep_think)]", "") response_content = response_content.replace("[(deep_think)]", "")
...@@ -97,13 +101,20 @@ def qianwen_process_original_data(data): ...@@ -97,13 +101,20 @@ def qianwen_process_original_data(data):
if mu.get('content').get('status') == 'complete': if mu.get('content').get('status') == 'complete':
think_content = mu.get('content').get('think_content') think_content = mu.get('content').get('think_content')
if mu.get('type') == 'ref_source_inline': if mu.get('type') == 'ref_source_inline':
mu_content = mu.get('content') mu_content = mu.get('content')
if mu_content: if mu_content:
if mu_content.get('query_list'): if mu_content.get('query_list'):
search_keyword.extend(mu_content.get('query_list')) search_keyword.extend(mu_content.get('query_list'))
if mu_content.get('list'): if mu_content.get('list'):
url_list_batch.extend(mu_content.get('list')) url_list_batch.extend(mu_content.get('list'))
if mu.get('type') == 'video_note_list':
rich_media_block.append(
{
"url": mu.get('content').get('list'),
"source_seq": mu.get('source_seq')
}
)
if ms.get('mime_type') =='bar/iframe' and ms.get('status') == 'complete': if ms.get('mime_type') =='bar/iframe' and ms.get('status') == 'complete':
if ms.get('meta_data').get('sources'): if ms.get('meta_data').get('sources'):
...@@ -128,7 +139,7 @@ def qianwen_process_original_data(data): ...@@ -128,7 +139,7 @@ def qianwen_process_original_data(data):
) )
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e: except Exception as e:
parts = file_path.split('/') parts = file_path.split('/')
...@@ -162,7 +173,7 @@ if __name__ == '__main__': ...@@ -162,7 +173,7 @@ if __name__ == '__main__':
# # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text' # # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
# qianwen_process_original_data(file_path2) # qianwen_process_original_data(file_path2)
# for i in task_id: # for i in task_id:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '16e53643-b1ac-4566-a0aa-68340576a3ff' and platform = 'TYQW'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '3e317dd8-2512-476c-a31a-104c0bcaca75' and platform = 'TYQW'")
# # # # # #
# # # # # #
......
aidso_geo/models/spider_save_tos.py
View file @ cf03ab12
...@@ -73,6 +73,22 @@ def doubao_android_process_rich_media(url_list): ...@@ -73,6 +73,22 @@ def doubao_android_process_rich_media(url_list):
quto_list.append(raw_data) quto_list.append(raw_data)
return quto_list return quto_list
def yuanbao_android_process_rich_media(url_list):
quto_list = []
for item in url_list:
video_id = item.get('mediaId', '')
raw_data = {
"video_id": video_id,
"video_captions": item.get('title', ''),
"video_first_frame_image": item.get('url', '') or item.get('resourceUrl') or '',
"main_site_url": '',
"source_app_name": '视频号'
}
quto_list.append(raw_data)
return quto_list
def doubao_process_rich_media(url_list): def doubao_process_rich_media(url_list):
quto_list = [] quto_list = []
...@@ -121,6 +137,26 @@ def qianwen_android_process_rich_media(url_list): ...@@ -121,6 +137,26 @@ def qianwen_android_process_rich_media(url_list):
quto_list.append(raw_data) quto_list.append(raw_data)
return quto_list return quto_list
def qianwen_process_rich_media(url_list):
quto_list = []
for item in url_list:
source_seq= item.get('source_seq', '')
if isinstance(item.get('url'),list):
for url in item.get('url'):
raw_data = {
"video_id": url.get('zhidaye_id', ''),
"video_captions": url.get('title', ''),
"video_first_frame_image": url.get('cover',''),
"main_site_url": url.get('norm_url', '') or url.get('url', ''),
"author": url.get('author', ''),
"publishTime": url.get('publishTime', ''),
"duration": url.get('duration', ''),
"source_seq": source_seq
}
quto_list.append(raw_data)
return quto_list
def baidu_ai_rich_media(url_list): def baidu_ai_rich_media(url_list):
quto_list = [] quto_list = []
for item in url_list: for item in url_list:
...@@ -343,6 +379,10 @@ def save_data_to_tos(target_dir, content, file_name): ...@@ -343,6 +379,10 @@ def save_data_to_tos(target_dir, content, file_name):
content = qianwen_android_process_rich_media(content) content = qianwen_android_process_rich_media(content)
if platform =='BDAI': if platform =='BDAI':
content = baidu_ai_rich_media(content) content = baidu_ai_rich_media(content)
if platform =='TYQW':
content = qianwen_process_rich_media(content)
if platform == 'TXYBA':
content = yuanbao_android_process_rich_media(content)
if file_name == "quote.txt": if file_name == "quote.txt":
if platform == 'DB': if platform == 'DB':
......
aidso_geo/models/yuanbao_android_data_process.py
View file @ cf03ab12
import json import json
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos from aidso_geo.utils.tos_utils import get_string_from_tos
...@@ -17,6 +17,8 @@ def yuanbao_android_process_original_data(data): ...@@ -17,6 +17,8 @@ def yuanbao_android_process_original_data(data):
is_think = False is_think = False
think_bool = False think_bool = False
suggestions = [] suggestions = []
rich_media_block = []
mini_program = []
response_bool = False response_bool = False
try: try:
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
...@@ -29,6 +31,7 @@ def yuanbao_android_process_original_data(data): ...@@ -29,6 +31,7 @@ def yuanbao_android_process_original_data(data):
# 提取并解析JSON数据 # 提取并解析JSON数据
data_str = i.split("data: ")[1] data_str = i.split("data: ")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue # 跳过格式错误的数据 continue # 跳过格式错误的数据
if json_data.get('type') == 'searchGuid': if json_data.get('type') == 'searchGuid':
...@@ -48,10 +51,35 @@ def yuanbao_android_process_original_data(data): ...@@ -48,10 +51,35 @@ def yuanbao_android_process_original_data(data):
url_list.extend(json_data.get('contents')[0].get('docs')) url_list.extend(json_data.get('contents')[0].get('docs'))
if json_data.get('type') == 'image': if json_data.get('type') == 'image':
response_content = "生成了图片" response_content = "生成了图片"
if json_data.get('type') == 'replace':
if json_data.get('replace').get('display') =='videoBoxV2':
rich_media_block.extend(json_data.get('replace').get('multimedias'))
if json_data.get('replace').get('display') =='miniProgramBox':
response_content+=json_data.get('replace').get('title')
response_content += '\n\nrender_ecom_card_widget_miniprogram_start:\n\n'
print(json_data.get('replace').get('multimedias'))
for pro in json_data.get('replace').get('multimedias'):
title = pro.get('title','')
icon = pro.get('linkInfo', {}).get('icon','') or pro.get('linkInfo', {}).get('iconDark','')
wx_app_id = pro.get('linkInfo', {}).get('wxAppId','')
response_mini = {
"title":title,
"icon":icon,
"wx_app_id":wx_app_id,
}
mini_program.append(response_mini)
mini_program_str = json.dumps(mini_program, ensure_ascii=False)
response_content += mini_program_str
response_content += '\n\nrender_ecom_card_widget_miniprogram_end:'
print(response_content)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e: except Exception as e:
parts = file_path.split('/') parts = file_path.split('/')
platform = parts[2] platform = parts[2]
task_id = parts[1] task_id = parts[1]
...@@ -76,5 +104,33 @@ def yuanbao_android_process_original_data(data): ...@@ -76,5 +104,33 @@ def yuanbao_android_process_original_data(data):
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
if __name__ == '__main__': if __name__ == '__main__':
file_path2 = 'geo/5582d220-f032-4095-ad39-1dbc33a536c3/TXYBA/original.text' # file_path2 = 'geo/6f49084e-f191-447d-849b-bc6c60fd5d20/TXYBA/original.text'
yuanbao_android_process_original_data(file_path2) # yuanbao_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '6de35e20-4a67-4761-8a16-b91dd1f2fb7f' and platform = 'TXYBA'")
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return yuanbao_android_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/utils/ai_utils.py
View file @ cf03ab12
...@@ -60,7 +60,7 @@ def ai_get_brand_list(content, prompt): ...@@ -60,7 +60,7 @@ def ai_get_brand_list(content, prompt):
"temperature": 0 "temperature": 0
}) })
headers = { headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
...@@ -122,36 +122,13 @@ def ai_get_product_list(content, prompt): ...@@ -122,36 +122,13 @@ def ai_get_product_list(content, prompt):
"content": f"""请从下面的文本中抽取所有产品词。 文本如下{content}""" "content": f"""请从下面的文本中抽取所有产品词。 文本如下{content}"""
} }
], ],
# "response_format": {
# "type": "json_schema",
# "json_schema": {
# "name": "brand_list",
# "strict": True,
# "schema": {
# "type": "object",
# "properties": {
# "brands": {
# "type": "array",
# "items": {
# "type": "string"
# },
# "description": "原文中的产品词"
# }
# },
# "required": [
# "brands"
# ],
# "additionalProperties": False
# }
# }
# },
"thinking": { "thinking": {
"type": "disabled" "type": "disabled"
}, },
"temperature": 0 "temperature": 0
}) })
headers = { headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
...@@ -218,7 +195,7 @@ def ai_get_product_list_search(product_list): ...@@ -218,7 +195,7 @@ def ai_get_product_list_search(product_list):
"temperature": 0 "temperature": 0
}) })
headers = { headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
...@@ -325,7 +302,7 @@ def ai_get_brand_name(content): ...@@ -325,7 +302,7 @@ def ai_get_brand_name(content):
"temperature": 0 "temperature": 0
}) })
headers = { headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
...@@ -441,7 +418,7 @@ def ai_get_product_sentiment_and_mentions(brand_list, ai_answer_text): ...@@ -441,7 +418,7 @@ def ai_get_product_sentiment_and_mentions(brand_list, ai_answer_text):
} }
headers = { headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9", "Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json" "Content-Type": "application/json"
} }
...@@ -573,7 +550,7 @@ def ai_get_brand_sentiment_and_mentions(brand_list, ai_answer_text): ...@@ -573,7 +550,7 @@ def ai_get_brand_sentiment_and_mentions(brand_list, ai_answer_text):
} }
headers = { headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9", "Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json" "Content-Type": "application/json"
} }
...@@ -717,7 +694,7 @@ def ai_get_product_and_brands(content, prompt): ...@@ -717,7 +694,7 @@ def ai_get_product_and_brands(content, prompt):
"temperature": 0 "temperature": 0
}) })
headers = { headers = {
'Authorization': 'Bearer fcc424e5-58af-494d-9683-5787413a26c9', 'Authorization': 'Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
...@@ -918,7 +895,7 @@ def _call_single_batch(brand_name, batch_articles): ...@@ -918,7 +895,7 @@ def _call_single_batch(brand_name, batch_articles):
} }
headers = { headers = {
"Authorization": "Bearer fcc424e5-58af-494d-9683-5787413a26c9", "Authorization": "Bearer ark-7afc3be2-37a8-47fd-9f02-996258a3d305-27da0",
"Content-Type": "application/json" "Content-Type": "application/json"
} }
...@@ -1025,8 +1002,9 @@ if __name__ == '__main__': ...@@ -1025,8 +1002,9 @@ if __name__ == '__main__':
'崔字牌', '崔字牌',
'海南金永丰', '海南金永丰',
'金永丰', '金永丰',
# fcc424e5-58af-494d-9683-5787413a26c9
] ]
print(ai_get_brand_sentiment_and_mentions(brand_list,content)) print(ai_get_brand_list(brand_list,content))
# print(ai_result) # print(ai_result)
# pro = # pro =
# ai_get_product_list() # ai_get_product_list()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment