Skip to content

A
aidso-data
Commit 82720788 authored by Yaowentong's avatar Yaowentong
Browse files
Options

jdeco 京东 TXYBA 

dyeco 抖音  DOUBA
dypoi 抖音本地生活 DOUBA
taobao 淘宝 TYQWA
damai 大麦 TYQWA
gaode 高德地图 TYQWA
txmap 腾讯地图 TXYBA
parent e7a8582c
Hide whitespace changes
Inline Side-by-side
Showing with 174 additions and 18 deletions
aidso_geo/models/doubao_android_data_process.py
View file @ 82720788
...@@ -238,6 +238,7 @@ def doubao_mobile_process_original_data(task_data): ...@@ -238,6 +238,7 @@ def doubao_mobile_process_original_data(task_data):
if poi_data: if poi_data:
for po in poi_data: for po in poi_data:
image_list = po.get( image_list = po.get(
"image_list") or [] "image_list") or []
...@@ -250,9 +251,12 @@ def doubao_mobile_process_original_data(task_data): ...@@ -250,9 +251,12 @@ def doubao_mobile_process_original_data(task_data):
''), ''),
"score": po.get('score', "score": po.get('score',
''), ''),
"rank": po.get('rank', '') "rank": po.get('rank', ''),
"poi_id": po.get('poi_id', '')
} }
poi_list.append(poi) poi_list.append(poi)
dy_poi_list.extend(poi_list)
if media_list: if media_list:
...@@ -323,6 +327,8 @@ def doubao_mobile_process_original_data(task_data): ...@@ -323,6 +327,8 @@ def doubao_mobile_process_original_data(task_data):
new_url_list.append(item) new_url_list.append(item)
if dy_eco_list: if dy_eco_list:
save_eco_data_to_bh(task_data,'dyeco',dy_eco_list) save_eco_data_to_bh(task_data,'dyeco',dy_eco_list)
if dy_poi_list:
save_eco_data_to_bh(task_data,'dypoi',dy_poi_list)
url_list = new_url_list url_list = new_url_list
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
...@@ -360,7 +366,7 @@ if __name__ == '__main__': ...@@ -360,7 +366,7 @@ if __name__ == '__main__':
# file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt' # file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '019cd67a-7cef-4390-934e-500719a16ec8' and platform = 'DOUBA'") f"select * from geo_commit_task where taskId = '57e07263-9844-4aaf-b455-db2de2e828d8' and platform = 'DOUBA'")
......
aidso_geo/models/eco_data_process.py
View file @ 82720788
import re
from aidso_geo.utils import bh_utils from aidso_geo.utils import bh_utils
def extract_price(price):
if not price:
return ''
price = str(price).strip()
# 提取“元”前面的整数或小数
match = re.search(r'(\d+(?:\.\d+)?)\s*元', price)
return match.group(1) if match else ''
def qianwen_android_process_taobao_eco(data,eco_list): def qianwen_android_process_taobao_eco(data,eco_list):
eco_result = [] eco_result = []
task_id = data.get('taskId') task_id = data.get('taskId')
...@@ -51,6 +61,30 @@ def qianwen_android_process_damai_eco(data,eco_list): ...@@ -51,6 +61,30 @@ def qianwen_android_process_damai_eco(data,eco_list):
return eco_result return eco_result
def qianwen_android_process_gaode_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
raw_data = {
"eco_id": item.get('poi_id', ''),
"eco_url": item.get('url', '') ,
"eco_title": item.get('name', ''),
"eco_index": index+1,
"eco_pic": item.get('photos', ''),
"eco_price": item.get('cost', ''),
"eco_source": 'gaode',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def doubao_android_process_douyin_eco(data,eco_list): def doubao_android_process_douyin_eco(data,eco_list):
eco_result = [] eco_result = []
task_id = data.get('taskId') task_id = data.get('taskId')
...@@ -75,6 +109,30 @@ def doubao_android_process_douyin_eco(data,eco_list): ...@@ -75,6 +109,30 @@ def doubao_android_process_douyin_eco(data,eco_list):
return eco_result return eco_result
def doubao_android_process_douyin_poi_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
raw_data = {
"eco_id": item.get('poi_id', ''),
"eco_url": item.get('poi_url', '') ,
"eco_title": item.get('poi_name', ''),
"eco_index": index+1,
"eco_pic": item.get('image_list', [])[0].get('large_url','') or item.get('image_list', [])[0].get('small_url',''),
"eco_price": extract_price(item.get('price', '')),
"eco_source": 'dypoi',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def yuanbao_android_process_jd_eco(data,eco_list): def yuanbao_android_process_jd_eco(data,eco_list):
...@@ -101,6 +159,29 @@ def yuanbao_android_process_jd_eco(data,eco_list): ...@@ -101,6 +159,29 @@ def yuanbao_android_process_jd_eco(data,eco_list):
return eco_result return eco_result
def yuanbao_android_process_txmap_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
raw_data = {
"eco_id": item.get('poiId', ''),
"eco_url": item.get('jumpUrl', '') ,
"eco_title": item.get('title', ''),
"eco_index": index+1,
"eco_pic": item.get('image_url', 'https://tcdn.aidso.com/dso/pc/aiDialoguePng/112_112.jpg?v=2'),
"eco_price": item.get('avgPrice', ''),
"eco_source": 'txmap',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def save_eco_data_to_bh(data,eco_type,eco_list): def save_eco_data_to_bh(data,eco_type,eco_list):
...@@ -112,14 +193,19 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -112,14 +193,19 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
eco_result = qianwen_android_process_taobao_eco(data,eco_list) eco_result = qianwen_android_process_taobao_eco(data,eco_list)
if eco_type == 'damai': if eco_type == 'damai':
eco_result = qianwen_android_process_damai_eco(data, eco_list) eco_result = qianwen_android_process_damai_eco(data, eco_list)
if eco_type == 'gaode':
eco_result = qianwen_android_process_gaode_eco(data, eco_list)
if platform == 'DOUBA': if platform == 'DOUBA':
if eco_type == 'dypoi': if eco_type == 'dypoi':
... eco_result = doubao_android_process_douyin_poi_eco(data, eco_list)
if eco_type == 'dyeco': if eco_type == 'dyeco':
eco_result =doubao_android_process_douyin_eco(data,eco_list) eco_result =doubao_android_process_douyin_eco(data,eco_list)
if platform == 'TXYBA': if platform == 'TXYBA':
if eco_type == 'jdeco': if eco_type == 'jdeco':
eco_result =yuanbao_android_process_jd_eco(data,eco_list) eco_result =yuanbao_android_process_jd_eco(data,eco_list)
if eco_type == 'txmap':
eco_result = yuanbao_android_process_txmap_eco(data,eco_list)
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
......
aidso_geo/models/process.py
View file @ 82720788
...@@ -1603,6 +1603,9 @@ def result_v2(response_content, data): ...@@ -1603,6 +1603,9 @@ def result_v2(response_content, data):
"render_ecom_card_widget_damai_start", "render_ecom_card_widget_damai_start",
"render_ecom_card_widget_product_start", "render_ecom_card_widget_product_start",
"render_ecom_card_widget_jd_start", "render_ecom_card_widget_jd_start",
"render_ecom_card_widget_gaode_start",
"render_ecom_card_widget_poi_start",
"render_ecom_card_widget_txmap_start",
) )
result["hasGoods"] = int(any(marker in response_content for marker in ecom_markers)) result["hasGoods"] = int(any(marker in response_content for marker in ecom_markers))
...@@ -2011,7 +2014,7 @@ if __name__ == '__main__': ...@@ -2011,7 +2014,7 @@ if __name__ == '__main__':
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc") # data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status = 'ING'") data_list = bh_utils.query_data("select * from geo_commit_task where status != 'SUCCESS' and prompt like '%附近美食推荐%'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'") # data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ") # data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
......
aidso_geo/models/qianwen_android_data_process.py
View file @ 82720788
...@@ -23,6 +23,7 @@ def qianwen_android_process_original_data(task_data): ...@@ -23,6 +23,7 @@ def qianwen_android_process_original_data(task_data):
response_bool = False response_bool = False
result = [] result = []
damai_data_list = [] damai_data_list = []
gaode_data_list = []
try: try:
json_original = "" json_original = ""
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
...@@ -34,6 +35,7 @@ def qianwen_android_process_original_data(task_data): ...@@ -34,6 +35,7 @@ def qianwen_android_process_original_data(task_data):
think_content = sse_json_data.get('deepthink_text','') think_content = sse_json_data.get('deepthink_text','')
paa_answer = sse_json_data.get('paa_answer','') paa_answer = sse_json_data.get('paa_answer','')
bar_sources = sse_json_data.get('bar_sources') bar_sources = sse_json_data.get('bar_sources')
# amap_pois = sse_json_data.get('amap_pois')
source_content = {} source_content = {}
if bar_sources: if bar_sources:
source_content =next( source_content =next(
...@@ -299,7 +301,40 @@ def qianwen_android_process_original_data(task_data): ...@@ -299,7 +301,40 @@ def qianwen_android_process_original_data(task_data):
}) })
damai_data_list.extend(damai_list) damai_data_list.extend(damai_list)
gaode_list = [] gaode_list = []
if mime_type == 'multi_load/iframe' and multi_load_type == 'new_amap_poi_list' and multi_load_status == 'complete': if mime_type == 'multi_load/iframe' and multi_load_type == 'new_amap_poi_list' and status == 'complete':
for mu in multi_load:
mu_content = mu.get('content')
mu_source_seq = mu.get('source_seq')
mu_content_model_input = mu_content.get('modelInput')
mu_content_pois = mu_content.get('pois')
for poi in mu_content_pois:
poi_summary = poi.get('summary1')
distance_formatted = poi.get('distance_formatted')
photos = poi.get('photos')
distance_address = poi.get('address')
cost = poi.get('cost')
poi_id = mu_content_model_input.get("poi_id")
gaode_result = {
"url": f"https://www.amap.com/place/{poi_id}",
"source_seq": mu_source_seq,
"name": mu_content_model_input.get("name"),
"poi_id": poi_id,
"cost": cost,
"poi_summary": poi_summary,
"distance_formatted": distance_formatted,
"photos": photos[0].get("url", "") if isinstance(photos,
list) and photos and isinstance(
photos[0], dict) else "",
"address": distance_address,
}
gaode_list.append(gaode_result)
gaode_data_list.extend(gaode_list)
if mime_type == 'multi_load/iframe' and multi_load_type == 'new_amap_poi_list' and( multi_load_status == 'complete'):
for mu in multi_load: for mu in multi_load:
mu_content = mu.get('content') mu_content = mu.get('content')
mu_source_seq = mu.get('source_seq') mu_source_seq = mu.get('source_seq')
...@@ -312,28 +347,33 @@ def qianwen_android_process_original_data(task_data): ...@@ -312,28 +347,33 @@ def qianwen_android_process_original_data(task_data):
distance_formatted = poi.get('distance_formatted') distance_formatted = poi.get('distance_formatted')
photos = poi.get('photos') photos = poi.get('photos')
distance_address = poi.get('address') distance_address = poi.get('address')
cost = poi.get('cost')
poi_id = mu_content_model_input.get("poi_id")
gaode_result = { gaode_result = {
"url":f"https://www.amap.com/place/{poi_id}",
"source_seq":mu_source_seq, "source_seq":mu_source_seq,
"name":mu_content_model_input.get("name"), "name":mu_content_model_input.get("name"),
"poi_id":mu_content_model_input.get("poi_id"), "poi_id":poi_id,
"cost":cost,
"poi_summary":poi_summary, "poi_summary":poi_summary,
"distance_formatted":distance_formatted, "distance_formatted":distance_formatted,
"photos":photos[0].get("url", "") if isinstance(photos, list) and photos and isinstance(photos[0], dict) else "", "photos":photos[0].get("url", "") if isinstance(photos, list) and photos and isinstance(photos[0], dict) else "",
"address":distance_address, "address":distance_address,
} }
gaode_list.append(gaode_result) gaode_list.append(gaode_result)
gaode_data_list.extend(gaode_list)
if damai_list: if damai_list:
response_content += 'render_ecom_card_widget_damai_start:' response_content += 'render_ecom_card_widget_damai_start:'
damai_str = json.dumps(damai_list, ensure_ascii=False) damai_str = json.dumps(damai_list, ensure_ascii=False)
response_content += damai_str response_content += damai_str
response_content += 'render_ecom_card_widget_damai_end:\n' response_content += 'render_ecom_card_widget_damai_end:\n'
if gaode_list: if gaode_list:
response_content += 'render_ecom_card_widget_gaode_start:' for g in gaode_list:
gaode_str = json.dumps(gaode_list, ensure_ascii=False) response_content += 'render_ecom_card_widget_gaode_start:'
response_content += gaode_str gaode_str = json.dumps([g], ensure_ascii=False)
response_content += 'render_ecom_card_widget_gaode_end:\n' response_content += gaode_str
response_content += 'render_ecom_card_widget_gaode_end:\n'
if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete': if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete':
think_content += multi_load_content.get('think_content') think_content += multi_load_content.get('think_content')
# 引用来源 回答 视频列表 # 引用来源 回答 视频列表
...@@ -365,11 +405,10 @@ def qianwen_android_process_original_data(task_data): ...@@ -365,11 +405,10 @@ def qianwen_android_process_original_data(task_data):
url_list = url_list_batch url_list = url_list_batch
if damai_data_list: if damai_data_list:
save_eco_data_to_bh(task_data, 'damai', damai_data_list) save_eco_data_to_bh(task_data, 'damai', damai_data_list)
if gaode_data_list:
save_eco_data_to_bh(task_data, 'gaode', gaode_data_list)
response_content = response_content.replace("[(deep_think)]", "") response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "") response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -406,9 +445,10 @@ if __name__ == '__main__': ...@@ -406,9 +445,10 @@ if __name__ == '__main__':
# file_path2 = 'geo/ef66d5f9-cadb-43f3-b8cf-824d75a81e7c/TYQWA/original.text' # file_path2 = 'geo/ef66d5f9-cadb-43f3-b8cf-824d75a81e7c/TYQWA/original.text'
# qianwen_android_process_original_data(file_path2) # qianwen_android_process_original_data(file_path2)
# fb8de08c16484143a7299c06ba20c90d 老版本
# e09e1b7db91f4d0382e86458d8462147 新版本
# data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'") # data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'")
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'b347cfe7-0f74-47c4-8328-3c92b9cad1ea' and platform = 'TYQWA'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '09385d1a412b48859543254410c56cc8' and platform = 'TYQWA'")
def handle_item(i): def handle_item(i):
......
aidso_geo/models/yuanbao_android_data_process.py
View file @ 82720788
...@@ -38,6 +38,7 @@ def yuanbao_android_process_original_data(task_data): ...@@ -38,6 +38,7 @@ def yuanbao_android_process_original_data(task_data):
rich_media_block = [] rich_media_block = []
mini_program = [] mini_program = []
jd_goods = [] jd_goods = []
tx_goods = []
response_bool = False response_bool = False
try: try:
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
...@@ -50,7 +51,6 @@ def yuanbao_android_process_original_data(task_data): ...@@ -50,7 +51,6 @@ def yuanbao_android_process_original_data(task_data):
# 提取并解析JSON数据 # 提取并解析JSON数据
data_str = i.split("data: ")[1] data_str = i.split("data: ")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue # 跳过格式错误的数据 continue # 跳过格式错误的数据
if json_data.get('type') == 'searchGuid': if json_data.get('type') == 'searchGuid':
...@@ -139,6 +139,21 @@ def yuanbao_android_process_original_data(task_data): ...@@ -139,6 +139,21 @@ def yuanbao_android_process_original_data(task_data):
"pc_url": endpoint_url.get("pcUrl", "") "pc_url": endpoint_url.get("pcUrl", "")
} }
jd_goods.append(goods_data) jd_goods.append(goods_data)
if json_data.get('replace').get('display') == 'surroundingsV3':
for media in replace_data.get("multimedias") or []:
if not isinstance(media, dict):
continue
tx_data = {
"title":media.get('title'),
"address":media.get('address'),
"jumpUrl":media.get('jumpUrl'),
"poiId":media.get('poiId'),
"distance":media.get('distance'),
"avgPrice":media.get('avgPrice'),
"starLevel":media.get('starLevel')
}
tx_goods.append(tx_data)
if jd_goods: if jd_goods:
response_content += '\n\nrender_ecom_card_widget_jd_start:\n\n' response_content += '\n\nrender_ecom_card_widget_jd_start:\n\n'
...@@ -146,6 +161,12 @@ def yuanbao_android_process_original_data(task_data): ...@@ -146,6 +161,12 @@ def yuanbao_android_process_original_data(task_data):
response_content += jd_goods_str response_content += jd_goods_str
response_content += '\n\nrender_ecom_card_widget_jd_end:' response_content += '\n\nrender_ecom_card_widget_jd_end:'
save_eco_data_to_bh(task_data,'jdeco',jd_goods) save_eco_data_to_bh(task_data,'jdeco',jd_goods)
if tx_goods:
response_content += '\n\nrender_ecom_card_widget_txmap_start:\n\n'
tx_goods_str = json.dumps(tx_goods, ensure_ascii=False)
response_content += tx_goods_str
response_content += '\n\nrender_ecom_card_widget_txmap_end:'
save_eco_data_to_bh(task_data,'txmap',tx_goods)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -178,7 +199,7 @@ if __name__ == '__main__': ...@@ -178,7 +199,7 @@ if __name__ == '__main__':
# yuanbao_android_process_original_data(file_path2) # yuanbao_android_process_original_data(file_path2)
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '2499b6c1-d974-4b11-830a-90f0b5e7b9e5' and platform = 'TXYBA'") f"select * from geo_commit_task where taskId = '0f934ad9-c45e-43eb-93e4-9a0fa5dce5a0' and platform = 'TXYBA'")
# # # # # #
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment