Skip to content

A
aidso-data
Commit d26f7bd6 authored by Yaowentong's avatar Yaowentong
Browse files
Options

电商数据 

百看平台上线
parent 1f266c65
Hide whitespace changes
Inline Side-by-side
Showing with 192 additions and 66 deletions
aidso_geo/config/base_config.py
View file @ d26f7bd6
...@@ -70,6 +70,7 @@ class PlatformType(Enum): ...@@ -70,6 +70,7 @@ class PlatformType(Enum):
YUANBAOANDROID = "TXYBA" YUANBAOANDROID = "TXYBA"
XIAOHONGSHUANDROID = "XHSA" XIAOHONGSHUANDROID = "XHSA"
BAIDUBAIKAN = "BK" BAIDUBAIKAN = "BK"
BAIDUWENXIN = "BDWX"
@classmethod @classmethod
...@@ -138,6 +139,10 @@ class BaseConfig: ...@@ -138,6 +139,10 @@ class BaseConfig:
PlatformType.BAIDUBAIKAN.value: { PlatformType.BAIDUBAIKAN.value: {
"url": f"{base_url}baikan", "url": f"{base_url}baikan",
"storage_path": lambda tid: f"geo/{tid}/BK/original.text" "storage_path": lambda tid: f"geo/{tid}/BK/original.text"
},
PlatformType.BAIDUWENXIN.value: {
"url": f"{base_url}baiduai",
"storage_path": lambda tid: f"geo/{tid}/BDWX/original.text"
} }
} }
...@@ -199,12 +204,25 @@ if __name__ == '__main__': ...@@ -199,12 +204,25 @@ if __name__ == '__main__':
'XHSA:geo:stream_batch:list', 'XHSA:geo:stream_batch:list',
'XHSA:geo:batch:list', 'XHSA:geo:batch:list',
'geo:task_commit:list'] 'geo:task_commit:list']
t = init_redis()
def init_redis4():
try:
redis_client = redis.Redis(
host="172.16.0.24",
port=6379,
db=4,
password="aiyingli@@123",
socket_timeout=5,
decode_responses=True
)
return redis_client
except Exception as e:
return None
# aa = list(aa) # aa = list(aa)
t.delete('mt_third_task')
# for a in aa: # for a in aa:
# sec_uid = get_sec_id(a) # sec_uid = get_sec_id(a)
# get_user_info(sec_uid) # get_user_info(sec_uid)
......
aidso_geo/core/feishu_snipaste.py
View file @ d26f7bd6
...@@ -14,10 +14,7 @@ from openpyxl.workbook import Workbook ...@@ -14,10 +14,7 @@ from openpyxl.workbook import Workbook
import os ,sys import os ,sys
from loguru import logger from loguru import logger
from aidso_geo.utils.tos_utils import put_string_to_tos
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(BASE_DIR)
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
from aidso_geo.utils import bh_utils, tos_utils from aidso_geo.utils import bh_utils, tos_utils
...@@ -306,7 +303,6 @@ def doubao_process_original_data(file_path, original_content): ...@@ -306,7 +303,6 @@ def doubao_process_original_data(file_path, original_content):
response_bool = False response_bool = False
file_path_result = os.path.dirname(file_path) file_path_result = os.path.dirname(file_path)
print(file_path_result)
content_list = original_content.split("\n") content_list = original_content.split("\n")
for i in content_list: for i in content_list:
...@@ -332,9 +328,9 @@ def doubao_process_original_data(file_path, original_content): ...@@ -332,9 +328,9 @@ def doubao_process_original_data(file_path, original_content):
inner_user_ip = extra_json.get('inner_user_ip') inner_user_ip = extra_json.get('inner_user_ip')
inner_pc_version = extra_json.get('inner_pc_version') inner_pc_version = extra_json.get('inner_pc_version')
if inner_user_ip: if inner_user_ip:
put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip) tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
if inner_pc_version: if inner_pc_version:
put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version) tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
if json_content.get('event_type') == 2001: if json_content.get('event_type') == 2001:
even_data = json.loads(json_content.get('event_data')) even_data = json.loads(json_content.get('event_data'))
......
aidso_geo/core/routes/interface.py
View file @ d26f7bd6
...@@ -166,37 +166,6 @@ def task_commit(): ...@@ -166,37 +166,6 @@ def task_commit():
"status": f"缺少必要字段：{', '.join(missing_fields)}", "status": f"缺少必要字段：{', '.join(missing_fields)}",
"reqId": req_id "reqId": req_id
}) })
# if type == 'stream':
# insert_ok = commit_task(data, 'ING')
# ok = submit_background_task(main_process, data)
# if insert_ok and ok:
# return jsonify({
# "code": 200,
# "msg": 'success',
# "reqId": req_id
# })
# else:
# logger.success(f"{data['reqId']}--{platform}--{prompt}--任务提交--{type}")
# ret = redis_client.lpush("geo:task_commit:list",json.dumps(data, ensure_ascii=False))
# if ret and ret > 0:
# resp_cache_key = f"geo:task_check:resp:{req_id}"
# resp = {
# "code": 200,
# "msg": "success",
# "data": {
# "status": 'ING',
# "result": {}
# }
# }
# if type == 'stream_batch':
# _cache_set_json(resp_cache_key, resp, 600)
# else:
# _cache_set_json(resp_cache_key, resp, 18000)
# return jsonify({
# "code": 200,
# "msg": "任务已提交",
# "reqId": req_id
# })
logger.success(f"{data['reqId']}--{platform}--{prompt}--任务提交--{type}") logger.success(f"{data['reqId']}--{platform}--{prompt}--任务提交--{type}")
ret = redis_client.lpush("geo:task_commit:list",json.dumps(data, ensure_ascii=False)) ret = redis_client.lpush("geo:task_commit:list",json.dumps(data, ensure_ascii=False))
if ret and ret > 0: if ret and ret > 0:
......
aidso_geo/models/baidu_baikan_data_process.py
View file @ d26f7bd6
...@@ -28,9 +28,9 @@ def baidu_baikan_data_process_original_data(data): ...@@ -28,9 +28,9 @@ def baidu_baikan_data_process_original_data(data):
citation_list_data = json_content.get('citationListData','') citation_list_data = json_content.get('citationListData','')
err_code = json_content.get('there_is_no_answer_to_this_prompt','') err_code = json_content.get('there_is_no_answer_to_this_prompt','')
if err_code =='99': if err_code =='99':
response_content = "There is no answer to this prompt"
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions) suggestions)
response_content = "There is no answer to this prompt"
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
if citation_list_data: if citation_list_data:
......
aidso_geo/models/doubao_android_data_process.py
View file @ d26f7bd6
...@@ -2,13 +2,14 @@ import json ...@@ -2,13 +2,14 @@ import json
import traceback import traceback
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
from aidso_geo.models.eco_data_process import save_eco_data_to_bh
from aidso_geo.utils import robot_utils, bh_utils from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos from aidso_geo.utils.tos_utils import get_string_from_tos
def doubao_mobile_process_original_data(data): def doubao_mobile_process_original_data(task_data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text' file_path = f'geo/{task_data["taskId"]}/{task_data["platform"]}/original.text'
url_list = [] url_list = []
think_content = "" think_content = ""
response_content = "" response_content = ""
...@@ -19,6 +20,8 @@ def doubao_mobile_process_original_data(data): ...@@ -19,6 +20,8 @@ def doubao_mobile_process_original_data(data):
is_think = False is_think = False
think_bool = False think_bool = False
response_bool = False response_bool = False
dy_poi_list =[]
dy_eco_list =[]
try: try:
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
...@@ -195,15 +198,19 @@ def doubao_mobile_process_original_data(data): ...@@ -195,15 +198,19 @@ def doubao_mobile_process_original_data(data):
for pr in pro_data: for pr in pro_data:
if isinstance(pr,dict): if isinstance(pr,dict):
pid = pr.get('pid', '')
url = f'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id={pid}&origin_type=604'
product = { product = {
'text': pr.get('text', ''), 'text': pr.get('text', ''),
'seller_name': pr.get( 'seller_name': pr.get(
'seller_name', ''), 'seller_name', ''),
'image_url': pr.get( 'image_url': pr.get(
'image_url', ''), 'image_url', ''),
'pid': pr.get('pid', '') 'pid': pid,
'jump_url': url,
} }
media_list.append(product) media_list.append(product)
dy_eco_list.extend(media_list)
poi_data = widget_data_data.get( poi_data = widget_data_data.get(
'poi_list') 'poi_list')
...@@ -291,7 +298,8 @@ def doubao_mobile_process_original_data(data): ...@@ -291,7 +298,8 @@ def doubao_mobile_process_original_data(data):
url_list_seen.add(key) url_list_seen.add(key)
new_url_list.append(item) new_url_list.append(item)
if dy_eco_list:
save_eco_data_to_bh(task_data,'dyeco',dy_eco_list)
url_list = new_url_list url_list = new_url_list
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
...@@ -329,7 +337,7 @@ if __name__ == '__main__': ...@@ -329,7 +337,7 @@ if __name__ == '__main__':
# file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt' # file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '6a5a03b63da04b9d809ea9fd27e266f6' and platform = 'DOUBA'") f"select * from geo_commit_task where taskId = '369df725-ef53-4050-8b94-233a007950c3' and platform = 'DOUBA'")
......
aidso_geo/models/eco_data_process.py 0 → 100644
View file @ d26f7bd6
from aidso_geo.utils import bh_utils
def qianwen_android_process_taobao_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
if item.get('card_type') =='single_product':
raw_data = {
"eco_id": item.get('item_id', ''),
"eco_url": item.get('auctionURL', '') or item.get('jump_url',''),
"eco_title": item.get('title', ''),
"eco_index": index+1,
"eco_pic": item.get('pic_path', ''),
"eco_price": item.get('price', ''),
"eco_source": 'taobao',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def qianwen_android_process_damai_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
raw_data = {
"eco_id": item.get('id', ''),
"eco_url": item.get('webURL', '') ,
"eco_title": item.get('name', ''),
"eco_index": index+1,
"eco_pic": item.get('verticalPic', ''),
"eco_price": item.get('priceShowText', ''),
"eco_source": 'damai',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def doubao_android_process_douyin_eco(data,eco_list):
eco_result = []
task_id = data.get('taskId')
platform = data.get('platform')
req_id = data.get('reqId')
prompt = data.get('prompt')
for index, item in enumerate(eco_list):
raw_data = {
"eco_id": item.get('pid', ''),
"eco_url": item.get('jump_url', '') ,
"eco_title": item.get('text', ''),
"eco_index": index+1,
"eco_pic": item.get('image_url', ''),
"eco_price": item.get('priceShowText', ''),
"eco_source": 'dyeco',
"platform": platform,
"task_id": task_id,
"req_id": req_id,
"prompt": prompt
}
eco_result.append(raw_data)
return eco_result
def save_eco_data_to_bh(data,eco_type,eco_list):
platform = data.get('platform')
eco_result = []
if platform == 'TYQWA':
if eco_type == 'taobao':
eco_result = qianwen_android_process_taobao_eco(data,eco_list)
if eco_type == 'damai':
eco_result = qianwen_android_process_damai_eco(data, eco_list)
if platform == 'DOUBA':
if eco_type == 'dypoi':
...
if eco_type == 'dyeco':
eco_result =doubao_android_process_douyin_eco(data,eco_list)
...
bh_utils.insert_data('geo_eco_data',eco_result)
\ No newline at end of file
aidso_geo/models/process.py
View file @ d26f7bd6
...@@ -369,6 +369,7 @@ def filter_ai_word_list(ai_word_list_with_rank, brand_words_rank, com_word_list_ ...@@ -369,6 +369,7 @@ def filter_ai_word_list(ai_word_list_with_rank, brand_words_rank, com_word_list_
if item.get("brand") if item.get("brand")
} }
com_words_brand_set = { com_words_brand_set = {
normalize_brand(item.get("brand")) normalize_brand(item.get("brand"))
for item in com_word_list_rank for item in com_word_list_rank
...@@ -1266,7 +1267,28 @@ def add_keyword_type(ai_word_list, brand_words, com_word_list, all_keyword_with_ ...@@ -1266,7 +1267,28 @@ def add_keyword_type(ai_word_list, brand_words, com_word_list, all_keyword_with_
return result return result
def reset_rank(all_vos, brand_vos, competitor_vos):
"""
将三个列表中的 rank 重新编号为连续排名（从 1 开始）。
rank=0 的数据不参与排序，保持为 0。
Returns:
all_vos, brand_vos, competitor_vos
"""
# 合并所有对象（引用，不会复制）
all_items = all_vos + brand_vos + competitor_vos
# 参与排序的数据
ranked_items = [item for item in all_items if item.get("rank", 0) > 0]
# 按原 rank 排序
ranked_items.sort(key=lambda x: x["rank"])
# 重新编号
for new_rank, item in enumerate(ranked_items, start=1):
item["rank"] = new_rank
return all_vos, brand_vos, competitor_vos
def result_v2(response_content, data): def result_v2(response_content, data):
data = process_com_map(data) data = process_com_map(data)
reqId = data.get('reqId') reqId = data.get('reqId')
...@@ -1309,13 +1331,13 @@ def result_v2(response_content, data): ...@@ -1309,13 +1331,13 @@ def result_v2(response_content, data):
all_word_set_list = [str(x).strip() for x in all_word_set_list if str(x).strip()] all_word_set_list = [str(x).strip() for x in all_word_set_list if str(x).strip()]
# 获取所有词的排名 word + rank_list # 获取所有词的排名 word + rank_list
all_word_rank_list = get_keyword_ranks(response_content, all_word_set_list) all_word_rank_list = get_keyword_ranks(response_content, all_word_set_list)
# 获取所有词的排名 word + rank + count # 获取所有词的排名 word + rank + count
all_keyword_with_rank = convert_rank_data(all_word_rank_list) all_keyword_with_rank = convert_rank_data(all_word_rank_list)
# 获取所有词的品牌 # 获取所有词的品牌
all_keyword_with_brand = keyword_map_brand(all_word_list) all_keyword_with_brand = keyword_map_brand(all_word_list)
# 追加所有词的品牌 # 追加所有词的品牌
all_keyword_with_rank_sentiment_brand = keyword_with_brand(all_keyword_with_rank, all_keyword_with_brand) all_keyword_with_rank_sentiment_brand = keyword_with_brand(all_keyword_with_rank, all_keyword_with_brand)
...@@ -1325,6 +1347,7 @@ def result_v2(response_content, data): ...@@ -1325,6 +1347,7 @@ def result_v2(response_content, data):
com_word_list, com_word_list,
all_keyword_with_rank_sentiment_brand) all_keyword_with_rank_sentiment_brand)
# ---------------------------------调整排名----------------------------- # ---------------------------------调整排名-----------------------------
brand_words_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(brand_words_list_with_rank, brand_words_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(brand_words_list_with_rank,
...@@ -1342,13 +1365,11 @@ def result_v2(response_content, data): ...@@ -1342,13 +1365,11 @@ def result_v2(response_content, data):
# 重新计算外层rank count # 重新计算外层rank count
com_word_list_with_rank = rebuild_count_and_rank(com_word_list_with_rank) com_word_list_with_rank = rebuild_count_and_rank(com_word_list_with_rank)
else: else:
# list 调整竞争品牌词排名 去除竞争品牌词中ai的排名 # list 调整竞争品牌词排名 去除竞争品牌词中ai的排名
com_word_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(com_word_list_with_rank, com_word_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(com_word_list_with_rank,
ai_word_list_with_rank) ai_word_list_with_rank)
# ---------------------------------调整排名----------------------------- # ---------------------------------调整排名-----------------------------
# 去除all # 去除all
ai_word_list_with_rank = filter_ai_word_list(ai_word_list_with_rank, brand_words_list_with_rank, ai_word_list_with_rank = filter_ai_word_list(ai_word_list_with_rank, brand_words_list_with_rank,
com_word_list_with_rank) com_word_list_with_rank)
...@@ -1384,6 +1405,12 @@ def result_v2(response_content, data): ...@@ -1384,6 +1405,12 @@ def result_v2(response_content, data):
com_word_list_with_rank = attach_favorability_brand_score(all_sentiment_and_mentions_score, com_word_list_with_rank) com_word_list_with_rank = attach_favorability_brand_score(all_sentiment_and_mentions_score, com_word_list_with_rank)
ai_word_list_with_rank = attach_favorability_score(all_sentiment_and_mentions_score, ai_word_list_with_rank) ai_word_list_with_rank = attach_favorability_score(all_sentiment_and_mentions_score, ai_word_list_with_rank)
ai_word_list_with_rank,brand_words_list_with_rank,com_word_list_with_rank=reset_rank(ai_word_list_with_rank,
brand_words_list_with_rank,
com_word_list_with_rank)
mentionsList = build_mentions_list(brand_words_list_with_rank, all_sentiment_and_mentions) mentionsList = build_mentions_list(brand_words_list_with_rank, all_sentiment_and_mentions)
# ---------------------------------情感倾向----------------------------- # ---------------------------------情感倾向-----------------------------
...@@ -1616,6 +1643,7 @@ def platform_process(data): ...@@ -1616,6 +1643,7 @@ def platform_process(data):
'TXYBA': yuanbao_android_data_process.yuanbao_android_process_original_data, 'TXYBA': yuanbao_android_data_process.yuanbao_android_process_original_data,
'XHSA': xiaohongshu_android_data_process.xiaohongshu_android_process_original_data, 'XHSA': xiaohongshu_android_data_process.xiaohongshu_android_process_original_data,
'BK': baidu_baikan_data_process.baidu_baikan_data_process_original_data, 'BK': baidu_baikan_data_process.baidu_baikan_data_process_original_data,
'BDWX': baiduai_data_process.baiduai_process_original_data,
} }
try: try:
...@@ -1924,7 +1952,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS): ...@@ -1924,7 +1952,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
if __name__ == '__main__': if __name__ == '__main__':
data_list = bh_utils.query_data(f"select * from geo_commit_task where status = 'ING' and type !='stream' ") data_list = bh_utils.query_data(f"select * from geo_commit_task where status = 'ING' ")
# data_list = bh_utils.query_data(query_sql) # data_list = bh_utils.query_data(query_sql)
# print(data_list) # print(data_list)
# # # # # #
...@@ -1942,10 +1970,10 @@ if __name__ == '__main__': ...@@ -1942,10 +1970,10 @@ if __name__ == '__main__':
if i.get('productWordsMap'): if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap')) i['productWordsMap'] = json.loads(i.get('productWordsMap'))
type_t = i.get('type') type_t = i.get('type')
type_t = 'batch' # type_t = 'batch'
# commit_task(i,'ING') # commit_task(i,'ING')
# return task_send_queue(i,type_t) return task_send_queue(i,type_t)
return platform_process(i) # return platform_process(i)
# #
if data_list: if data_list:
......
aidso_geo/models/qianwen_android_data_process.py
View file @ d26f7bd6
...@@ -4,14 +4,15 @@ from asyncio import as_completed ...@@ -4,14 +4,15 @@ from asyncio import as_completed
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from aidso_geo.models import spider_save_tos from aidso_geo.models import spider_save_tos
from aidso_geo.models.eco_data_process import save_eco_data_to_bh
from aidso_geo.utils import robot_utils, bh_utils from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos from aidso_geo.utils.tos_utils import get_string_from_tos
def qianwen_android_process_original_data(data): def qianwen_android_process_original_data(task_data):
# def qianwen_android_process_original_data(file_path):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text' file_path = f'geo/{task_data["taskId"]}/{task_data["platform"]}/original.text'
url_list = [] url_list = []
url_list_batch = [] url_list_batch = []
think_content = "" think_content = ""
...@@ -23,12 +24,13 @@ def qianwen_android_process_original_data(data): ...@@ -23,12 +24,13 @@ def qianwen_android_process_original_data(data):
think_bool = False think_bool = False
response_bool = False response_bool = False
result = [] result = []
damai_data_list = []
try: try:
json_original = "" json_original = ""
original_content = get_string_from_tos(file_path) original_content = get_string_from_tos(file_path)
try: try:
json_original = json.loads(original_content) sse_json_data = json.loads(original_content)
sse_json_data = json.loads(json_original.get('sse')) # sse_json_data = json.loads(json_original.get('sse'))
response_content = sse_json_data.get('reply_text','') response_content = sse_json_data.get('reply_text','')
think_content = sse_json_data.get('deepthink_text','') think_content = sse_json_data.get('deepthink_text','')
paa_answer = sse_json_data.get('paa_answer','') paa_answer = sse_json_data.get('paa_answer','')
...@@ -156,6 +158,7 @@ def qianwen_android_process_original_data(data): ...@@ -156,6 +158,7 @@ def qianwen_android_process_original_data(data):
for mu in multi_load: for mu in multi_load:
if mu.get('type') == 'taoassistant_fold_product_feeds' : if mu.get('type') == 'taoassistant_fold_product_feeds' :
source_seq = mu.get('source_seq') source_seq = mu.get('source_seq')
jump_url = next( jump_url = next(
( (
...@@ -181,12 +184,13 @@ def qianwen_android_process_original_data(data): ...@@ -181,12 +184,13 @@ def qianwen_android_process_original_data(data):
query = json.loads(query) query = json.loads(query)
if isinstance(query,list): if isinstance(query,list):
search_keyword.extend(query) search_keyword.extend(query)
if isinstance(mu.get('content').get('cardData').get('data').get('items'), list): if isinstance(mu.get('content').get('cardData').get('data').get('items'), list) and not taobao_list :
for pro in mu.get('content').get('cardData').get('data').get('items'):
cateTitle = mu.get('content').get('cardData').get('data').get('cateTitle')
for pro in mu.get('content').get('cardData').get('data').get('items'):
price = (pro.get('priceShowWithIcon') or {}).get('price') or pro.get('itemPrice') or '' price = (pro.get('priceShowWithIcon') or {}).get('price') or pro.get('itemPrice') or ''
taobao_list.append({ taobao_list.append({
"title": pro.get('title'), "title": cateTitle or pro.get('title', ''),
"shop_name": next( "shop_name": next(
( (
item.get('text', '') item.get('text', '')
...@@ -201,11 +205,15 @@ def qianwen_android_process_original_data(data): ...@@ -201,11 +205,15 @@ def qianwen_android_process_original_data(data):
"jump_url": jump_url, "jump_url": jump_url,
"source_seq": source_seq, "source_seq": source_seq,
"auctionURL": pro.get('auctionURL'), "auctionURL": pro.get('auctionURL'),
"item_id": pro.get('item_id'),
'card_type':'fold_product'
}) })
if mu.get('type') == 'taoassistant_single_product': if mu.get('type') == 'taoassistant_single_product':
mu_content = mu.get('content',{}) mu_content = mu.get('content',{})
mu_content_cardData = mu_content.get('cardData',{}) mu_content_cardData = mu_content.get('cardData',{})
mu_content_cardData_data = mu_content_cardData.get('data',{}) mu_content_cardData_data = mu_content_cardData.get('data',{})
if mu_content_cardData_data: if mu_content_cardData_data:
source_seq = mu_content.get('cardId') or mu_content.get('businessId') source_seq = mu_content.get('cardId') or mu_content.get('businessId')
price = (mu_content_cardData_data.get('priceShowWithIcon') or {}).get('price') or mu_content_cardData_data.get( price = (mu_content_cardData_data.get('priceShowWithIcon') or {}).get('price') or mu_content_cardData_data.get(
...@@ -226,6 +234,8 @@ def qianwen_android_process_original_data(data): ...@@ -226,6 +234,8 @@ def qianwen_android_process_original_data(data):
"jump_url": mu_content_cardData_data.get('auctionURL'), "jump_url": mu_content_cardData_data.get('auctionURL'),
"source_seq": source_seq, "source_seq": source_seq,
"auctionURL": mu_content_cardData_data.get('auctionURL'), "auctionURL": mu_content_cardData_data.get('auctionURL'),
"item_id": mu_content_cardData_data.get('item_id'),
'card_type': 'single_product'
}) })
if taobao_list: if taobao_list:
...@@ -233,7 +243,7 @@ def qianwen_android_process_original_data(data): ...@@ -233,7 +243,7 @@ def qianwen_android_process_original_data(data):
damai_str = json.dumps(taobao_list, ensure_ascii=False) damai_str = json.dumps(taobao_list, ensure_ascii=False)
response_content += damai_str response_content += damai_str
response_content += 'render_ecom_card_widget_taobao_end:\n' response_content += 'render_ecom_card_widget_taobao_end:\n'
save_eco_data_to_bh(task_data,'taobao',taobao_list)
damai_list = [] damai_list = []
if mime_type == 'multi_load/iframe' and multi_load_type == 'damai_shows_list' and multi_load_status == 'complete': if mime_type == 'multi_load/iframe' and multi_load_type == 'damai_shows_list' and multi_load_status == 'complete':
...@@ -258,14 +268,16 @@ def qianwen_android_process_original_data(data): ...@@ -258,14 +268,16 @@ def qianwen_android_process_original_data(data):
'maxPrice': damai.get('data').get('maxPrice', ''), 'maxPrice': damai.get('data').get('maxPrice', ''),
'priceStr': damai.get('data').get('priceStr', ''), 'priceStr': damai.get('data').get('priceStr', ''),
'showVenueName': cityName + venueName, 'showVenueName': cityName + venueName,
'source_seq_id': source_seq 'source_seq_id': source_seq,
'id': damai.get('data').get('id')
}) })
damai_data_list.extend(damai_list)
if damai_list: if damai_list:
response_content += 'render_ecom_card_widget_damai_start:' response_content += 'render_ecom_card_widget_damai_start:'
damai_str = json.dumps(damai_list, ensure_ascii=False) damai_str = json.dumps(damai_list, ensure_ascii=False)
response_content += damai_str response_content += damai_str
response_content += 'render_ecom_card_widget_damai_end:\n' response_content += 'render_ecom_card_widget_damai_end:\n'
if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete': if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete':
think_content += multi_load_content.get('think_content') think_content += multi_load_content.get('think_content')
# 引用来源 回答 视频列表 # 引用来源 回答 视频列表
...@@ -295,10 +307,10 @@ def qianwen_android_process_original_data(data): ...@@ -295,10 +307,10 @@ def qianwen_android_process_original_data(data):
suggestions.append(pa.get('show_text')) suggestions.append(pa.get('show_text'))
if url_list_batch: if url_list_batch:
url_list = url_list_batch url_list = url_list_batch
if damai_data_list:
save_eco_data_to_bh(task_data, 'damai', damai_data_list)
response_content = response_content.replace("[(deep_think)]", "") response_content = response_content.replace("[(deep_think)]", "")
response_content = response_content.replace("[(multimodal_chat_think_1)]", "") response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block) suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -336,7 +348,7 @@ if __name__ == '__main__': ...@@ -336,7 +348,7 @@ if __name__ == '__main__':
# qianwen_android_process_original_data(file_path2) # qianwen_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '27721b42-3368-4526-b092-d3157aaa617a' and platform = 'TYQWA'") data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId = '4b285be0-c62c-48f7-9d72-2892511c306c' and platform = 'TYQWA'")
def handle_item(i): def handle_item(i):
if i.get('comWordsMap'): if i.get('comWordsMap'):
......
aidso_geo/models/spider_save_tos.py
View file @ d26f7bd6
...@@ -509,6 +509,8 @@ def save_data_to_tos(target_dir, content, file_name): ...@@ -509,6 +509,8 @@ def save_data_to_tos(target_dir, content, file_name):
content = xiaohongshu_android_process_quote(content) content = xiaohongshu_android_process_quote(content)
elif platform == 'BK': elif platform == 'BK':
content = baikan_process_quote(content) content = baikan_process_quote(content)
elif platform == 'BDWX':
content = baiduai_process_quote(content)
else: else:
content = doubao_process_quote(content) content = doubao_process_quote(content)
task_id = target_dir.split('/')[1] task_id = target_dir.split('/')[1]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment