Skip to content

A
aidso-data
Commit bcb7ea21 authored by Yaowentong's avatar Yaowentong
Browse files
Options

电商搜索接口更新

parent e2cb3b9a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 261 additions and 90 deletions
aidso_geo/config/base_config.py
View file @ bcb7ea21
...@@ -187,6 +187,67 @@ def init_redis4(): ...@@ -187,6 +187,67 @@ def init_redis4():
return redis_client return redis_client
except Exception as e: except Exception as e:
return None return None
import uuid
def deduplicate_redis_list(redis_client, key, batch_size=500):
old_count = redis_client.llen(key)
if old_count == 0:
return 0, 0
temp_key = f"{key}:dedup:{uuid.uuid4().hex}"
seen_req_ids = set()
new_count = 0
try:
for start in range(0, old_count, batch_size):
end = min(start + batch_size - 1, old_count - 1)
values = redis_client.lrange(key, start, end)
unique_values = []
for value in values:
if isinstance(value, bytes):
text = value.decode("utf-8")
else:
text = value
try:
data = json.loads(text)
except (TypeError, json.JSONDecodeError):
# 非法 JSON 暂时保留
unique_values.append(value)
continue
req_id = data.get("reqId")
# 没有 reqId 的数据保留
if not req_id:
unique_values.append(value)
continue
if req_id in seen_req_ids:
continue
seen_req_ids.add(req_id)
unique_values.append(value)
if unique_values:
# RPUSH 保持原 List 的顺序
redis_client.rpush(temp_key, *unique_values)
new_count += len(unique_values)
if new_count > 0:
# 原子地用临时 List 覆盖原 List
redis_client.rename(temp_key, key)
else:
redis_client.delete(key)
return old_count, new_count
except Exception:
# 发生异常时保留原 List，只清理临时数据
redis_client.delete(temp_key)
raise
if __name__ == '__main__': if __name__ == '__main__':
key_list = ['BDAI:geo:stream_batch:list', key_list = ['BDAI:geo:stream_batch:list',
...@@ -217,11 +278,17 @@ if __name__ == '__main__': ...@@ -217,11 +278,17 @@ if __name__ == '__main__':
'XHSA:geo:batch:list', 'XHSA:geo:batch:list',
'geo:task_commit:list'] 'geo:task_commit:list']
redis5 = init_redis4() redis_client8 = init_redis8()
# delete(redis_key)
print(redis5.delete("mt:DP:snipaste_v3:only_content")) # old_count, new_count = deduplicate_redis_list(
# print(redis5.delete('mt:snipaste_v3:only_content')) # redis_client8,
# print(redis5.delete('mt:snipaste_v3:with_share')) # "geo:task_commit:list",
# batch_size=500,
# )
print(redis_client8.llen("geo:task_commit:list"))
......
aidso_geo/core/down_load_bot.py
View file @ bcb7ea21
This diff is collapsed.
aidso_geo/core/meituan_process.py
View file @ bcb7ea21
...@@ -74,10 +74,10 @@ t1.channel, ...@@ -74,10 +74,10 @@ t1.channel,
ifnull(t1.is_share,0) as is_share ifnull(t1.is_share,0) as is_share
from (SELECT * from (SELECT *
FROM geo_third_task_data FROM geo_third_task_data
where status != 'SUCCESS') t1 where status = 'ING') t1
left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2
on t1.reqId = t2.reqId on t1.reqId = t2.reqId
where t2.reqId is null where t2.reqId is null
""" """
) )
...@@ -134,7 +134,7 @@ def get_result_api(): ...@@ -134,7 +134,7 @@ def get_result_api():
SELECT * SELECT *
FROM geo_third_task_data FROM geo_third_task_data
WHERE status = 'PROCESSING' WHERE status = 'PROCESSING'
LIMIT 500) t1 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 on t1.reqId = t2.reqId where t2.status = 'SUCCESS' ) t1 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 on t1.reqId = t2.reqId where t2.status = 'SUCCESS'
""") """)
if not query_result: if not query_result:
......
aidso_geo/core/routes/interface.py
View file @ bcb7ea21
...@@ -342,8 +342,6 @@ def check_eco(): ...@@ -342,8 +342,6 @@ def check_eco():
"message": "reqids必须是数组", "message": "reqids必须是数组",
"data": None, "data": None,
}) })
# 过滤空值并去重，同时保持原顺序。
reqids = list(dict.fromkeys( reqids = list(dict.fromkeys(
str(req_id).strip() str(req_id).strip()
for req_id in reqids for req_id in reqids
...@@ -375,11 +373,23 @@ def check_eco(): ...@@ -375,11 +373,23 @@ def check_eco():
SELECT * SELECT *
FROM geo_eco_data FROM geo_eco_data
WHERE req_id IN ({placeholders}) WHERE req_id IN ({placeholders})
""", """,
tuple(reqids), tuple(reqids),
) )
if not result: if not result:
return jsonify({
"code": 200,
"message": "没有查询到数据",
"data": None,
})
statuses = [
str(item.get("isCalculateSearchWord"))
for item in result
]
if all(status == "1" for status in statuses):
return jsonify({ return jsonify({
"code": 200, "code": 200,
"message": "处理成功", "message": "处理成功",
...@@ -387,25 +397,57 @@ def check_eco(): ...@@ -387,25 +397,57 @@ def check_eco():
"reqids": reqids, "reqids": reqids,
"keyword": keyword, "keyword": keyword,
"query_rows": len(result), "query_rows": len(result),
"insert_rows": len(result), "insert_rows": 0,
}, },
}) })
eco_result = process_eco_product_relation( if any(status == "0" for status in statuses):
return jsonify({
"code": 400,
"message": "处理中",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": 0,
},
})
for item in result:
item["isCalculateSearchWord"] = 0
bh_utils.insert_data(
"geo_eco_data",
result,
)
ok = submit_background_task(
process_eco_product_relation,
result, result,
keyword, keyword,
) )
bh_utils.insert_data("geo_eco_data",eco_result) if not ok:
for item in result:
item["isCalculateSearchWord"] = None
bh_utils.insert_data(
"geo_eco_data",
result,
)
return jsonify({
"code": 500,
"message": "后台任务提交失败",
"data": None,
})
return jsonify({ return jsonify({
"code": 200, "code": 400,
"message": "处理成功", "message": "任务已提交，正在处理",
"data": { "data": {
"reqids": reqids, "reqids": reqids,
"keyword": keyword, "keyword": keyword,
"query_rows": len(result), "query_rows": len(result),
"insert_rows": len(eco_result), "insert_rows": 0,
}, },
}) })
......
aidso_geo/models/doubao_data_process.py
View file @ bcb7ea21
...@@ -116,13 +116,13 @@ def doubao_process_original_data(data): ...@@ -116,13 +116,13 @@ def doubao_process_original_data(data):
index = next( index = next(
( (
item["text_card"]["index"] item["text_card"]["index"]
for item in url_list for item in (url_list or [])
if item.get("text_card", {}).get("title") == target_title if item.get("text_card", {}).get("title") == target_title
), ),
None None
) )
if index: if index is not None:
response_content+=f"[reference:{index}]" response_content += f"[reference:{index}]"
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( "block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
...@@ -137,9 +137,10 @@ def doubao_process_original_data(data): ...@@ -137,9 +137,10 @@ def doubao_process_original_data(data):
is_think = False is_think = False
continue continue
if json_content.get('patch_op')[0].get("patch_object") == 50: if json_content.get('patch_op')[0].get("patch_object") == 50:
for sug in json.loads( if json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2"):
for sug in json.loads(
json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")): json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")):
suggestions.append(sug.get("content")) suggestions.append(sug.get("content"))
if is_think: if is_think:
if json_content.get("text"): if json_content.get("text"):
think_content += json_content.get("text") think_content += json_content.get("text")
...@@ -157,6 +158,7 @@ def doubao_process_original_data(data): ...@@ -157,6 +158,7 @@ def doubao_process_original_data(data):
response_content = content_block[0].get("content").get("text_block").get("text") response_content = content_block[0].get("content").get("text_block").get("text")
suggestions = list(set(suggestions)) suggestions = list(set(suggestions))
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions,rich_media_block) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -204,7 +206,7 @@ if __name__ == '__main__': ...@@ -204,7 +206,7 @@ if __name__ == '__main__':
# 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35' # 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35'
# ] # ]
# for task in task_id_list: # for task in task_id_list:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'e5b63f95-f713-4c51-b282-78c8c1c130ad' and platform = 'DB'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '9bd06279-57d5-4da7-b8a7-f533fb7365f9' and platform = 'DB'")
# # # # # #
# # # # # #
......
aidso_geo/models/douyinai_data_process.py
View file @ bcb7ea21
...@@ -31,11 +31,7 @@ def douyin_ai_process_original_data(data): ...@@ -31,11 +31,7 @@ def douyin_ai_process_original_data(data):
# 提取并解析JSON数据 # 提取并解析JSON数据
data_str = item.split("data:")[1] data_str = item.split("data:")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
print(json_data)
print('----')
print('----')
print('----')
print('----')
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
......
aidso_geo/models/eco_data_process.py
View file @ bcb7ea21
...@@ -124,6 +124,9 @@ def process_eco_product_relation( ...@@ -124,6 +124,9 @@ def process_eco_product_relation(
**relation_data, **relation_data,
} }
) )
for item in final_result:
item["isCalculateSearchWord"] = 1
bh_utils.insert_data('geo_eco_data', final_result)
return final_result return final_result
...@@ -388,10 +391,13 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -388,10 +391,13 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
if search_goods_word: if search_goods_word:
eco_result = process_eco_product_relation(eco_result, search_goods_word) eco_result = process_eco_product_relation(eco_result, search_goods_word)
for item in eco_result:
item["isCalculateSearchWord"] = None
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
if __name__ == '__main__': if __name__ == '__main__':
result = bh_utils.query_data("select * from geo_eco_data where req_id in ('794107f1e46b436f81f18aac77043f89')") result = bh_utils.query_data("select * from geo_eco_data where req_id in ('3a9edb7d-c98b-4ad2-8ae5-f53c4803ba99')")
eco_result = process_eco_product_relation(result, "小米洗地机") eco_result = process_eco_product_relation(result, "忠芝")
bh_utils.insert_data('geo_eco_data', eco_result) for item in result:
# item["isCalculateSearchWord"] = None
bh_utils.insert_data('geo_eco_data', result)
aidso_geo/models/process.py
View file @ bcb7ea21
...@@ -2018,7 +2018,7 @@ if __name__ == '__main__': ...@@ -2018,7 +2018,7 @@ if __name__ == '__main__':
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc") # data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status = 'ING'") data_list = bh_utils.query_data("select * from geo_commit_task where pt = 20260806 and status = 'ING' and platform !='DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'") # data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ") # data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
...@@ -2030,7 +2030,7 @@ if __name__ == '__main__': ...@@ -2030,7 +2030,7 @@ if __name__ == '__main__':
i["keywords"] = safe_json_loads(i.get("keywords"), []) i["keywords"] = safe_json_loads(i.get("keywords"), [])
i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), []) i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), [])
type_t = i.get('type') type_t = i.get('type')
type_t = 'batch' type_t = 'stream_batch'
# commit_task(i,'ING') # commit_task(i,'ING')
return task_send_queue(i,type_t) return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
......
aidso_geo/models/qianwen_android_data_process.py
View file @ bcb7ea21
...@@ -394,29 +394,29 @@ def qianwen_android_process_original_data(task_data): ...@@ -394,29 +394,29 @@ def qianwen_android_process_original_data(task_data):
mu_content_model_input = mu_content.get('modelInput') mu_content_model_input = mu_content.get('modelInput')
mu_content_pois = mu_content.get('pois') mu_content_pois = mu_content.get('pois')
if mu_content_pois:
for poi in mu_content_pois: for poi in mu_content_pois:
poi_summary = poi.get('summary1') poi_summary = poi.get('summary1')
distance_formatted = poi.get('distance_formatted') distance_formatted = poi.get('distance_formatted')
photos = poi.get('photos') photos = poi.get('photos')
distance_address = poi.get('address') distance_address = poi.get('address')
cost = poi.get('cost') cost = poi.get('cost')
poi_id = mu_content_model_input.get("poi_id") poi_id = mu_content_model_input.get("poi_id")
gaode_result = { gaode_result = {
"url": f"https://www.amap.com/place/{poi_id}", "url": f"https://www.amap.com/place/{poi_id}",
"source_seq": mu_source_seq, "source_seq": mu_source_seq,
"name": mu_content_model_input.get("name"), "name": mu_content_model_input.get("name"),
"poi_id": poi_id, "poi_id": poi_id,
"cost": cost, "cost": cost,
"poi_summary": poi_summary, "poi_summary": poi_summary,
"distance_formatted": distance_formatted, "distance_formatted": distance_formatted,
"photos": photos[0].get("url", "") if isinstance(photos, "photos": photos[0].get("url", "") if isinstance(photos,
list) and photos and isinstance( list) and photos and isinstance(
photos[0], dict) else "", photos[0], dict) else "",
"address": distance_address, "address": distance_address,
} }
gaode_list.append(gaode_result) gaode_list.append(gaode_result)
gaode_data_list.extend(gaode_list) gaode_data_list.extend(gaode_list)
...@@ -557,7 +557,7 @@ if __name__ == '__main__': ...@@ -557,7 +557,7 @@ if __name__ == '__main__':
# 04b2b061-a5c0-4429-8d3b-d2b6504f1e8b # 04b2b061-a5c0-4429-8d3b-d2b6504f1e8b
# data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'") # data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'")
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '9b82af12-97ba-47e0-aaca-4d88b93c70e9' and platform = 'TYQWA'") f"select * from geo_commit_task where taskId = '8e38e641-38fa-4105-9a73-0031132c7c28' and platform = 'TYQWA'")
def handle_item(i): def handle_item(i):
......
aidso_geo/utils/ai_utils.py
View file @ bcb7ea21
...@@ -252,7 +252,6 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -252,7 +252,6 @@ def ai_get_product_relation_spu(product_list, keyword):
content = json.loads(content) content = json.loads(content)
product_words = content.get("product_words", []) product_words = content.get("product_words", [])
print(product_words if isinstance(product_words, list) else [])
return product_words if isinstance(product_words, list) else [] return product_words if isinstance(product_words, list) else []
except Exception as e: except Exception as e:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment