Skip to content

A
aidso-data
Commit bcb7ea21 authored by Yaowentong's avatar Yaowentong
Browse files
Options

电商搜索接口更新

parent e2cb3b9a
Show whitespace changes
Inline Side-by-side
Showing with 261 additions and 90 deletions
aidso_geo/config/base_config.py
View file @ bcb7ea21
...@@ -187,6 +187,67 @@ def init_redis4(): ...@@ -187,6 +187,67 @@ def init_redis4():
return redis_client return redis_client
except Exception as e: except Exception as e:
return None return None
import uuid
def deduplicate_redis_list(redis_client, key, batch_size=500):
old_count = redis_client.llen(key)
if old_count == 0:
return 0, 0
temp_key = f"{key}:dedup:{uuid.uuid4().hex}"
seen_req_ids = set()
new_count = 0
try:
for start in range(0, old_count, batch_size):
end = min(start + batch_size - 1, old_count - 1)
values = redis_client.lrange(key, start, end)
unique_values = []
for value in values:
if isinstance(value, bytes):
text = value.decode("utf-8")
else:
text = value
try:
data = json.loads(text)
except (TypeError, json.JSONDecodeError):
# 非法 JSON 暂时保留
unique_values.append(value)
continue
req_id = data.get("reqId")
# 没有 reqId 的数据保留
if not req_id:
unique_values.append(value)
continue
if req_id in seen_req_ids:
continue
seen_req_ids.add(req_id)
unique_values.append(value)
if unique_values:
# RPUSH 保持原 List 的顺序
redis_client.rpush(temp_key, *unique_values)
new_count += len(unique_values)
if new_count > 0:
# 原子地用临时 List 覆盖原 List
redis_client.rename(temp_key, key)
else:
redis_client.delete(key)
return old_count, new_count
except Exception:
# 发生异常时保留原 List，只清理临时数据
redis_client.delete(temp_key)
raise
if __name__ == '__main__': if __name__ == '__main__':
key_list = ['BDAI:geo:stream_batch:list', key_list = ['BDAI:geo:stream_batch:list',
...@@ -217,11 +278,17 @@ if __name__ == '__main__': ...@@ -217,11 +278,17 @@ if __name__ == '__main__':
'XHSA:geo:batch:list', 'XHSA:geo:batch:list',
'geo:task_commit:list'] 'geo:task_commit:list']
redis5 = init_redis4() redis_client8 = init_redis8()
# delete(redis_key)
print(redis5.delete("mt:DP:snipaste_v3:only_content")) # old_count, new_count = deduplicate_redis_list(
# print(redis5.delete('mt:snipaste_v3:only_content')) # redis_client8,
# print(redis5.delete('mt:snipaste_v3:with_share')) # "geo:task_commit:list",
# batch_size=500,
# )
print(redis_client8.llen("geo:task_commit:list"))
......
aidso_geo/core/down_load_bot.py
View file @ bcb7ea21
...@@ -661,7 +661,7 @@ def qian_report(phone, begin, end, plan_name, platform=None): ...@@ -661,7 +661,7 @@ def qian_report(phone, begin, end, plan_name, platform=None):
"platform": plat_form_map[q.get('platform')], "platform": plat_form_map[q.get('platform')],
"content": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/context.txt"), "content": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/context.txt"),
"quote": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/quote.txt"), "quote": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/quote.txt"),
"created_at": req_time_map[q.get('reqId')] "created_at": q.get('pt')
} }
result.append(r) result.append(r)
all_file = f"/Users/yaowentong/Desktop/{plan_name}_all.txt" all_file = f"/Users/yaowentong/Desktop/{plan_name}_all.txt"
...@@ -670,31 +670,31 @@ def qian_report(phone, begin, end, plan_name, platform=None): ...@@ -670,31 +670,31 @@ def qian_report(phone, begin, end, plan_name, platform=None):
for item in result: for item in result:
f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n") f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n")
# remove_keys = {"site_icon", "task_id", "quto_id"} remove_keys = {"site_icon", "task_id", "quto_id"}
# data = [] data = []
# with open(all_file, "r", encoding="utf-8") as f: with open(all_file, "r", encoding="utf-8") as f:
# for line in f: for line in f:
# line = line.strip() line = line.strip()
# if not line: if not line:
# continue continue
#
# item = json.loads(line) item = json.loads(line)
#
# quote = item.get("quote") quote = item.get("quote")
# if quote: if quote:
# quote_list = json.loads(quote) # 把 quote 字符串转成 list quote_list = json.loads(quote) # 把 quote 字符串转成 list
#
# for one in quote_list: # one 是每个 dict for one in quote_list: # one 是每个 dict
# if isinstance(one, dict): if isinstance(one, dict):
# for k in remove_keys: for k in remove_keys:
# one.pop(k, None) one.pop(k, None)
#
# item["quote"] = json.dumps(quote_list, ensure_ascii=False) item["quote"] = json.dumps(quote_list, ensure_ascii=False)
# data.append(item) data.append(item)
#
# with open(lite_file, "w", encoding="utf-8") as f: with open(lite_file, "w", encoding="utf-8") as f:
# for item in data: for item in data:
# f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n") f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n")
def zhou_report(phone, begin, end, brand_name): def zhou_report(phone, begin, end, brand_name):
...@@ -1290,12 +1290,71 @@ def txt_to_excel(txt_file, excel_file): ...@@ -1290,12 +1290,71 @@ def txt_to_excel(txt_file, excel_file):
print(f"Excel 文件：{excel_file}") print(f"Excel 文件：{excel_file}")
def qian_report2(req_ids,plan_name):
# req_list = get_req_id(phone, begin, end)
# req_ids = []
# req_time_map = {}
# for item in req_list:
# if item.get('plan_name') == plan_name:
# req_id = item.get("req_id")
# created_at = item.get("created_at")
# req_ids.append(req_id)
# req_time_map[req_id] = created_at
req_id_sql = ",".join([f"'{req_id}'" for req_id in req_ids])
query_sql = f"select * from geo_commit_task where reqId in ({req_id_sql})"
query_list = bh_utils.query_data(query_sql)
result = []
if query_list:
for q in query_list:
print( q.get('reqId'))
r = {
"thinkingEnabled":q.get('thinkingEnabled'),
"prompt": q.get('prompt'),
"reqId": q.get('reqId'),
"platform": plat_form_map[q.get('platform')],
"content": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/context.txt"),
"quote": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/quote.txt"),
"created_at": q.get('pt')
}
result.append(r)
all_file = f"/Users/yaowentong/Desktop/{plan_name}_all.txt"
lite_file = f"/Users/yaowentong/Desktop/{plan_name}_lite.txt"
with open(all_file, "w", encoding="utf-8") as f:
for item in result:
f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n")
remove_keys = {"site_icon", "task_id", "quto_id"}
data = []
with open(all_file, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
item = json.loads(line)
quote = item.get("quote")
if quote:
quote_list = json.loads(quote) # 把 quote 字符串转成 list
for one in quote_list: # one 是每个 dict
if isinstance(one, dict):
for k in remove_keys:
one.pop(k, None)
item["quote"] = json.dumps(quote_list, ensure_ascii=False)
data.append(item)
with open(lite_file, "w", encoding="utf-8") as f:
for item in data:
f.write(json.dumps(item, ensure_ascii=False) + "\n\n\n")
# ========================= # =========================
# 使用示例 # 使用示例
# ========================= # =========================
if __name__ == "__main__": if __name__ == "__main__":
plan_name = "/Users/yaowentong/Desktop/Oral-B（0714~0716）_all.txt" req_list = ['e49ee7f8-7e71-47bd-9d10-10f062647697','a2695d7a-e475-40b8-b5ab-e4123f642d76','a1560a90-23a3-468c-a691-3ae66929bcfb','2078e9cd-6788-4987-b4b9-c2d3a04e8a1e','fccdcef7-91d4-41d8-a436-adcb73700130','b7f841ac-2eed-4c3e-8d53-65aa12315a49','47d57e4a-5588-4f40-8650-ef27eaf6da70','9ddc6d73-876d-4fda-b803-38aec72548e6','9dd7df07-3fec-4c76-86ec-07c2f9b456ab','3517e57f-9416-4226-9e11-6b7afb9d35e4','24cb447a-21b8-41ea-82a0-ebcd62e0f41d','f340eb27-ddaa-4e06-af19-5d99e341af0d','63fa1930-67b8-48ca-9093-c5493d281fb8','e860be96-5a76-49d9-b463-664710e07d76','765d7c2c-a790-41d4-ba7d-68bc637d433c','f2ac7a93-ccb6-403c-9f67-9baa1c7ac694','a5650d00-f5cd-4614-96c8-c5470ff8aa3a','9a21345b-c2b8-4971-909c-3cc064f37946','1e294b97-ffd2-4f2b-8a09-19712d27b2d8','15592422-4138-4edc-985f-39f277819108','b35ea5da-0e06-444a-bbb1-b767afa1cb09','435ea67b-4419-4f8f-bcaa-8d2792ebb4a6','145b7a20-1d41-43f9-add0-5a5fd2493ff2','c29be6d6-9616-4ce7-bdcb-4f072c63d57a','d9733d4a-5852-4d50-bf74-6cffb79cd252','7d0c25f9-ecd1-4e42-9d14-8174069d609e','828f5e2b-e9a6-4d71-ab42-71df9e8947e7','01956eeb-fa19-4639-9bfe-131953ad7c14','f4e0296f-607d-44a8-a116-f9f5b5acfe08','29600e8b-c197-4473-96b5-c3d631ab9abe','439fc7b4-11f9-48ad-b6a3-6c08fdb0653f','294ad0f6-817b-4ab5-b896-de9cdffc9e12','4133491f-0d41-4c09-a1f9-3d1fd3142801','167c1b01-bd2f-47ca-829a-66714070b034','38fe3c6b-779c-40b8-9632-578a2e56841e','3a2438c2-cdd5-469e-b05c-3a8c074ed914','8f1368fd-eafe-406f-8a6c-92c543e7ca80','3f287fa3-80e9-45b8-b67d-8b62ab42ba1f','7f777d9d-2ab6-4b16-9c80-273c58f4fcdd','a8172551-d28e-40fe-bb62-fdb9dfb3994c','c68d1a5a-aee2-489a-b721-2c8f0fb21751','9da2b561-89f6-41fc-bf1e-46e1c8c1d73f','a005cba6-3d6f-42d1-a41d-3a49b9c50da6','24a2c763-9c68-49e1-8f6d-07fbd3a03829','2df5e59f-45e8-4d27-b826-e83db7588e5e','4b6bcbaf-e4d9-4404-a3f3-de0880cd6acf','afe77b23-b04b-40b3-bd01-8c313d8ac8f0','bc865bab-6320-44c5-8f7d-b26802209ab7','9f0a9e74-c5c0-4d89-8d0a-9bbbdac2f1c8','04ed4c8a-92d1-4bca-a5fd-3172f46a09a6','0bdc1cf0-f734-430e-bf8c-7075ae27f0f3','6bfc60d4-5796-4cb2-91a9-ba6b85cf424c','feb0d838-303f-4cb2-8f81-a53e76c4fd2f','ce865c09-b29c-4d12-b8eb-d02cce9c9672','9601342d-aef3-4bda-ad18-bdeb0d4c52b8','72742f91-7e4e-4854-be1a-8f5895511451','51c53925-5ff0-4915-b329-1cbad89f5619','da854174-55e3-46b0-865b-1c18f894f908','5a8133f2-31e6-4735-a559-eb0cd0169248','ac6c17cc-e9f8-4c60-8590-b866140bf70a','9c7f3153-cc8a-43be-a719-fa6a320110c7','1bd034ff-dc1b-4a20-b98f-9bd29c68b68b','1331fb78-deae-4171-9cc6-1abcce00b2d4','7ca71831-6828-43cf-9019-b068b5962f33','bd169869-011d-4dcb-a5bc-a20abe473da4','a0be46ac-2d80-4b3b-909d-b194c1caf91e','98b594c1-f10c-4d6c-a4e6-be4a8bdb1964','63dae6aa-4082-419e-a344-4289ba5c9901','dd67abc4-3421-44e7-8b20-f58c7c68f067','5e6b759a-6ad9-4e98-aadb-499422f231da','75c905f3-f1a8-46a7-9d9b-cbd6cd08f42a','dfeb515a-78b2-4aed-a9d7-c3bc1a514f8d','b565eb6a-e66f-45b2-9cfd-ff557817e86a','97373e1a-b118-46a1-8e13-b29ea932806d','910fd1a7-c745-4da1-9a7d-30ef513e15b4','d8e65bc2-5825-4fa3-a120-102f7af0e039','69e735c1-8ce9-41dd-8e10-a3127af2406a','3a7a5550-aabe-41f7-ad33-d8b2a7c28948','2bdc094a-bd0e-4878-8ad1-b4640e68cfac','c5186a50-4c44-402a-861c-ce53e26a306b','6af52a28-7aaa-4352-9d88-90790878602e','0e906b34-a898-4a2b-ace5-564f2dbda170','77f0b68b-9e8b-4a9a-ac17-6a920dce9ee1','1fec9e2b-ddb5-48db-befe-e3a644067e62','fa1e2e0a-7dbf-4cef-a5bf-dbfcbf4f701e','2c8a6e4b-d4ce-4291-808f-abe028404652','484e4381-4c9f-4099-95c6-1b7ef52c6711','21460360-0e13-4a5d-9d1a-f15593c3bab9','3463d72b-9ee5-4191-bf4d-dc1731a59816','f501b913-9b61-4f44-86a1-6971a94c08a1','54bd3f65-bb0d-480c-a1d9-7c337c633c31','bd8cca4e-5bff-424f-8675-3c32ffa15066','d58a7960-2261-4ed1-a437-8748ab2f0307','caea36b7-230c-4930-986f-ada0cc28b2d2','515c9a58-2eb3-4836-ada5-f1e2b220fc26','cfed2f60-f233-4b8a-9f94-774b2785dba9','078c0464-f917-4df6-bf52-583bf9139ce3','84afcfc3-bbb1-4bfd-b3f7-4b795f6beecc','c32e234f-e2c7-41bd-b1cc-c8e7bff9daab','f0700778-3a3b-4950-9486-f2de7b5bca2f','35f1bc35-a98d-42a4-9982-0eecd24bb69b','6a9fb614-9b40-488a-979a-f94bd65e974d','042690b2-62a1-4f2d-b519-eacef8fc9ed7','dad0bf58-ead9-4866-b4b0-ec74236851d6','a923c37e-670f-45c0-b378-08bd033fd09a','3bad0631-cb94-4ca9-aeca-1e1b8066d6a9','4c11caf6-5647-4e81-af6e-c34ee6dd90d5','4ad5002e-2f8e-42e2-8889-385206b7c5d5','06543bb4-fb7b-4bdb-950b-5c303171b413','21d337d8-3b2e-43b9-8c58-b5fb327c0224','5cf52bdb-6bf7-4e2a-b2f5-d83a42109a01','32a2cad5-686d-443c-a824-af4411f6b513','496bb514-7b13-444a-b1b7-e47fa6d07323','679ea973-2df5-4e2a-b9c7-63518ce4d3f7','63d0a699-9ca6-4859-9d3e-aa01cacc62dd','a953a6ff-7812-43a2-af3a-9b3d51fe0bc2','9958488b-6751-4e2a-8d5a-7d0f39490536','e74a9f1a-109d-4249-8cc1-cb9d528aa8ba','fd1cfbd4-611e-4190-a55c-29f468d02da4','d9e55f46-dc9e-4f99-a86a-4ba86a5133c8','278156a2-457e-4d80-8d36-0be1b29e1d5d','00e65cb3-3ecb-40d2-92e9-4a15ee78c4f7','328edc10-ad06-44c4-b00b-7db6d11ccd7a','9ba47b9a-4810-48f4-8ca6-8699109b35a7','f962ee62-7fb9-4f7e-bd1c-e3b50a42e544','5ba63557-6b78-4c3e-8120-d42a2db0cbf9','ae3cfc92-53be-4de3-a457-002bff636960','1c31b295-8339-4fcc-83ae-72c2d6b192cc','a0abb2b8-54f9-4636-8caa-963c89398af5','4f117c0a-309b-4601-9d4b-e600f20e2d90','a8c46fcb-865c-4328-9709-0ca7ad444861','bc882877-4a97-40c5-95e1-5585e3c6c756','ec297cce-9aa5-4471-92fd-3a8b010f0fbe','b339eafd-483f-42c8-8369-6e633d73a7e3','c11d9b31-24ca-4822-9cf0-053477145fe0','cf974b31-ca64-47a5-b6cb-08b23148fc10','1747f918-94fb-43c0-b34f-06be3ec62199','1652fba3-4fc0-4368-8180-fd55705aa6db','5dfafcf0-7add-4483-81f9-190bd30fd8c3','8ec80c71-5169-4be1-bb01-4e36e7ad2cf0']
plan_name2 = "/Users/yaowentong/Desktop/Oral-B（0714~0716）_all.xlsx" qian_report2(req_list,'海尔麦浪冰箱')
qian_report(15100000026,'2026-07-14','2026-07-16','Oral-B（0714~0716）') \ No newline at end of file
txt_to_excel(plan_name,plan_name2)
# qian_report(15100000026,'2026-06-21','2026-06-21','oral')
aidso_geo/core/meituan_process.py
View file @ bcb7ea21
...@@ -74,7 +74,7 @@ t1.channel, ...@@ -74,7 +74,7 @@ t1.channel,
ifnull(t1.is_share,0) as is_share ifnull(t1.is_share,0) as is_share
from (SELECT * from (SELECT *
FROM geo_third_task_data FROM geo_third_task_data
where status != 'SUCCESS') t1 where status = 'ING') t1
left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2
on t1.reqId = t2.reqId on t1.reqId = t2.reqId
where t2.reqId is null where t2.reqId is null
...@@ -134,7 +134,7 @@ def get_result_api(): ...@@ -134,7 +134,7 @@ def get_result_api():
SELECT * SELECT *
FROM geo_third_task_data FROM geo_third_task_data
WHERE status = 'PROCESSING' WHERE status = 'PROCESSING'
LIMIT 500) t1 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 on t1.reqId = t2.reqId where t2.status = 'SUCCESS' ) t1 left join (select * from geo_commit_task where pt > date_format(date_sub(now(), 10), '%Y%m%d')) t2 on t1.reqId = t2.reqId where t2.status = 'SUCCESS'
""") """)
if not query_result: if not query_result:
......
aidso_geo/core/routes/interface.py
View file @ bcb7ea21
...@@ -342,8 +342,6 @@ def check_eco(): ...@@ -342,8 +342,6 @@ def check_eco():
"message": "reqids必须是数组", "message": "reqids必须是数组",
"data": None, "data": None,
}) })
# 过滤空值并去重，同时保持原顺序。
reqids = list(dict.fromkeys( reqids = list(dict.fromkeys(
str(req_id).strip() str(req_id).strip()
for req_id in reqids for req_id in reqids
...@@ -380,6 +378,18 @@ def check_eco(): ...@@ -380,6 +378,18 @@ def check_eco():
) )
if not result: if not result:
return jsonify({
"code": 200,
"message": "没有查询到数据",
"data": None,
})
statuses = [
str(item.get("isCalculateSearchWord"))
for item in result
]
if all(status == "1" for status in statuses):
return jsonify({ return jsonify({
"code": 200, "code": 200,
"message": "处理成功", "message": "处理成功",
...@@ -387,25 +397,57 @@ def check_eco(): ...@@ -387,25 +397,57 @@ def check_eco():
"reqids": reqids, "reqids": reqids,
"keyword": keyword, "keyword": keyword,
"query_rows": len(result), "query_rows": len(result),
"insert_rows": len(result), "insert_rows": 0,
},
})
if any(status == "0" for status in statuses):
return jsonify({
"code": 400,
"message": "处理中",
"data": {
"reqids": reqids,
"keyword": keyword,
"query_rows": len(result),
"insert_rows": 0,
}, },
}) })
for item in result:
item["isCalculateSearchWord"] = 0
bh_utils.insert_data(
"geo_eco_data",
result,
)
eco_result = process_eco_product_relation( ok = submit_background_task(
process_eco_product_relation,
result, result,
keyword, keyword,
) )
bh_utils.insert_data("geo_eco_data",eco_result) if not ok:
for item in result:
item["isCalculateSearchWord"] = None
bh_utils.insert_data(
"geo_eco_data",
result,
)
return jsonify({ return jsonify({
"code": 200, "code": 500,
"message": "处理成功", "message": "后台任务提交失败",
"data": None,
})
return jsonify({
"code": 400,
"message": "任务已提交，正在处理",
"data": { "data": {
"reqids": reqids, "reqids": reqids,
"keyword": keyword, "keyword": keyword,
"query_rows": len(result), "query_rows": len(result),
"insert_rows": len(eco_result), "insert_rows": 0,
}, },
}) })
......
aidso_geo/models/doubao_data_process.py
View file @ bcb7ea21
...@@ -116,13 +116,13 @@ def doubao_process_original_data(data): ...@@ -116,13 +116,13 @@ def doubao_process_original_data(data):
index = next( index = next(
( (
item["text_card"]["index"] item["text_card"]["index"]
for item in url_list for item in (url_list or [])
if item.get("text_card", {}).get("title") == target_title if item.get("text_card", {}).get("title") == target_title
), ),
None None
) )
if index: if index is not None:
response_content+=f"[reference:{index}]" response_content += f"[reference:{index}]"
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( "block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
...@@ -137,6 +137,7 @@ def doubao_process_original_data(data): ...@@ -137,6 +137,7 @@ def doubao_process_original_data(data):
is_think = False is_think = False
continue continue
if json_content.get('patch_op')[0].get("patch_object") == 50: if json_content.get('patch_op')[0].get("patch_object") == 50:
if json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2"):
for sug in json.loads( for sug in json.loads(
json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")): json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")):
suggestions.append(sug.get("content")) suggestions.append(sug.get("content"))
...@@ -157,6 +158,7 @@ def doubao_process_original_data(data): ...@@ -157,6 +158,7 @@ def doubao_process_original_data(data):
response_content = content_block[0].get("content").get("text_block").get("text") response_content = content_block[0].get("content").get("text_block").get("text")
suggestions = list(set(suggestions)) suggestions = list(set(suggestions))
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions,rich_media_block) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
...@@ -204,7 +206,7 @@ if __name__ == '__main__': ...@@ -204,7 +206,7 @@ if __name__ == '__main__':
# 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35' # 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35'
# ] # ]
# for task in task_id_list: # for task in task_id_list:
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = 'e5b63f95-f713-4c51-b282-78c8c1c130ad' and platform = 'DB'") data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '9bd06279-57d5-4da7-b8a7-f533fb7365f9' and platform = 'DB'")
# # # # # #
# # # # # #
......
aidso_geo/models/douyinai_data_process.py
View file @ bcb7ea21
...@@ -31,11 +31,7 @@ def douyin_ai_process_original_data(data): ...@@ -31,11 +31,7 @@ def douyin_ai_process_original_data(data):
# 提取并解析JSON数据 # 提取并解析JSON数据
data_str = item.split("data:")[1] data_str = item.split("data:")[1]
json_data = json.loads(data_str) json_data = json.loads(data_str)
print(json_data)
print('----')
print('----')
print('----')
print('----')
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
......
aidso_geo/models/eco_data_process.py
View file @ bcb7ea21
...@@ -124,6 +124,9 @@ def process_eco_product_relation( ...@@ -124,6 +124,9 @@ def process_eco_product_relation(
**relation_data, **relation_data,
} }
) )
for item in final_result:
item["isCalculateSearchWord"] = 1
bh_utils.insert_data('geo_eco_data', final_result)
return final_result return final_result
...@@ -388,10 +391,13 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -388,10 +391,13 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
if search_goods_word: if search_goods_word:
eco_result = process_eco_product_relation(eco_result, search_goods_word) eco_result = process_eco_product_relation(eco_result, search_goods_word)
for item in eco_result:
item["isCalculateSearchWord"] = None
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
if __name__ == '__main__': if __name__ == '__main__':
result = bh_utils.query_data("select * from geo_eco_data where req_id in ('794107f1e46b436f81f18aac77043f89')") result = bh_utils.query_data("select * from geo_eco_data where req_id in ('3a9edb7d-c98b-4ad2-8ae5-f53c4803ba99')")
eco_result = process_eco_product_relation(result, "小米洗地机") eco_result = process_eco_product_relation(result, "忠芝")
bh_utils.insert_data('geo_eco_data', eco_result) for item in result:
# item["isCalculateSearchWord"] = None
bh_utils.insert_data('geo_eco_data', result)
aidso_geo/models/process.py
View file @ bcb7ea21
...@@ -2018,7 +2018,7 @@ if __name__ == '__main__': ...@@ -2018,7 +2018,7 @@ if __name__ == '__main__':
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc") # data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where status = 'ING'") data_list = bh_utils.query_data("select * from geo_commit_task where pt = 20260806 and status = 'ING' and platform !='DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'") # data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ") # data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
...@@ -2030,7 +2030,7 @@ if __name__ == '__main__': ...@@ -2030,7 +2030,7 @@ if __name__ == '__main__':
i["keywords"] = safe_json_loads(i.get("keywords"), []) i["keywords"] = safe_json_loads(i.get("keywords"), [])
i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), []) i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), [])
type_t = i.get('type') type_t = i.get('type')
type_t = 'batch' type_t = 'stream_batch'
# commit_task(i,'ING') # commit_task(i,'ING')
return task_send_queue(i,type_t) return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
......
aidso_geo/models/qianwen_android_data_process.py
View file @ bcb7ea21
...@@ -394,7 +394,7 @@ def qianwen_android_process_original_data(task_data): ...@@ -394,7 +394,7 @@ def qianwen_android_process_original_data(task_data):
mu_content_model_input = mu_content.get('modelInput') mu_content_model_input = mu_content.get('modelInput')
mu_content_pois = mu_content.get('pois') mu_content_pois = mu_content.get('pois')
if mu_content_pois:
for poi in mu_content_pois: for poi in mu_content_pois:
poi_summary = poi.get('summary1') poi_summary = poi.get('summary1')
distance_formatted = poi.get('distance_formatted') distance_formatted = poi.get('distance_formatted')
...@@ -557,7 +557,7 @@ if __name__ == '__main__': ...@@ -557,7 +557,7 @@ if __name__ == '__main__':
# 04b2b061-a5c0-4429-8d3b-d2b6504f1e8b # 04b2b061-a5c0-4429-8d3b-d2b6504f1e8b
# data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'") # data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0001e01d-a8ea-405e-acef-93e4f55abbff' and platform = 'TYQWA'")
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '9b82af12-97ba-47e0-aaca-4d88b93c70e9' and platform = 'TYQWA'") f"select * from geo_commit_task where taskId = '8e38e641-38fa-4105-9a73-0031132c7c28' and platform = 'TYQWA'")
def handle_item(i): def handle_item(i):
......
aidso_geo/utils/ai_utils.py
View file @ bcb7ea21
...@@ -252,7 +252,6 @@ def ai_get_product_relation_spu(product_list, keyword): ...@@ -252,7 +252,6 @@ def ai_get_product_relation_spu(product_list, keyword):
content = json.loads(content) content = json.loads(content)
product_words = content.get("product_words", []) product_words = content.get("product_words", [])
print(product_words if isinstance(product_words, list) else [])
return product_words if isinstance(product_words, list) else [] return product_words if isinstance(product_words, list) else []
except Exception as e: except Exception as e:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment