Skip to content

A
aidso-data
Commit bcf81c1d authored by Yaowentong's avatar Yaowentong
Browse files
Options

元宝本地生活数据兼容 

豆包编码修复
电商搜索调度修复
抖音ai报错兜底重试增加
豆包积压处理
parent bcb7ea21
Hide whitespace changes
Inline Side-by-side
Showing with 279 additions and 351 deletions
aidso_geo/config/base_config.py
View file @ bcf81c1d
...@@ -277,15 +277,17 @@ if __name__ == '__main__': ...@@ -277,15 +277,17 @@ if __name__ == '__main__':
'XHSA:geo:stream_batch:list', 'XHSA:geo:stream_batch:list',
'XHSA:geo:batch:list', 'XHSA:geo:batch:list',
'geo:task_commit:list'] 'geo:task_commit:list']
# mt:snipaste_v3:only_content
init_redis4_1 = init_redis()
redis_client8 = init_redis8() init_redis4_1.delete("third_cx_user_id")
print(init_redis4_1.scard('third_cx_user_id'))
# old_count, new_count = deduplicate_redis_list( # old_count, new_count = deduplicate_redis_list(
# redis_client8, # redis_client8,
# "geo:task_commit:list", # "geo:task_commit:list",
# batch_size=500, # batch_size=500,
# ) # )
print(redis_client8.llen("geo:task_commit:list")) # print(redis_client8.llen("geo:task_commit:list"))
......
aidso_geo/core/down_load_bot.py
View file @ bcf81c1d
...@@ -1316,6 +1316,7 @@ def qian_report2(req_ids,plan_name): ...@@ -1316,6 +1316,7 @@ def qian_report2(req_ids,plan_name):
"platform": plat_form_map[q.get('platform')], "platform": plat_form_map[q.get('platform')],
"content": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/context.txt"), "content": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/context.txt"),
"quote": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/quote.txt"), "quote": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/quote.txt"),
"original": tos_utils.get_string_from_tos(f"geo/{q.get('taskId')}/{q.get('platform')}/original.text"),
"created_at": q.get('pt') "created_at": q.get('pt')
} }
result.append(r) result.append(r)
...@@ -1356,5 +1357,5 @@ def qian_report2(req_ids,plan_name): ...@@ -1356,5 +1357,5 @@ def qian_report2(req_ids,plan_name):
# 使用示例 # 使用示例
# ========================= # =========================
if __name__ == "__main__": if __name__ == "__main__":
req_list = ['e49ee7f8-7e71-47bd-9d10-10f062647697','a2695d7a-e475-40b8-b5ab-e4123f642d76','a1560a90-23a3-468c-a691-3ae66929bcfb','2078e9cd-6788-4987-b4b9-c2d3a04e8a1e','fccdcef7-91d4-41d8-a436-adcb73700130','b7f841ac-2eed-4c3e-8d53-65aa12315a49','47d57e4a-5588-4f40-8650-ef27eaf6da70','9ddc6d73-876d-4fda-b803-38aec72548e6','9dd7df07-3fec-4c76-86ec-07c2f9b456ab','3517e57f-9416-4226-9e11-6b7afb9d35e4','24cb447a-21b8-41ea-82a0-ebcd62e0f41d','f340eb27-ddaa-4e06-af19-5d99e341af0d','63fa1930-67b8-48ca-9093-c5493d281fb8','e860be96-5a76-49d9-b463-664710e07d76','765d7c2c-a790-41d4-ba7d-68bc637d433c','f2ac7a93-ccb6-403c-9f67-9baa1c7ac694','a5650d00-f5cd-4614-96c8-c5470ff8aa3a','9a21345b-c2b8-4971-909c-3cc064f37946','1e294b97-ffd2-4f2b-8a09-19712d27b2d8','15592422-4138-4edc-985f-39f277819108','b35ea5da-0e06-444a-bbb1-b767afa1cb09','435ea67b-4419-4f8f-bcaa-8d2792ebb4a6','145b7a20-1d41-43f9-add0-5a5fd2493ff2','c29be6d6-9616-4ce7-bdcb-4f072c63d57a','d9733d4a-5852-4d50-bf74-6cffb79cd252','7d0c25f9-ecd1-4e42-9d14-8174069d609e','828f5e2b-e9a6-4d71-ab42-71df9e8947e7','01956eeb-fa19-4639-9bfe-131953ad7c14','f4e0296f-607d-44a8-a116-f9f5b5acfe08','29600e8b-c197-4473-96b5-c3d631ab9abe','439fc7b4-11f9-48ad-b6a3-6c08fdb0653f','294ad0f6-817b-4ab5-b896-de9cdffc9e12','4133491f-0d41-4c09-a1f9-3d1fd3142801','167c1b01-bd2f-47ca-829a-66714070b034','38fe3c6b-779c-40b8-9632-578a2e56841e','3a2438c2-cdd5-469e-b05c-3a8c074ed914','8f1368fd-eafe-406f-8a6c-92c543e7ca80','3f287fa3-80e9-45b8-b67d-8b62ab42ba1f','7f777d9d-2ab6-4b16-9c80-273c58f4fcdd','a8172551-d28e-40fe-bb62-fdb9dfb3994c','c68d1a5a-aee2-489a-b721-2c8f0fb21751','9da2b561-89f6-41fc-bf1e-46e1c8c1d73f','a005cba6-3d6f-42d1-a41d-3a49b9c50da6','24a2c763-9c68-49e1-8f6d-07fbd3a03829','2df5e59f-45e8-4d27-b826-e83db7588e5e','4b6bcbaf-e4d9-4404-a3f3-de0880cd6acf','afe77b23-b04b-40b3-bd01-8c313d8ac8f0','bc865bab-6320-44c5-8f7d-b26802209ab7','9f0a9e74-c5c0-4d89-8d0a-9bbbdac2f1c8','04ed4c8a-92d1-4bca-a5fd-3172f46a09a6','0bdc1cf0-f734-430e-bf8c-7075ae27f0f3','6bfc60d4-5796-4cb2-91a9-ba6b85cf424c','feb0d838-303f-4cb2-8f81-a53e76c4fd2f','ce865c09-b29c-4d12-b8eb-d02cce9c9672','9601342d-aef3-4bda-ad18-bdeb0d4c52b8','72742f91-7e4e-4854-be1a-8f5895511451','51c53925-5ff0-4915-b329-1cbad89f5619','da854174-55e3-46b0-865b-1c18f894f908','5a8133f2-31e6-4735-a559-eb0cd0169248','ac6c17cc-e9f8-4c60-8590-b866140bf70a','9c7f3153-cc8a-43be-a719-fa6a320110c7','1bd034ff-dc1b-4a20-b98f-9bd29c68b68b','1331fb78-deae-4171-9cc6-1abcce00b2d4','7ca71831-6828-43cf-9019-b068b5962f33','bd169869-011d-4dcb-a5bc-a20abe473da4','a0be46ac-2d80-4b3b-909d-b194c1caf91e','98b594c1-f10c-4d6c-a4e6-be4a8bdb1964','63dae6aa-4082-419e-a344-4289ba5c9901','dd67abc4-3421-44e7-8b20-f58c7c68f067','5e6b759a-6ad9-4e98-aadb-499422f231da','75c905f3-f1a8-46a7-9d9b-cbd6cd08f42a','dfeb515a-78b2-4aed-a9d7-c3bc1a514f8d','b565eb6a-e66f-45b2-9cfd-ff557817e86a','97373e1a-b118-46a1-8e13-b29ea932806d','910fd1a7-c745-4da1-9a7d-30ef513e15b4','d8e65bc2-5825-4fa3-a120-102f7af0e039','69e735c1-8ce9-41dd-8e10-a3127af2406a','3a7a5550-aabe-41f7-ad33-d8b2a7c28948','2bdc094a-bd0e-4878-8ad1-b4640e68cfac','c5186a50-4c44-402a-861c-ce53e26a306b','6af52a28-7aaa-4352-9d88-90790878602e','0e906b34-a898-4a2b-ace5-564f2dbda170','77f0b68b-9e8b-4a9a-ac17-6a920dce9ee1','1fec9e2b-ddb5-48db-befe-e3a644067e62','fa1e2e0a-7dbf-4cef-a5bf-dbfcbf4f701e','2c8a6e4b-d4ce-4291-808f-abe028404652','484e4381-4c9f-4099-95c6-1b7ef52c6711','21460360-0e13-4a5d-9d1a-f15593c3bab9','3463d72b-9ee5-4191-bf4d-dc1731a59816','f501b913-9b61-4f44-86a1-6971a94c08a1','54bd3f65-bb0d-480c-a1d9-7c337c633c31','bd8cca4e-5bff-424f-8675-3c32ffa15066','d58a7960-2261-4ed1-a437-8748ab2f0307','caea36b7-230c-4930-986f-ada0cc28b2d2','515c9a58-2eb3-4836-ada5-f1e2b220fc26','cfed2f60-f233-4b8a-9f94-774b2785dba9','078c0464-f917-4df6-bf52-583bf9139ce3','84afcfc3-bbb1-4bfd-b3f7-4b795f6beecc','c32e234f-e2c7-41bd-b1cc-c8e7bff9daab','f0700778-3a3b-4950-9486-f2de7b5bca2f','35f1bc35-a98d-42a4-9982-0eecd24bb69b','6a9fb614-9b40-488a-979a-f94bd65e974d','042690b2-62a1-4f2d-b519-eacef8fc9ed7','dad0bf58-ead9-4866-b4b0-ec74236851d6','a923c37e-670f-45c0-b378-08bd033fd09a','3bad0631-cb94-4ca9-aeca-1e1b8066d6a9','4c11caf6-5647-4e81-af6e-c34ee6dd90d5','4ad5002e-2f8e-42e2-8889-385206b7c5d5','06543bb4-fb7b-4bdb-950b-5c303171b413','21d337d8-3b2e-43b9-8c58-b5fb327c0224','5cf52bdb-6bf7-4e2a-b2f5-d83a42109a01','32a2cad5-686d-443c-a824-af4411f6b513','496bb514-7b13-444a-b1b7-e47fa6d07323','679ea973-2df5-4e2a-b9c7-63518ce4d3f7','63d0a699-9ca6-4859-9d3e-aa01cacc62dd','a953a6ff-7812-43a2-af3a-9b3d51fe0bc2','9958488b-6751-4e2a-8d5a-7d0f39490536','e74a9f1a-109d-4249-8cc1-cb9d528aa8ba','fd1cfbd4-611e-4190-a55c-29f468d02da4','d9e55f46-dc9e-4f99-a86a-4ba86a5133c8','278156a2-457e-4d80-8d36-0be1b29e1d5d','00e65cb3-3ecb-40d2-92e9-4a15ee78c4f7','328edc10-ad06-44c4-b00b-7db6d11ccd7a','9ba47b9a-4810-48f4-8ca6-8699109b35a7','f962ee62-7fb9-4f7e-bd1c-e3b50a42e544','5ba63557-6b78-4c3e-8120-d42a2db0cbf9','ae3cfc92-53be-4de3-a457-002bff636960','1c31b295-8339-4fcc-83ae-72c2d6b192cc','a0abb2b8-54f9-4636-8caa-963c89398af5','4f117c0a-309b-4601-9d4b-e600f20e2d90','a8c46fcb-865c-4328-9709-0ca7ad444861','bc882877-4a97-40c5-95e1-5585e3c6c756','ec297cce-9aa5-4471-92fd-3a8b010f0fbe','b339eafd-483f-42c8-8369-6e633d73a7e3','c11d9b31-24ca-4822-9cf0-053477145fe0','cf974b31-ca64-47a5-b6cb-08b23148fc10','1747f918-94fb-43c0-b34f-06be3ec62199','1652fba3-4fc0-4368-8180-fd55705aa6db','5dfafcf0-7add-4483-81f9-190bd30fd8c3','8ec80c71-5169-4be1-bb01-4e36e7ad2cf0'] req_list = ['05e40120-018e-430d-9fb7-c921bde61920','09b179f5-8de3-4e10-a2f4-576e7e1af661','3716b2e7-41f3-45bb-beb2-1329e6c44670','585e35c9-2783-4706-87f5-469085349a3f','eccc4af3-2320-4fee-a415-2c6d26a6b5e8','0320b50f-ebfc-42d5-9050-01071b5205bd','75f9fc76-c27f-4573-8954-cea9e915b618','72e688fa-2d73-438e-81aa-1460290f6a82','cb18a066-8c88-4821-9d22-9e517332cf05','d8b3a651-abec-450f-8052-72e3d17a3503','b0ffffa2-5900-46ba-a049-ead25a895d96','1cc4a4a1-2923-47bb-b392-f4ffdde112c7','e352fd0e-7f70-4e88-9034-547c39267d61','440b52a2-beba-4c84-9cce-c09c7998d14c','91f485f3-1a3d-4ed5-a733-8197254f3f3a','c8990254-ba68-4790-bea9-ba40bf1c9774','24bf1e91-eb2f-4e4a-96fe-d1d91fda7889','90054909-aa64-444e-b935-d59044f9407d','0e5b86ba-ac03-4179-827f-696ff6fba15f','1ef447ef-aa86-44ec-b4fd-4cd20834edd9','4e09725d-228f-4bd7-9547-b484cbd9efdd','3a6e128e-7c77-4fdf-b0f4-0881fe6dfeee','d0fba64d-c2ab-438a-b5f2-47fb7eaffcd1','49288de5-9cf8-4cfe-8c7f-d81c8909ca35','e50d3df5-395b-4c26-a885-66088ba1c8a5','ed79787c-78be-41df-866d-777b29d0fe67','be59190a-452d-4e58-873b-a65dbfd87945','ad827a32-9c04-4b1f-b314-a4f3520f6cd9','91cb4eed-e8d9-4666-975b-6e079d279c72','19781ed3-fee9-4848-b936-0dc1517682f5','c6b962a9-6851-4238-83c6-4be6fc09d299','38f0eb5b-ed46-4ab9-8d57-3b6f5ebee2a3','55769d35-310f-4d81-8685-dbecd928e14a','cdc316ad-c8bb-407b-8587-e8f01deb306c','821a68df-7f6c-491d-9dc9-5b550e307aaf','dc1ee718-6e8c-4a99-8bda-aaa518523705','0ea1f698-d9fb-4085-9334-f72226566a70']
qian_report2(req_list,'海尔麦浪冰箱') qian_report2(req_list,'养生酒')
\ No newline at end of file \ No newline at end of file
aidso_geo/core/feishu_snipaste_v3.py
View file @ bcf81c1d
...@@ -38,8 +38,6 @@ WIKI_TOKEN = "RndAwboqZiuCAFkTGkScaWHPnmg" ...@@ -38,8 +38,6 @@ WIKI_TOKEN = "RndAwboqZiuCAFkTGkScaWHPnmg"
TABLE_NAME = "douchacha_data.geo_feishu_snipaste_v3" TABLE_NAME = "douchacha_data.geo_feishu_snipaste_v3"
LOG_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_log_v3" LOG_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_log_v3"
RESULT_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_result_v3" RESULT_TABLE_NAME = "douchacha_data.geo_feishu_snipaste_result_v3"
ROUND_COUNT = 30
MAX_ROUND_COUNT = 30
REDIS_QUEUE_KEY = "mt:snipaste_v3:only_content" REDIS_QUEUE_KEY = "mt:snipaste_v3:only_content"
REDIS_QUEUE_KEY_SHARE = "mt:snipaste_v3:with_share" REDIS_QUEUE_KEY_SHARE = "mt:snipaste_v3:with_share"
SYNC_HOUR = 0 SYNC_HOUR = 0
...@@ -51,6 +49,8 @@ PLATFORM_CONFIGS = { ...@@ -51,6 +49,8 @@ PLATFORM_CONFIGS = {
"tos_directory": "doubao", "tos_directory": "doubao",
"content_redis_key": REDIS_QUEUE_KEY, "content_redis_key": REDIS_QUEUE_KEY,
"share_redis_key": REDIS_QUEUE_KEY_SHARE, "share_redis_key": REDIS_QUEUE_KEY_SHARE,
"round_count": 10,
"hit_threshold": 8,
}, },
"DP": { "DP": {
"name": "DeepSeek网页版", "name": "DeepSeek网页版",
...@@ -58,6 +58,8 @@ PLATFORM_CONFIGS = { ...@@ -58,6 +58,8 @@ PLATFORM_CONFIGS = {
"tos_directory": "DP", "tos_directory": "DP",
"content_redis_key": "mt:DP:snipaste_v3:only_content", "content_redis_key": "mt:DP:snipaste_v3:only_content",
"share_redis_key": "mt:DP:snipaste_v3:with_share", "share_redis_key": "mt:DP:snipaste_v3:with_share",
"round_count": 10,
"hit_threshold": 8,
}, },
} }
THIRTY_ROUND_EXPORT_WORKERS = 20 THIRTY_ROUND_EXPORT_WORKERS = 20
...@@ -87,6 +89,22 @@ def normalize_platform(platform: str = "DB") -> str: ...@@ -87,6 +89,22 @@ def normalize_platform(platform: str = "DB") -> str:
return normalized_platform return normalized_platform
def get_platform_round_strategy(platform: str) -> tuple[int, int]:
"""返回平台的总轮次和达标轮次阈值。"""
platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform]
round_count = int(platform_config["round_count"])
hit_threshold = int(platform_config["hit_threshold"])
if round_count <= 0:
raise ValueError(f"平台 {platform} 的 round_count 必须大于 0")
if not 1 <= hit_threshold <= round_count:
raise ValueError(
f"平台 {platform} 的 hit_threshold 必须满足 "
f"1 <= hit_threshold <= round_count"
)
return round_count, hit_threshold
def normalize_pt(pt=None) -> str: def normalize_pt(pt=None) -> str:
""" """
规范化运行批次。 规范化运行批次。
...@@ -379,6 +397,7 @@ def create_round_logs( ...@@ -379,6 +397,7 @@ def create_round_logs(
platform: str = "DB", platform: str = "DB",
) -> int: ) -> int:
platform = normalize_platform(platform) platform = normalize_platform(platform)
round_count, _ = get_platform_round_strategy(platform)
existing_rows = bh_utils.query_data( existing_rows = bh_utils.query_data(
( (
f"SELECT taskId, `count` FROM {LOG_TABLE_NAME} " f"SELECT taskId, `count` FROM {LOG_TABLE_NAME} "
...@@ -397,7 +416,7 @@ def create_round_logs( ...@@ -397,7 +416,7 @@ def create_round_logs(
log_items = [] log_items = []
for task in tasks: for task in tasks:
task_id = str(task["taskId"]) task_id = str(task["taskId"])
for round_number in range(1, ROUND_COUNT + 1): for round_number in range(1, round_count + 1):
count = str(round_number) count = str(round_number)
if (task_id, count) in existing_keys: if (task_id, count) in existing_keys:
continue continue
...@@ -415,8 +434,9 @@ def create_round_logs( ...@@ -415,8 +434,9 @@ def create_round_logs(
if not log_items: if not log_items:
logger.success( logger.success(
f"[第一阶段 pt={pt} count=1-30] " f"[第一阶段 platform={platform} pt={pt} "
f"当天 30 轮日志已全部存在" f"count=1-{round_count}] "
f"当天 {round_count} 轮日志已全部存在"
) )
return 0 return 0
if not bh_utils.insert_data(LOG_TABLE_NAME, log_items): if not bh_utils.insert_data(LOG_TABLE_NAME, log_items):
...@@ -433,6 +453,7 @@ def push_rounds_to_redis( ...@@ -433,6 +453,7 @@ def push_rounds_to_redis(
platform: str = "DB", platform: str = "DB",
) -> int: ) -> int:
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_round_count, _ = get_platform_round_strategy(platform)
is_share_round = ( is_share_round = (
start_count == end_count start_count == end_count
and start_count in (0, 99) and start_count in (0, 99)
...@@ -440,14 +461,14 @@ def push_rounds_to_redis( ...@@ -440,14 +461,14 @@ def push_rounds_to_redis(
if ( if (
start_count < 0 start_count < 0
or ( or (
end_count > MAX_ROUND_COUNT end_count > platform_round_count
and not is_share_round and not is_share_round
) )
or start_count > end_count or start_count > end_count
): ):
raise ValueError( raise ValueError(
f"轮次范围必须满足 0 <= start_count <= end_count " f"轮次范围必须满足 0 <= start_count <= end_count "
f"<= {MAX_ROUND_COUNT}，或使用分享轮次 0、99" f"<= {platform_round_count}，或使用分享轮次 0、99"
) )
if not redis_key or not redis_key.strip(): if not redis_key or not redis_key.strip():
raise ValueError("redis_key 不能为空") raise ValueError("redis_key 不能为空")
...@@ -538,6 +559,7 @@ def sync_feishu_to_database( ...@@ -538,6 +559,7 @@ def sync_feishu_to_database(
validate_config() validate_config()
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform] platform_config = PLATFORM_CONFIGS[platform]
round_count, _ = get_platform_round_strategy(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
tasks = get_tasks(pt, platform=platform) tasks = get_tasks(pt, platform=platform)
title = "" title = ""
...@@ -564,7 +586,8 @@ def sync_feishu_to_database( ...@@ -564,7 +586,8 @@ def sync_feishu_to_database(
raise RuntimeError(f"写入任务表失败: {TABLE_NAME}") raise RuntimeError(f"写入任务表失败: {TABLE_NAME}")
else: else:
logger.success( logger.success(
f"[第一阶段 pt={pt} count=1-30] " f"[第一阶段 platform={platform} pt={pt} "
f"count=1-{round_count}] "
f"当前批次主任务已存在，直接补充轮次日志: " f"当前批次主任务已存在，直接补充轮次日志: "
f"rows={len(tasks)}" f"rows={len(tasks)}"
) )
...@@ -576,7 +599,8 @@ def sync_feishu_to_database( ...@@ -576,7 +599,8 @@ def sync_feishu_to_database(
) )
logger.success( logger.success(
f"[第一阶段 platform={platform} pt={pt} count=1-30] " f"[第一阶段 platform={platform} pt={pt} "
f"count=1-{round_count}] "
f"飞书数据库同步完成: title={title}, " f"飞书数据库同步完成: title={title}, "
f"tasks={len(tasks)}, inserted_logs={inserted_log_count}" f"tasks={len(tasks)}, inserted_logs={inserted_log_count}"
) )
...@@ -1580,15 +1604,22 @@ def create_and_process_share_rounds( ...@@ -1580,15 +1604,22 @@ def create_and_process_share_rounds(
pt: str, pt: str,
redis_key: str, redis_key: str,
hit_start_count: int = 1, hit_start_count: int = 1,
hit_end_count: int = 30, hit_end_count: int = None,
hit_threshold: int = 24, hit_threshold: int = None,
platform: str = "DB", platform: str = "DB",
) -> int: ) -> int:
""" """
第 1-30 轮中关键词命中轮数达到阈值的 taskId， 平台配置轮次中关键词命中轮数达到阈值的 taskId，
生成 count=0 的分享任务并完成 Redis、TOS 和结果入库流程。 生成 count=0 的分享任务并完成 Redis、TOS 和结果入库流程。
""" """
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_round_count, platform_hit_threshold = (
get_platform_round_strategy(platform)
)
if hit_end_count is None:
hit_end_count = platform_round_count
if hit_threshold is None:
hit_threshold = platform_hit_threshold
qualified_rows = bh_utils.query_data( qualified_rows = bh_utils.query_data(
f""" f"""
SELECT taskId SELECT taskId
...@@ -1709,15 +1740,22 @@ def create_and_process_unqualified_share_rounds( ...@@ -1709,15 +1740,22 @@ def create_and_process_unqualified_share_rounds(
pt: str, pt: str,
redis_key: str, redis_key: str,
hit_start_count: int = 1, hit_start_count: int = 1,
hit_end_count: int = 30, hit_end_count: int = None,
hit_threshold: int = 24, hit_threshold: int = None,
platform: str = "DB", platform: str = "DB",
) -> int: ) -> int:
""" """
第 1-30 轮中关键词命中轮数不足阈值的 taskId， 平台配置轮次中关键词命中轮数不足阈值的 taskId，
生成 count=99 的分享任务并完成 Redis、TOS 和结果入库流程。 生成 count=99 的分享任务并完成 Redis、TOS 和结果入库流程。
""" """
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_round_count, platform_hit_threshold = (
get_platform_round_strategy(platform)
)
if hit_end_count is None:
hit_end_count = platform_round_count
if hit_threshold is None:
hit_threshold = platform_hit_threshold
qualified_rows = bh_utils.query_data( qualified_rows = bh_utils.query_data(
f""" f"""
SELECT taskId SELECT taskId
...@@ -1844,15 +1882,16 @@ def export_share_rounds_to_excel( ...@@ -1844,15 +1882,16 @@ def export_share_rounds_to_excel(
raise ValueError("Excel 导出仅支持分享轮次 count=0 或 count=99") raise ValueError("Excel 导出仅支持分享轮次 count=0 或 count=99")
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform] platform_config = PLATFORM_CONFIGS[platform]
round_count_limit, hit_threshold = get_platform_round_strategy(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
tos_pt = pt[:8] tos_pt = pt[:8]
tos_directory = get_platform_tos_directory(platform) tos_directory = get_platform_tos_directory(platform)
stage_name = "第四阶段" if share_count == 0 else "第六阶段" stage_name = "第四阶段" if share_count == 0 else "第六阶段"
file_label = "达标数据" if share_count == 0 else "未达标" file_label = "达标数据" if share_count == 0 else "未达标"
qualification_filter_sql = ( qualification_filter_sql = (
"AND ifNull(a3.hit_round_count, 0) >= 24" f"AND ifNull(a3.hit_round_count, 0) >= {hit_threshold}"
if share_count == 0 if share_count == 0
else "AND ifNull(a3.hit_round_count, 0) < 24" else f"AND ifNull(a3.hit_round_count, 0) < {hit_threshold}"
) )
rows = bh_utils.query_data( rows = bh_utils.query_data(
...@@ -1887,7 +1926,7 @@ def export_share_rounds_to_excel( ...@@ -1887,7 +1926,7 @@ def export_share_rounds_to_excel(
FROM {RESULT_TABLE_NAME} FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND %s
GROUP BY taskId, `count` GROUP BY taskId, `count`
) AS round_result ) AS round_result
GROUP BY taskId GROUP BY taskId
...@@ -1906,6 +1945,7 @@ def export_share_rounds_to_excel( ...@@ -1906,6 +1945,7 @@ def export_share_rounds_to_excel(
( (
int(pt), int(pt),
platform, platform,
round_count_limit,
pt, pt,
platform, platform,
share_count, share_count,
...@@ -1937,13 +1977,13 @@ def export_share_rounds_to_excel( ...@@ -1937,13 +1977,13 @@ def export_share_rounds_to_excel(
FROM {RESULT_TABLE_NAME} FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND %s
""", """,
(int(pt), platform), (int(pt), platform, round_count_limit),
) )
if round_keyword_rows is None: if round_keyword_rows is None:
raise RuntimeError( raise RuntimeError(
f"{stage_name}查询 1-30 轮关键词次数失败" f"{stage_name}查询 1-{round_count_limit} 轮关键词次数失败"
) )
all_keywords = [] all_keywords = []
...@@ -1973,7 +2013,7 @@ def export_share_rounds_to_excel( ...@@ -1973,7 +2013,7 @@ def export_share_rounds_to_excel(
for round_row in round_keyword_rows: for round_row in round_keyword_rows:
task_id = str(round_row.get("taskId") or "") task_id = str(round_row.get("taskId") or "")
count = int(round_row.get("count") or 0) count = int(round_row.get("count") or 0)
if not task_id or not 1 <= count <= 30: if not task_id or not 1 <= count <= round_count_limit:
continue continue
task_round_key = (task_id, count) task_round_key = (task_id, count)
...@@ -2089,14 +2129,14 @@ def export_share_rounds_to_excel( ...@@ -2089,14 +2129,14 @@ def export_share_rounds_to_excel(
processed_round_count = row.get("processed_round_count") processed_round_count = row.get("processed_round_count")
hit_round_count = row.get("hit_round_count") hit_round_count = row.get("hit_round_count")
hit_round_count_value = int(hit_round_count or 0) hit_round_count_value = int(hit_round_count or 0)
hit_rate = hit_round_count_value / 30 hit_rate = hit_round_count_value / round_count_limit
recommendation_rate = ( recommendation_rate = (
len(recommended_rounds_by_task.get(task_id, set())) len(recommended_rounds_by_task.get(task_id, set()))
/ 30 / round_count_limit
) )
if not processed_round_count: if not processed_round_count:
qualified_text = "没处理" qualified_text = "没处理"
elif hit_round_count_value >= 24: elif hit_round_count_value >= hit_threshold:
qualified_text = "是" qualified_text = "是"
else: else:
qualified_text = "否" qualified_text = "否"
...@@ -2340,7 +2380,7 @@ def _safe_get_tos_text(path: str) -> str: ...@@ -2340,7 +2380,7 @@ def _safe_get_tos_text(path: str) -> str:
return tos_utils.get_string_from_tos(path) or "" return tos_utils.get_string_from_tos(path) or ""
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
f"30轮明细读取 TOS 失败: path={path}, error={exc}" f"轮次明细读取 TOS 失败: path={path}, error={exc}"
) )
return "" return ""
...@@ -2459,6 +2499,7 @@ def _style_thirty_round_worksheet( ...@@ -2459,6 +2499,7 @@ def _style_thirty_round_worksheet(
def _upload_thirty_round_excel_to_tos( def _upload_thirty_round_excel_to_tos(
file_path: str, file_path: str,
object_key: str, object_key: str,
round_count: int,
retry_times: int = 3, retry_times: int = 3,
) -> str: ) -> str:
client = tos_client.get_client() client = tos_client.get_client()
...@@ -2479,7 +2520,8 @@ def _upload_thirty_round_excel_to_tos( ...@@ -2479,7 +2520,8 @@ def _upload_thirty_round_excel_to_tos(
), ),
) )
logger.success( logger.success(
f"[第七阶段 count=1-30] 30轮明细已上传 TOS: " f"[第七阶段 count=1-{round_count}] "
f"{round_count}轮明细已上传 TOS: "
f"tos://{bucket}/{object_key}, " f"tos://{bucket}/{object_key}, "
f"attempt={attempt}/{retry_times}" f"attempt={attempt}/{retry_times}"
) )
...@@ -2487,13 +2529,14 @@ def _upload_thirty_round_excel_to_tos( ...@@ -2487,13 +2529,14 @@ def _upload_thirty_round_excel_to_tos(
except Exception as exc: except Exception as exc:
last_error = exc last_error = exc
logger.warning( logger.warning(
f"[第七阶段 count=1-30] 30轮明细上传失败: " f"[第七阶段 count=1-{round_count}] "
f"{round_count}轮明细上传失败: "
f"object_key={object_key}, " f"object_key={object_key}, "
f"attempt={attempt}/{retry_times}, error={exc}" f"attempt={attempt}/{retry_times}, error={exc}"
) )
raise RuntimeError( raise RuntimeError(
f"30轮明细上传 TOS 连续失败 {retry_times} 次: " f"{round_count}轮明细上传 TOS 连续失败 {retry_times} 次: "
f"object_key={object_key}" f"object_key={object_key}"
) from last_error ) from last_error
...@@ -2503,7 +2546,7 @@ def export_thirty_round_details( ...@@ -2503,7 +2546,7 @@ def export_thirty_round_details(
platform: str = "DB", platform: str = "DB",
) -> list[dict]: ) -> list[dict]:
""" """
按业务线导出当前 pt、当前平台的 1-30 轮明细并上传到 TOS。 按业务线导出当前 pt、当前平台配置轮次的明细并上传到 TOS。
返回可直接写入飞书日报的业务线及 CDN 下载地址列表。 返回可直接写入飞书日报的业务线及 CDN 下载地址列表。
""" """
...@@ -2513,6 +2556,7 @@ def export_thirty_round_details( ...@@ -2513,6 +2556,7 @@ def export_thirty_round_details(
platform = normalize_platform(platform) platform = normalize_platform(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
platform_config = PLATFORM_CONFIGS[platform] platform_config = PLATFORM_CONFIGS[platform]
round_count_limit, hit_threshold = get_platform_round_strategy(platform)
tos_directory = get_platform_tos_directory(platform) tos_directory = get_platform_tos_directory(platform)
log_rows = bh_utils.query_data( log_rows = bh_utils.query_data(
...@@ -2536,12 +2580,14 @@ def export_thirty_round_details( ...@@ -2536,12 +2580,14 @@ def export_thirty_round_details(
ON a1.reqId = a3.reqId ON a1.reqId = a3.reqId
WHERE toString(a1.pt) = %s WHERE toString(a1.pt) = %s
AND ifNull(a1.platform, 'DB') = %s AND ifNull(a1.platform, 'DB') = %s
AND toInt32(a1.`count`) BETWEEN 1 AND 30 AND toInt32(a1.`count`) BETWEEN 1 AND %s
""", """,
(pt, platform), (pt, platform, round_count_limit),
) )
if log_rows is None: if log_rows is None:
raise RuntimeError("第七阶段查询 1-30 轮日志数据失败") raise RuntimeError(
f"第七阶段查询 1-{round_count_limit} 轮日志数据失败"
)
result_rows = bh_utils.query_data( result_rows = bh_utils.query_data(
f""" f"""
...@@ -2554,12 +2600,14 @@ def export_thirty_round_details( ...@@ -2554,12 +2600,14 @@ def export_thirty_round_details(
FROM {RESULT_TABLE_NAME} FROM {RESULT_TABLE_NAME}
WHERE toString(pt) = %s WHERE toString(pt) = %s
AND ifNull(platform, 'DB') = %s AND ifNull(platform, 'DB') = %s
AND toInt32(`count`) BETWEEN 1 AND 30 AND toInt32(`count`) BETWEEN 1 AND %s
""", """,
(pt, platform), (pt, platform, round_count_limit),
) )
if result_rows is None: if result_rows is None:
raise RuntimeError("第七阶段查询 1-30 轮结果数据失败") raise RuntimeError(
f"第七阶段查询 1-{round_count_limit} 轮结果数据失败"
)
result_by_req_id = {} result_by_req_id = {}
task_round_hits = defaultdict(dict) task_round_hits = defaultdict(dict)
...@@ -2568,7 +2616,7 @@ def export_thirty_round_details( ...@@ -2568,7 +2616,7 @@ def export_thirty_round_details(
task_id = str(result_row.get("taskId") or "") task_id = str(result_row.get("taskId") or "")
result_pt = str(result_row.get("pt") or "") result_pt = str(result_row.get("pt") or "")
try: try:
round_count = int( result_round_count = int(
result_row.get("round_count") or 0 result_row.get("round_count") or 0
) )
has_keyword = int( has_keyword = int(
...@@ -2579,7 +2627,7 @@ def export_thirty_round_details( ...@@ -2579,7 +2627,7 @@ def export_thirty_round_details(
if ( if (
not req_id not req_id
or not task_id or not task_id
or not 1 <= round_count <= 30 or not 1 <= result_round_count <= round_count_limit
): ):
continue continue
...@@ -2594,8 +2642,8 @@ def export_thirty_round_details( ...@@ -2594,8 +2642,8 @@ def export_thirty_round_details(
has_keyword, has_keyword,
) )
task_key = (result_pt, task_id) task_key = (result_pt, task_id)
task_round_hits[task_key][round_count] = max( task_round_hits[task_key][result_round_count] = max(
task_round_hits[task_key].get(round_count, 0), task_round_hits[task_key].get(result_round_count, 0),
has_keyword, has_keyword,
) )
...@@ -2613,18 +2661,18 @@ def export_thirty_round_details( ...@@ -2613,18 +2661,18 @@ def export_thirty_round_details(
rows_by_round = defaultdict(list) rows_by_round = defaultdict(list)
for log_row in business_line_rows: for log_row in business_line_rows:
try: try:
round_count = int( log_round_count = int(
log_row.get("round_count") or 0 log_row.get("round_count") or 0
) )
except (TypeError, ValueError): except (TypeError, ValueError):
continue continue
if 1 <= round_count <= 30: if 1 <= log_round_count <= round_count_limit:
rows_by_round[round_count].append(log_row) rows_by_round[log_round_count].append(log_row)
safe_business_line = _safe_file_name_part(business_line) safe_business_line = _safe_file_name_part(business_line)
file_name = ( file_name = (
f"{pt}_{platform_config['name']}_" f"{pt}_{platform_config['name']}_"
f"{safe_business_line}_30轮明细.xlsx" f"{safe_business_line}_{round_count_limit}轮明细.xlsx"
) )
local_path = os.path.join( local_path = os.path.join(
tempfile.gettempdir(), tempfile.gettempdir(),
...@@ -2633,7 +2681,7 @@ def export_thirty_round_details( ...@@ -2633,7 +2681,7 @@ def export_thirty_round_details(
headers = [ headers = [
"问题", "问题",
"批次", "批次",
"30次是否达标", f"{round_count_limit}次是否达标",
"本次是否达标", "本次是否达标",
"平台", "平台",
"回答", "回答",
...@@ -2662,7 +2710,7 @@ def export_thirty_round_details( ...@@ -2662,7 +2710,7 @@ def export_thirty_round_details(
if value == 1 if value == 1
) )
thirty_round_qualified = ( thirty_round_qualified = (
"是" if hit_round_count >= 24 else "否" "是" if hit_round_count >= hit_threshold else "否"
) )
if result_item is None: if result_item is None:
...@@ -2720,18 +2768,18 @@ def export_thirty_round_details( ...@@ -2720,18 +2768,18 @@ def export_thirty_round_details(
with ThreadPoolExecutor( with ThreadPoolExecutor(
max_workers=THIRTY_ROUND_EXPORT_WORKERS max_workers=THIRTY_ROUND_EXPORT_WORKERS
) as executor: ) as executor:
for round_count in range(1, 31): for current_round in range(1, round_count_limit + 1):
worksheet = workbook.create_sheet( worksheet = workbook.create_sheet(
title=( title=(
f"第" f"第"
f"{_chinese_round_number(round_count)}轮" f"{_chinese_round_number(current_round)}轮"
) )
) )
worksheet.append(headers) worksheet.append(headers)
round_rows = [ round_rows = [
row row
for row in sorted( for row in sorted(
rows_by_round.get(round_count, []), rows_by_round.get(current_round, []),
key=_sheet_order_sort_key, key=_sheet_order_sort_key,
) )
if row.get("reqId") if row.get("reqId")
...@@ -2750,7 +2798,8 @@ def export_thirty_round_details( ...@@ -2750,7 +2798,8 @@ def export_thirty_round_details(
logger.info( logger.info(
f"[第七阶段 platform={platform} pt={pt} " f"[第七阶段 platform={platform} pt={pt} "
f"business_line={business_line} " f"business_line={business_line} "
f"count={round_count}] 30轮明细 Sheet 完成: " f"count={current_round}] "
f"{round_count_limit}轮明细 Sheet 完成: "
f"rows={len(round_rows)}" f"rows={len(round_rows)}"
) )
...@@ -2764,6 +2813,7 @@ def export_thirty_round_details( ...@@ -2764,6 +2813,7 @@ def export_thirty_round_details(
_upload_thirty_round_excel_to_tos( _upload_thirty_round_excel_to_tos(
file_path=local_path, file_path=local_path,
object_key=object_key, object_key=object_key,
round_count=round_count_limit,
) )
finally: finally:
try: try:
...@@ -2784,15 +2834,17 @@ def export_thirty_round_details( ...@@ -2784,15 +2834,17 @@ def export_thirty_round_details(
) )
logger.success( logger.success(
f"[第七阶段 platform={platform} pt={pt} " f"[第七阶段 platform={platform} pt={pt} "
f"business_line={business_line} count=1-30] " f"business_line={business_line} "
f"30轮明细导出完成: rows={total_rows}, " f"count=1-{round_count_limit}] "
f"{round_count_limit}轮明细导出完成: rows={total_rows}, "
f"url={cdn_url}" f"url={cdn_url}"
) )
if not downloads: if not downloads:
logger.warning( logger.warning(
f"[第七阶段 platform={platform} pt={pt} count=1-30] " f"[第七阶段 platform={platform} pt={pt} "
f"没有可导出的30轮业务线数据" f"count=1-{round_count_limit}] "
f"没有可导出的{round_count_limit}轮业务线数据"
) )
return downloads return downloads
...@@ -2807,6 +2859,7 @@ def build_daily_run_report( ...@@ -2807,6 +2859,7 @@ def build_daily_run_report(
""" """
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_name = PLATFORM_CONFIGS[platform]["name"] platform_name = PLATFORM_CONFIGS[platform]["name"]
round_count, hit_threshold = get_platform_round_strategy(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
tasks = bh_utils.query_data( tasks = bh_utils.query_data(
f""" f"""
...@@ -2826,9 +2879,9 @@ def build_daily_run_report( ...@@ -2826,9 +2879,9 @@ def build_daily_run_report(
FROM {RESULT_TABLE_NAME} FROM {RESULT_TABLE_NAME}
WHERE pt = %s WHERE pt = %s
AND ifNull(platform, 'DB') = %s AND ifNull(platform, 'DB') = %s
AND `count` BETWEEN 1 AND 30 AND `count` BETWEEN 1 AND %s
""", """,
(int(pt), platform), (int(pt), platform, round_count),
) )
if result_rows is None: if result_rows is None:
raise RuntimeError("最后阶段查询关键词统计结果失败") raise RuntimeError("最后阶段查询关键词统计结果失败")
...@@ -2859,9 +2912,9 @@ def build_daily_run_report( ...@@ -2859,9 +2912,9 @@ def build_daily_run_report(
{ {
count count
for count in hit_rounds for count in hit_rounds
if 1 <= count <= 30 if 1 <= count <= round_count
} }
) >= 24 ) >= hit_threshold
} }
thirty_round_failed = set(task_map) - thirty_round_qualified thirty_round_failed = set(task_map) - thirty_round_qualified
...@@ -2871,8 +2924,9 @@ def build_daily_run_report( ...@@ -2871,8 +2924,9 @@ def build_daily_run_report(
f"**统计日期：** {format_pt_display(pt)}", f"**统计日期：** {format_pt_display(pt)}",
"", "",
( (
"**统计口径：** 同一问题按第 1–30 轮去重统计" f"**统计口径：** 同一问题按第 1–{round_count} 轮"
"关键词命中轮次数，命中次数达到 24 次视为达标。" f"去重统计关键词命中轮次数，命中次数达到 "
f"{hit_threshold} 次视为达标。"
), ),
"", "",
f"- 今日问题任务总数：{len(task_map)} 个", f"- 今日问题任务总数：{len(task_map)} 个",
...@@ -2883,7 +2937,7 @@ def build_daily_run_report( ...@@ -2883,7 +2937,7 @@ def build_daily_run_report(
markdown_lines.extend( markdown_lines.extend(
[ [
"", "",
"**30轮数据如下：**", f"**{round_count}轮数据如下：**",
"", "",
] ]
) )
...@@ -2896,7 +2950,7 @@ def build_daily_run_report( ...@@ -2896,7 +2950,7 @@ def build_daily_run_report(
continue continue
markdown_lines.append( markdown_lines.append(
f"- {business_line}：" f"- {business_line}："
f"[点击下载30轮明细]({download_url})" f"[点击下载{round_count}轮明细]({download_url})"
) )
markdown_lines.extend( markdown_lines.extend(
[ [
...@@ -2912,7 +2966,7 @@ def build_daily_run_report( ...@@ -2912,7 +2966,7 @@ def build_daily_run_report(
{ {
count count
for count in hit_rounds_by_task.get(task_id, set()) for count in hit_rounds_by_task.get(task_id, set())
if 1 <= count <= 30 if 1 <= count <= round_count
} }
) )
try: try:
...@@ -3010,7 +3064,7 @@ def send_markdown_report_to_feishu_chat( ...@@ -3010,7 +3064,7 @@ def send_markdown_report_to_feishu_chat(
f"飞书发送运行报告未返回 message_id: {data}" f"飞书发送运行报告未返回 message_id: {data}"
) )
logger.success( logger.success(
f"[第七阶段 count=1-30] 运行报告已发送: " f"[第七阶段 platform={platform}] 运行报告已发送: "
f"chat_id={chat_id}, message_id={message_id}" f"chat_id={chat_id}, message_id={message_id}"
) )
return message_id return message_id
...@@ -3025,19 +3079,20 @@ def run_daily_pipeline( ...@@ -3025,19 +3079,20 @@ def run_daily_pipeline(
""" """
platform = normalize_platform(platform) platform = normalize_platform(platform)
platform_config = PLATFORM_CONFIGS[platform] platform_config = PLATFORM_CONFIGS[platform]
round_count, hit_threshold = get_platform_round_strategy(platform)
pt = normalize_pt(pt) pt = normalize_pt(pt)
start_count = 1 start_count = 1
end_count = 30 end_count = round_count
logger.success( logger.success(
f"[每日任务 platform={platform} pt={pt} count=1-99] " f"[每日任务 platform={platform} pt={pt} count=1-99] "
f"七阶段任务开始执行" f"七阶段任务开始执行"
) )
# 第一阶段：飞书同步到主表和 30 轮日志表。 # 第一阶段：飞书同步到主表和平台配置轮次日志表。
sync_feishu_to_database(pt=pt, platform=platform) sync_feishu_to_database(pt=pt, platform=platform)
# 第二阶段：所有任务发送 1-30 轮，完成三轮补发、解析和结果入库。 # 第二阶段：发送平台配置轮次，完成补发、解析和结果入库。
process_rounds_to_result( process_rounds_to_result(
pt=pt, pt=pt,
start_count=start_count, start_count=start_count,
...@@ -3046,13 +3101,13 @@ def run_daily_pipeline( ...@@ -3046,13 +3101,13 @@ def run_daily_pipeline(
platform=platform, platform=platform,
) )
# 第三阶段：1-30 轮命中不少于 24 轮的任务生成 count=0 分享任务。 # 第三阶段：达到平台命中阈值的任务生成 count=0 分享任务。
create_and_process_share_rounds( create_and_process_share_rounds(
pt=pt, pt=pt,
redis_key=platform_config["share_redis_key"], redis_key=platform_config["share_redis_key"],
hit_start_count=1, hit_start_count=1,
hit_end_count=30, hit_end_count=round_count,
hit_threshold=24, hit_threshold=hit_threshold,
platform=platform, platform=platform,
) )
...@@ -3068,13 +3123,13 @@ def run_daily_pipeline( ...@@ -3068,13 +3123,13 @@ def run_daily_pipeline(
count_label="0", count_label="0",
) )
# 第五阶段：1-30 轮命中不足 24 轮的任务生成 count=99 分享任务。 # 第五阶段：未达到平台命中阈值的任务生成 count=99 分享任务。
create_and_process_unqualified_share_rounds( create_and_process_unqualified_share_rounds(
pt=pt, pt=pt,
redis_key=platform_config["share_redis_key"], redis_key=platform_config["share_redis_key"],
hit_start_count=1, hit_start_count=1,
hit_end_count=30, hit_end_count=round_count,
hit_threshold=24, hit_threshold=hit_threshold,
platform=platform, platform=platform,
) )
...@@ -3092,7 +3147,7 @@ def run_daily_pipeline( ...@@ -3092,7 +3147,7 @@ def run_daily_pipeline(
count_label="99", count_label="99",
) )
# 第七阶段：导出30轮明细，并在今日运行总结中发送下载链接。 # 第七阶段：导出平台配置轮次明细，并在日报中发送下载链接。
thirty_round_downloads = export_thirty_round_details( thirty_round_downloads = export_thirty_round_details(
pt=pt, pt=pt,
platform=platform, platform=platform,
...@@ -3128,9 +3183,4 @@ def run_daily_pipeline_safely( ...@@ -3128,9 +3183,4 @@ def run_daily_pipeline_safely(
if __name__ == "__main__": if __name__ == "__main__":
# pt = '20260803'
# platform_db = 'DB'
# platform_dp = 'DP'
# run_daily_pipeline(pt=pt, platform=platform_db)
# run_daily_pipeline(pt=pt, platform=platform_dp)
start_scheduler() start_scheduler()
aidso_geo/models/douyinai_data_process.py
View file @ bcf81c1d
...@@ -91,6 +91,7 @@ def douyin_ai_process_original_data(data): ...@@ -91,6 +91,7 @@ def douyin_ai_process_original_data(data):
suggestions_list = json.loads(business_data[0].get('data').get('raw_data')).get('display').get('query_list') suggestions_list = json.loads(business_data[0].get('data').get('raw_data')).get('display').get('query_list')
if suggestions_list: if suggestions_list:
suggestions = [item['content'] for item in suggestions_list] suggestions = [item['content'] for item in suggestions_list]
response_content = response_content.replace("[]()[]()", "")
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions,rich_media_block) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
......
aidso_geo/models/eco_data_process.py
View file @ bcf81c1d
...@@ -40,6 +40,7 @@ def process_eco_product_relation( ...@@ -40,6 +40,7 @@ def process_eco_product_relation(
keyword, keyword,
) )
) )
print(ai_result)
except Exception as exc: except Exception as exc:
print( print(
f"AI商品识别失败: " f"AI商品识别失败: "
...@@ -104,6 +105,7 @@ def process_eco_product_relation( ...@@ -104,6 +105,7 @@ def process_eco_product_relation(
) )
except (TypeError, ValueError): except (TypeError, ValueError):
current = 0 current = 0
if brand == spu_name: if brand == spu_name:
spu_name = brand spu_name = brand
else: else:
...@@ -124,6 +126,7 @@ def process_eco_product_relation( ...@@ -124,6 +126,7 @@ def process_eco_product_relation(
**relation_data, **relation_data,
} }
) )
print(final_result)
for item in final_result: for item in final_result:
item["isCalculateSearchWord"] = 1 item["isCalculateSearchWord"] = 1
bh_utils.insert_data('geo_eco_data', final_result) bh_utils.insert_data('geo_eco_data', final_result)
...@@ -393,11 +396,12 @@ def save_eco_data_to_bh(data,eco_type,eco_list): ...@@ -393,11 +396,12 @@ def save_eco_data_to_bh(data,eco_type,eco_list):
eco_result = process_eco_product_relation(eco_result, search_goods_word) eco_result = process_eco_product_relation(eco_result, search_goods_word)
for item in eco_result: for item in eco_result:
item["isCalculateSearchWord"] = None item["isCalculateSearchWord"] = None
# eco_result = process_eco_product_relation(eco_result, search_goods_word)
bh_utils.insert_data('geo_eco_data',eco_result) bh_utils.insert_data('geo_eco_data',eco_result)
if __name__ == '__main__': if __name__ == '__main__':
result = bh_utils.query_data("select * from geo_eco_data where req_id in ('3a9edb7d-c98b-4ad2-8ae5-f53c4803ba99')") result = bh_utils.query_data("select * from geo_eco_data where req_id in ('0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','0e2eacab462342e2864a85c625a9f275','10640b76d87747eaa6230d6b9f594e85','10640b76d87747eaa6230d6b9f594e85','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','196e4bbe93864eb6bd0293c152d46add','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','257c85d919bf49f8b96eaa8882f773de','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','2d16246643544e4bad1131e975113a8e','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','38160b94710c4d038c6f6f8d94bb0dbe','3bb66c2799514f52995cd2511d1e3daf','3bb66c2799514f52995cd2511d1e3daf','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','3fe79b3a348a49fabae424e1d2e38ff8','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','449d52b97baa4df5b76b19299ada8260','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','451a1fb1af7b446eac0943824e894314','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','465ab0c779d64b38b0d562e156847d6b','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','4665b0e22c054e3e82cc8410d2db2324','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','48e7a107ae7e4fdf99f1d3f0ea487a48','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','5768ef633cd54922a978fa14bca14a36','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','58b8e813708545dd9ffefd6444d8e218','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','5f0d0b47b08145d597f8dfcf4726412b','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','61ad4b20cd9a4386881b7b67c21ed057','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','676a5a8c265340518da4e400d5f5a3b9','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','69127d4b15804ba39fb14dc2f5ae8e42','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','6c135f845cce4fe7b9fe3ea5faa17e75','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','78d739b8bd1c4bb7bf1e4fad25239554','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','7ae879d6d7bb4b6087e75f7d4ded1ffc','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','85eef28d6aa14cafb621094c73df9911','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','8786a734f1554a089b8b4d08179ba0b3','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','87f373b2b340427a89a93b7293033f00','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','888f4e6e8a07446aaf569efa73259005','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8cf53d76ae5740afa268f09619bcfb8e','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','8d978e92b80c4f67a9b54266eb78b959','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','902f9a8a4c934b1c8266c9e2476cdbc0','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','90ea60112e76421eb5b74553d79278c1','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','94348debf16742b587bfdcc0ab195096','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','989e1f32f9164a4da9d2396a6bca0ec5','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','9a7cba2dce884a279525431f3fa97950','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a1169748b418488eb5ca4b236b2da12e','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','a98b908d8e304b229953cd4fa51c03f5','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','ad8ef6d073a24f1fba8a086e7de0ab5c','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','afd02a2ee3c542ac9713e2e1ee4077a3','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b206185d28ba494bb22a4b2e38113dd4','b31ac2d7b69248f8bc0253c2e0d27c3b','b31ac2d7b69248f8bc0253c2e0d27c3b','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b49fdfdf3f0c4ae6b46405b6b2111674','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b6ba043b847d47699e9133204cf54978','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','b9a6165b716448d6842b21cfd6ebc77c','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bb53d45b2f4f418e8ada0eb981871029','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','bbc59d9cb36041109d6a19d6eade9b0f','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','cf40224a49c740219a975242ec6f8130','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','d4d0ab9535fd4f179ce2521af945e6c9','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','e1e8b32050aa45a191507d438c133b01','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','ea2b9acd3ab644e093b4c7ce1675fa65','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','eca988c29d304089b09f3d5956e72f39','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','ef126fbefb234d68b96d7e88e231561a','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8','f97b2e3c391841888bc22581f8f508f8')")
eco_result = process_eco_product_relation(result, "忠芝") eco_result = process_eco_product_relation(result, "小米洗地机")
for item in result: for item in eco_result:
item["isCalculateSearchWord"] = None item["isCalculateSearchWord"] = None
bh_utils.insert_data('geo_eco_data', result) bh_utils.insert_data('geo_eco_data', eco_result)
aidso_geo/models/fix_encoding.py
View file @ bcf81c1d
...@@ -9,259 +9,126 @@ import requests ...@@ -9,259 +9,126 @@ import requests
import json import json
from aidso_geo.utils.tos_utils import get_string_from_tos, put_string_to_tos, check_file_in_tos from aidso_geo.utils.tos_utils import get_string_from_tos, put_string_to_tos, check_file_in_tos
_CP1252_SPECIAL = { _CP1252_REVERSE_MAP = {
0x20AC, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, "€": 0x80,
0x2030, 0x0160, 0x2039, 0x0152, 0x017D, 0x2018, 0x2019, 0x201C, "‚": 0x82,
0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, "ƒ": 0x83,
0x0153, 0x017E, 0x0178, "„": 0x84,
0x81, 0x8D, 0x8F, 0x90, 0x9D, "…": 0x85,
"†": 0x86,
"‡": 0x87,
"ˆ": 0x88,
"‰": 0x89,
"Š": 0x8A,
"‹": 0x8B,
"Œ": 0x8C,
"Ž": 0x8E,
"‘": 0x91,
"’": 0x92,
"“": 0x93,
"”": 0x94,
"•": 0x95,
"–": 0x96,
"—": 0x97,
"˜": 0x98,
"™": 0x99,
"š": 0x9A,
"›": 0x9B,
"œ": 0x9C,
"ž": 0x9E,
"Ÿ": 0x9F,
} }
_LATIN_EXT_RANGE = range(0x00C0, 0x0100)
_CP1252_REVERSE = {
0x20AC: 0x80, 0x201A: 0x82, 0x0192: 0x83, 0x201E: 0x84,
0x2026: 0x85, 0x2020: 0x86, 0x2021: 0x87, 0x02C6: 0x88,
0x2030: 0x89, 0x0160: 0x8A, 0x2039: 0x8B, 0x0152: 0x8C,
0x017D: 0x8E, 0x2018: 0x91, 0x2019: 0x92, 0x201C: 0x93,
0x201D: 0x94, 0x2022: 0x95, 0x2013: 0x96, 0x2014: 0x97,
0x02DC: 0x98, 0x2122: 0x99, 0x0161: 0x9A, 0x203A: 0x9B,
0x0153: 0x9C, 0x017E: 0x9E, 0x0178: 0x9F,
}
for _p in (0x81, 0x8D, 0x8F, 0x90, 0x9D):
_CP1252_REVERSE[_p] = _p
def is_mojibake(text, sample_size=200000, threshold=0.08):
"""
检测文本是否为 UTF-8 双重编码乱码。
"""
if not text:
return False
sample = text[:sample_size]
total = len(sample)
if total == 0:
return False
latin_ext_count = 0
cp1252_special_count = 0
chinese_count = 0
for c in sample:
code = ord(c)
if code in _LATIN_EXT_RANGE:
latin_ext_count += 1
elif code in _CP1252_SPECIAL:
cp1252_special_count += 1
elif 0x4E00 <= code <= 0x9FFF:
chinese_count += 1
# 如果已经有比较多正常中文，通常不认为是乱码
if chinese_count / total > 0.99:
return False
utf8_lead_chars = set("äåæçèé")
trigram_hits = 0
for i in range(len(sample) - 2):
if sample[i] in utf8_lead_chars:
b1 = ord(sample[i + 1])
b2 = ord(sample[i + 2])
b1_is_cont = (0x80 <= b1 <= 0xBF) or (b1 in _CP1252_SPECIAL)
b2_is_cont = (0x80 <= b2 <= 0xBF) or (b2 in _CP1252_SPECIAL)
if b1_is_cont and b2_is_cont:
trigram_hits += 1
if trigram_hits >= 2:
return True
latin_ratio = latin_ext_count / total
cp1252_ratio = cp1252_special_count / total
combined = latin_ratio + cp1252_ratio
if combined > 0.25:
return True
return False
def fix_mojibake_text(text):
"""
修复 UTF-8 被 cp1252/Latin-1 误读后产生的 mojibake 文本。
例如：
正在æœç´¢ -> 正在搜索
中文测试 -> 中文测试
"""
if not text:
return text
raw_bytes = bytearray()
for c in text:
code = ord(c)
if code < 0x100:
raw_bytes.append(code)
elif code in _CP1252_REVERSE:
raw_bytes.append(_CP1252_REVERSE[code])
else:
# 普通字符保持原样写回 UTF-8
raw_bytes.extend(c.encode("utf-8"))
return raw_bytes.decode("utf-8", errors="replace")
def get_mojibake_sample(text):
"""
提取一段疑似乱码样例。
"""
import re
match = re.search(
r'[äåæçèé][\x80-\xbf\u0080-\u00bf\u0152\u0153\u0160\u0161]{2,20}',
text
)
return match.group(0) if match else text[:100]
def is_mojibake_tos_file(file_path):
"""
从 TOS 读取 original.text 内容，并检测/修复 UTF-8 双重编码乱码。
Args:
file_path: TOS 文件路径
Returns:
dict: {
"is_mojibake": bool,
"encoding": str,
"sample": str,
"sample_fixed": str,
"fixed_content": str,
"original_content": str,
}
"""
try:
original_content = get_string_from_tos(file_path)
except Exception as e:
return {
"is_mojibake": False,
"encoding": f"TOS读取失败: {e}",
"sample": "",
"sample_fixed": "",
"fixed_content": "",
"original_content": "",
}
if not original_content:
return {
"is_mojibake": False,
"encoding": "TOS内容为空",
"sample": "",
"sample_fixed": "",
"fixed_content": "",
"original_content": "",
}
result = is_mojibake(original_content)
if not result:
return {
"is_mojibake": False,
"encoding": "UTF-8（正常）",
"sample": "",
"sample_fixed": "",
"fixed_content": original_content,
"original_content": original_content,
}
sample = get_mojibake_sample(original_content)
sample_fixed = fix_mojibake_text(sample)
fixed_content = fix_mojibake_text(original_content)
return {
"is_mojibake": True,
"encoding": "UTF-8 双重编码（经 cp1252/Latin-1 误读）",
"sample": sample,
"sample_fixed": sample_fixed,
"fixed_content": fixed_content,
"original_content": original_content,
}
def fix_req(req_ids):
url = "https://api.aidso.com/openapi/ywt/reReq"
# payload = json.dumps([
# "35506100-213c-4264-bcc4-67d3449431ac"
# ])
headers = {
'Content-Type': 'application/json'
}
response = requests.request("POST", url, headers=headers,json=req_ids )
print(response.text)
def _count_cjk(text: str) -> int:
return sum("\u4e00" <= ch <= "\u9fff" for ch in (text or ""))
def _looks_like_mojibake(text: str) -> bool:
if not isinstance(text, str) or not text:
return False
if _count_cjk(text) > 0:
return False
markers = ("æ", "å", "ä", "ç", "é", "ï", "â", "œ", "€", "™", "—", "–", "ã")
return any(mark in text for mark in markers)
def repair_sse_mojibake(text: str) -> str:
if not _looks_like_mojibake(text):
return text
buf = bytearray()
for ch in text:
o = ord(ch)
if o <= 0xFF:
buf.append(o)
continue
mapped = _CP1252_REVERSE_MAP.get(ch)
if mapped is None:
return text
buf.append(mapped)
try:
fixed = bytes(buf).decode("utf-8")
except UnicodeDecodeError:
return text
if _count_cjk(fixed) >= _count_cjk(text):
return fixed
return text
def repair_task_files(task_ids):
file_names = [
"context.txt",
"original.text",
"quote.txt",
"result.json",
"search_keyword.txt",
"spider_result.text",
"suggestions.txt",
"think.txt",
]
repair_results = []
for task_id in task_ids:
for file_name in file_names:
file_path = f"geo/{task_id}/DB/{file_name}"
try:
content = get_string_from_tos(file_path)
# 文件不存在，直接跳过
if content is False:
repair_results.append({
"task_id": task_id,
"file": file_name,
"status": "not_found",
})
continue
repaired_content = repair_sse_mojibake(content)
put_string_to_tos(
file_path,
repaired_content,
)
repair_results.append({
"task_id": task_id,
"file": file_name,
"status": "success",
})
except Exception as exc:
# 单个文件失败，不影响其他文件
repair_results.append({
"task_id": task_id,
"file": file_name,
"status": "failed",
"error": str(exc),
})
return repair_results
if __name__ == "__main__": if __name__ == "__main__":
query =bh_utils.query_data("select taskId,reqId from geo_commit_task where platform = 'TXYB' and insertime > 1780502400")
# query =bh_utils.query_data("select taskId,reqId from geo_commit_task where platform = 'TXYB' and taskId = 'd7984dca-3c17-40ac-89d5-ece0ba94a29c'")
# query =[1]
req_id_list = []
for q in query:
taskId = q.get('taskId')
reqId = q.get('reqId')
file_path = f"geo/{taskId}/TXYB/original.text"
result = is_mojibake_tos_file(file_path)
if result.get('is_mojibake'):
print(reqId)
req_id_list.append(reqId)
fixed_content = result["fixed_content"]
put_string_to_tos(f"geo/{taskId}/TXYB/original.text", fixed_content)
yuanbao_process_original_data(f"geo/{taskId}/TXYB/original.text")
print(req_id_list)
fix_req(req_id_list)
req_id_sql = ",".join([f"'{req_id}'" for req_id in req_id_list])
# # # print(len(req_ids))
query_sql = f"select * from geo_commit_task where reqId in ({req_id_sql})"
#
data_list = bh_utils.query_data(query_sql)
print(data_list)
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
type_t = i.get('type')
# return task_send_queue(i,type_t)
return platform_process(i)
if data_list:
with ThreadPoolExecutor(max_workers=30) as executor:
futures = [executor.submit(handle_item, i) for i in data_list]
for future in as_completed(futures): task_id = ['0aef4499-9cff-4ee0-b5c9-0a3593eb0b4a','1576a6e8-b117-40ea-80c2-606d1fb3cfa3','931f6338dd54410984b19f751b18ac9f','557158db-3696-4971-a079-a7604d230015','6ea6b1f3-3e93-4712-b55a-05aeb1ec2010','22c31660106744de9a2181d364dccac5','b1449e9b9c284a9aaaf495f62695d206','f4d2d862-1de5-4bb3-9a34-702ad8293e69','c13ca173-46c4-4486-b197-c04ff8818a72','45915929ecb74064889631bc81fa19d1','5b473601d67547a493fb23391d676d9d','f2316bb4901b4568a9c384c61810566c','f9a7300c-ba34-490e-8472-337f655ab515','b90cf36caa424174a7ded15f7e801a07','ed598e09-b9b0-40fe-b0e7-a90bc1a38579','ec981747-327b-4310-b9dd-e874d75ca5e6','488a231f-cdd4-4bab-8ba6-656071672903','8e39bb83-d62b-4736-b1fe-a76216bfe70a','8f3cfbaa-c16b-455b-8a66-7046a60cc787','c374defd2e70474396321caf316815b2','7a86d3857aa049139d96a0fc83bbe0e2','c75136b2-537e-4437-a3f6-deab301e3913','4c9986b8d3cb482ea6b3fd5c6ea7f242','e2295ace-93ca-488c-93a2-f5c33139a947','9777982fb872456e82c61963609d2ad3','49ad848f-b2e9-4136-905a-c0183df23a51','90caff80-8a66-487c-a98e-51a83dc28ad8','c3fe601d411b4db2a5d3150514b77008','bff9cc95a4b44d7a9eea3141e94f5f14','4c092f34b9424aee9f27f185605f9e75','9fb10b02-b003-48b9-8527-3bfb7ce1ad47','338ef00b-800e-435c-a60f-e5519e325cbb','01b37441-5ac1-4c8e-9875-64bff6dfdffb','451a857bdf1e4565ad8688f5364a99d0','804b5e3f-80b5-4c82-97b6-49dc54b2a030','5d07f7c3-79ce-4cfe-b7da-4fdd73500d11','82786b95-ba82-41ab-a1cb-2a96c1e4d72c','49916264-6370-411b-9acb-19984689399d','acf14bf5-d44a-468e-8e56-7767b0bba47c','bf1d178a-ca88-499a-b714-c636829be47c','06882840-b546-4164-9aad-0de56898ece9','5d57f4c3-7e6a-4cae-84c7-a05f43e267f2','946a2a36-2360-4dee-a86b-7a429a954e42','67d67541e71f41a59b76269ee193b3aa','937a1540-0cce-4c73-a604-7c3e8c66ae86','64bb103e-9f5d-4a1d-aeac-4f5cc5b02888','251e82e9-dc41-4a57-a607-9928905e7cc1','04d9ed6c-24e4-44b4-a493-415e8a719e1b','6d1d68e10cf841869530192d59741e49','8290f410-d11d-4547-8cf0-8679482337f6','46ef9bd4-a6a0-4401-9da1-4a7335c1f295','dcd57d6ff3f54cf59dcec93e88244e30','859c949c-6937-496e-96a3-f695cba3018e','7e55227cfc5c4f7caa6ca8f390875859','ac448ed0-cd06-4a49-a051-84b34651a4e4','a91efb3c-6c79-4090-a253-4064fbe1924b','c7392f2f8d9a4923a5ddc481d270d012','1e7411a081334449acaee2f30be9c26f','1b2ee945-26ef-402f-bf6e-722692b2f9de','53e8577d-aff0-4828-8189-c4dbbb408709','c0dfbdf589164367afc945785fc824ac','d4784da963d8427dbb48ad2e27977c47','536d41eb-aec1-48e9-bfe5-2e1aa9a3ad83','5b1cf55f895e42cb8e38f1b8126a09ed','d88bc97a151742dda6be33437f6e3465','60109261d66c4779b4466f66b9b098d9','47cb8ccf-8409-4dee-a343-6c4d0035a978','b757f641-a687-4ec5-88e2-6b0e81ec9d45','2352eaa1-a6aa-4929-906b-07e8404d8cd8','1f1be5fc-a125-4c3d-8c97-f35fbdc5c3cf','63112c88-893f-45f7-8b72-7177f181e8a5','4ac16096-a264-4d7f-9702-bfd055075f15','31bb05ce30ba4d1eb96e54bafbb7b5be','34266905dfe24460be8c295a8d30ebf9','f48e3d7a004a4cc29e521eacdff9821b','ee743cab4964493485cca4b677164557','8951aece-2e39-4068-a1b4-db3ec6c37260','c73cac3f-368c-4256-b1ee-959571fcd81c','4f838cfeb9ef4a8e91d47c6ef26cb4b3','aeaa45ea-a52b-4bd1-bb11-26157f73f310','e24703e9aa3747878b745c5b5e60dc57','b4605ae9fae74e28bd18461f40faf9dc','441e054b-7a50-429f-af3b-f842658b45d5','684efb73-dc66-4eb2-8c74-72c69ae29a77','845618411078491eb140c678a2243db4','3d3d8af7-824f-4554-9f9b-3d0a76bd267a','41fd8ebabdc3473aac29e104020403a4','6cd8b53d4777476c89ff47ea27805c34','3b4120af-3b17-463f-a46e-1494422f277e','9b9c4b6af8924308bf71a73bd77c6352','8de91024-ace2-4b11-9c2a-575ba17c63a4','0fe70530-5001-4f56-ac63-ac77f5eeeae0','3e9336f021ed48d9a5f7c5ed763d5c2d','e27df412-083f-4a2a-9b57-779759602edd','10409a92745a435eb5ab69dc119fb72f','b7b482234b344838a83979c8ee656b1d','6ea9c4bd833e4a4aa6d4e6d6fcc0d8c9','afa0972bc58b4600b523ae39611d4cd5','ece5f117cf164abc8ba682c0891fd8ca','29ea8cf5663a4860bdebe41ee4b66b1e','521748fd2f6b4fa1bdf3e5bc5f2e226e','185a5aff-3072-4b2c-8873-78c01bff6830','267b0544-e9f1-4846-9264-545aa8f6d8d6','1d9f169160ac4734833e88e6590f26b3','031b5fbb-e142-427c-ba6b-5cc9ff37d8ee','efaffbed-34eb-432e-8485-ab361014c9ab','235d0de5f59c4b028a9af3bc266c1797','efd6bc36-4acd-4343-85cf-49569ecfe8c1','a1c0abbcdd3d46eb9f2c7f6f151a4673','32e96ecf-77f3-4de6-8ee8-ca693cf27daf','2aed036c-104d-4040-a944-9ab5c329d3f4','f602bf55f51d41c6ba13ab0b18eeba14','fb62fcb2-7659-45c6-b6e5-ca68b121cf7c','376da7b8-2e43-486f-a36f-0f77a18d6094','496cb99b6b9e457995aabd9463c52080','277cec06-f18a-4ec7-b7d7-33fb3ad9dfb0','4e09a8d1-c135-453c-bd8d-c4762afdf204','083b4f626d9b47f3a48a7be9f56144b7','db11ee999e6b4f1397caf51323058f11','b51a4e19-ac93-482e-86e0-e020b509a15f','54d73128-60a0-4008-b957-36c591a2286f','2c9b4b4f-8538-4de1-8f2e-1929e774367a','eef8c9ef-4c22-4009-bd29-881202966251','2c6d5d23-6cf7-4ad5-8706-eda887758d0b','a926b274-63fb-40f1-96b7-af75dbf80a04','46fab7c8-b834-48b0-8b75-b67dc8a2cab3','0abb3d6c-331a-4d27-b74d-c0292b4aee55','253fe5e9-aeb2-47ab-bcd3-b73ce9b1771a','cfe6174d-a176-44e1-9f3b-402ef7c34321','8ef2e74d1d24473898fba4265eadf762','c617b1ba-db92-4208-9802-bb0092879f83','0d1eaf141478484bbce4e27e0f2d51d7','8ceec595-841c-426f-9046-25a38550e8bf','70255a98-629a-4172-ac2f-192aa9e4a6cf','cc74b1bb-f25c-4f54-b216-b533afba2484','d6ee9db5-6677-47b0-a5a3-6239247e6965','43b8d865-5882-4355-9a4e-9babbff37a8d','ecf540a480054bbe9ead0bf8caaa5920','c0e33a65-8b48-4e9b-a52a-e4f6df6adcf2','39b76dad0ca64c29b5e90706bb00d3e4','efe6bba9bde146ef9441ab9f6d2a9f70','4c15f577-c68b-4cdb-97a1-249d170500ad','470767aa-0422-42dd-b085-1a44c87f4b67','2812147e-98b0-4a26-96e7-518693f75d86','28e55422080d459fbc17a649dd09357a','50748633c161422fa3cc09bfb8eaf75c','e1aa09b9-a4f8-4c0e-bfe8-ff2c5b90ec7e','ec32715e98e44f18bc8f5f727dcf9873','db603e24-331c-4f29-997e-d8428e090fa3','5c014b82031a479eb15322220f912774','773cadd0-7928-4e60-bd50-28e9ee53e806','382aeed67cb4401fbd9bc1d3e83802bf','d89a17c650da40e9b80515e971187c1a','96162362-ff9a-47e2-aa52-72f4562f6127','30e51381-b240-4d7f-a567-4921dca8df64','de29620d3a784e468da6772f936c8533','0ccc38bd9f0b4251a6e6daf4ca89c62a','1c94eced-9e9c-4558-b89b-3353dfbc6bee','2c7f92768c9a4b6cad50542e6e43cdfc','0c9e764fcc36432096aa0f08d5d4b968','9b3ed69e-d9fa-4f0d-9604-1d1b30e6db86','96b7760fcd764e07a709675c44cf398b','367ffc65-08fe-4c37-b397-7559ed2ea966','6597e393-fd26-4fba-b61e-74942de77b49','9fdf182d-da2a-4dbe-8dfd-66deeb1541ee','9a8d6b9cf0dc4f76aaf450a90d1416a4','d544233744b24f47b74da2466269e307','eba6f424809447a191a10266f5d51e94','e7b799fa038347e58a00975d2b41c0e6','e6ef5ca6-5830-47d3-8bd5-911a73d716b8','61b02401-965e-43ed-b036-8fc6d88177fb','0fe1fb1d-2be5-4100-83cd-97d8a8017d6c','c1297db6229a4cb3a6ec2a45ec7d6c59','299bbf39287442c9824860b642788f38','b1b0ebc16d6a4cf9b24301539c7a8ea8','b73d4c12-9d03-4ce7-b319-58939e005812','bbb1f00f-3ef3-4d0f-92b9-4f9bc353aac1','79f5cd3884044ffa91dde8b95260ea26','b74b8323-180b-495c-855f-2bde27d57cdb','6dd6530a176f4dafab725fb70fe8208e','efe5b799f0db4df78c5cdca7812422c5','192d4bc6fa7d40708a20d1d63242a643','71e7f11a-7bf9-4d2a-91f7-f173d2dbda26','f97b0d6e70894fc28f4b0c656804f812','503991a7da3248df886920cb6505ca52','817d4c5503b3403fb75d807a74b9b9ff','98dddcde9b8a47189f9f5b2406d6650a','c323f81f137e40e197a22ae29d5b5323','04d3e4b4acc348b797df834ad990a620','03bb86fe47804cc39ec5b566884f0260','2ddaec5f-6501-424a-91bb-600accccbaf9','48009bdff56a4c82a701d346f13a0f1c','ca17c0a2088e414381caf1e01bdf4423','d4b9744b-6565-4f01-b51e-9aae385dc294','4469bccc-f427-4f55-b629-c49f94518d05','747dfbcbf77044b8b651ee5c5ce1bf6a','d152a02640fe466c915affb392177ec8','a124604c-60b8-416c-b400-106c9666543f','faa319a0494443db82bdbb64364917e3','f1cba541-6f1e-48be-86ce-958ec70024a9','ed1161e822524edc94fa88063d9fa167','b97c23ff-1856-4801-86ae-35f774ddb106','e03ebb4816ad49f79c88c4d9f6aeb122','d6a3ff1246d4458a94db2e48cba3a39e','44a15e6278754010a891033e9c885f50','174c0d8b-56c4-4c67-8c44-9bc6904638ea','0adae33a-74fe-44ea-9c70-4fb6dd8a5d48','adf9d136-d337-4cae-b868-3c582fb1d560','acc0407f-95d8-4d08-a58e-944e8943d807','f1c4b94b-f91d-4300-8acf-fe1e02146c4d','637ffe9a95f14f7f8cab1c94572952b3','43c2b18737d54fbca0b6000bf020893a','cadb59fd-0a3e-433c-9aa4-2186e59b83ae','2a2bbdf7-303b-4053-8516-8d9dee9709bf','5871e0da-c378-4974-b84e-f29ccaadf0fe','7df22874-3b6f-4d61-aa4f-6abd2b2f7b53','21c6c560321b4be39d0f6aa0801ee593','44a5a854-43e7-46d3-b198-7ebc30c9fb70','791429a5-9def-41eb-8f3e-cfe38dbe2bc0','33d79b24-af7c-4678-a9dd-153b018e651b','6e274d3b-ae2b-43f8-b756-d3b509f9292e','cefa149a88394a5e8dc6b763fd1fa3ff','3b709fd4-9157-4668-a60b-c36827a6b8a1','b71645305095419a82a8e0b45fda557d','f027c69b-ddbc-4192-b86f-b3e9ba407b89','0cb7e5f1-6c7e-45c4-bfcf-b6621bbc280d','0103f2d9eb39466985c8a0f5d198415f','3cc09e75-8464-45d4-9c43-fc1c1f0d0463','3073a30c-522a-409e-9148-b4be9c82a583','63d74c5b-30b8-4af4-a660-797d9388d690','e6b2a19c1b874f138958558ad01fe6a8','e4e38281084c468b89b79b8fce8e50ff','97698706-78c2-46de-a298-4dc94e5eca97','d6cd0fde-a9a9-4652-b8ff-d1b73c488f85','c02c90c27f3744199abc5284f208e492','f0df22b3-242d-4972-acae-00ddf98d8e3d','2fb523e435ab4ca9b1856d2c111accb4','b8ae39d8367948f5bddbd3983f64c5d7','a622ab37-3fe5-4355-b3d9-9c5e25c68d7c','0028f2781d8b4805a672c36de7416f00','28fab4dc-3de1-414d-819e-c92e203e6742','25aca583aa6042c2b7e7fc01cb7aab0b','8ad42a3b5b484f4cb5d2dd4de16c024f','6c7edfcd-a419-4203-b882-cb88165cf70a','3cf896c6-60d3-4286-94a6-c3ac7a12161b','94a5aff4-15e3-4c17-adf8-24f7c8a7a056','ba795003845e47479cf0d1a1f0787843','cf6d1bf4b4f8485194732947bb9d7f86','0bdbbc88-842d-41d3-a1fe-b1f7e9bc9e56','9ae86aca-526d-4200-ae1a-e3bc47f348d2','61dcb477-661a-4c49-a341-888088128f37','99d632eb-92ca-4df4-b912-8ce445546d44','507f99ff-adba-4a94-a7ee-d1b2a1c83067','4d165625-b0ee-4e46-8f9a-329d5bc99cf1','b516d03b-ee77-45e5-9dad-5882fb4be11c','a457c55d-c930-4a43-a307-0019d3865965','e0da62a6-159b-4e19-af73-4a83c3b4ec90','9d92ddc3-f147-4ea9-ba2d-8406f85d2f58','b8d20663-ffd2-4965-8c42-8b86ffe2fbaa','d7f7ea61-6945-494f-89fd-9c4e9f1edea6','47ad76ed-61ac-47e9-ac96-ad62564dad73','154a83d2-90cb-4a0a-841a-46f9fbb53458','78e9d55d-e3f9-4cbe-bcdf-c980ab1bbdab','284c848a-7435-46bd-939f-3a8811e584ab','d09172f0-d297-4030-90aa-db37f14d466f','ec010ba7-2a0e-4916-ad73-79b1f36543fa','5b292a8c4328434e8cb41862c905a0e4','1cfa608f-e5d0-4d6d-9240-d001d14cf55a','59739bb2-cf65-4c9a-9097-45b53ff6b973','ac0546a3-5aec-47d5-bf09-3ed27ff8d5f0','dc005232-fc2c-4d8d-8a1a-4e879698777d','175add0c-293a-4074-8e17-e5a62efc9c5d','594233b7-d783-4bed-9b4a-ea81ab281d5c','a21824c0-aaad-4576-967f-4df57c54205e','93e4a0a7-427a-4380-bb36-42724a7f4794','8055ea82-aecd-4e11-818c-8f10cb967b11','ac82671a-0fe0-460b-bfd3-7732aee17492','e1cd88fe-b589-4900-8027-fc23aa734f65','35edf2f6-89a3-428a-81cd-1bdbcacaec6b','35ae8a6c-62f3-4f05-a7d6-6bf5dfb09370','1534e1cb-82e3-4266-901c-ef348a30bd95','2640e8c2-effc-4df8-a8a8-919a5528dae3','73f223e7-9f1d-4eb5-919c-3cba8859e982','7a21acc9-eb30-4080-aa14-a1185f8e3a9f','0dfc4d32-95e7-4691-9f91-ce8834f88c08','1b305033-8a33-400f-9465-570054ba6b6c','642abe08-d716-4f0b-b187-b470cdf3495d','2f94dfd3-3a57-4d6a-a33f-d8e9d9686c8a','af09869e-2ab2-46ad-89e3-4914a8f91a2e','7a38463e-b613-4a4b-90c8-3c70be8fe2e2','2857fe92-5921-42de-89aa-1f740834b359','6371f4bd-0323-4624-b82f-72c88b4194a0','9f862fe8-924c-4f1e-a392-4487e324de57','e0edb086-9bb2-43ec-8889-168ab3d23dc3','8c949aab-d5af-499a-9d24-76f36f2d57be','94bbe802-9fb3-4f82-8a52-e40018527437','78f55976-17c0-4b33-a7dc-2397e327e115','9db8b8d1-29aa-4cba-b6ce-6fa0b2dc1fd9','a6d7418c-a04f-41c1-ac75-fa567beb50c4','d0e9ce02-d9fb-4703-af2e-b877a9f2b783','6190e968-83d5-4e9e-b350-9895514ba411','032d3e42-af14-445c-aecc-1f7ba98f4e53','40601dfe-8111-4f59-9687-3f86f3820e31','b2bfb16c-492d-4558-b955-450a2d04bb1b','364eb785-a338-43a9-b919-1329eb12eed8','8d98d3f1-2873-4e4c-bbc4-302d97b9d309','c6b06929-cf10-4875-8db1-639510f43237','30faf658-0e6a-4b38-ba65-9671758bd5ea','f2acac8b-d008-48da-9fc3-b46e857d2099','1844dbfa-02d4-48b9-92ee-dd41422dbbde','cf40a0ed-353c-42de-8265-3c33b6cfac5c','c36e6d76-9fd6-4740-a806-42b19bb0f3b1','14225c971c0347e4a8f8c68ad3c1e8ca','6409b006-8e2d-4e93-af29-4272e306414a','84e929e0-4df7-4f93-8e5b-13381f8a8d1b','8eac96917d8343978c27c126e94dd8b2','aa3d17ce-4769-4844-a012-b7d3af5be98e','1fdde8720b5449dea6b99b908bbbfae3','4ddc70ea7d964421bdff0753fcef1df2','4eaaaf99a23d4f4bb4d5e0ad88fba55d','36ed0d4f-7230-468a-ad85-3ea41b977665','e8121374-18a4-498d-aebf-2647e5ae8417','907df068-ce3f-4367-b0f0-1ba4d637d971','04624b0f-7f1a-4737-9e1b-a52c5ef41d11','ffd788ef-1f54-491a-a992-1ee7d2b64d09','cb24ec52-5e33-4f1d-b22b-0b2f2a1513ac','fa8d665b-a535-483d-a860-c8f4d404c6fb','cbf07f87-f5a8-4653-9a99-1983df9f80a0','805d361c-c292-4efe-907a-aa94aac5d074','139d7716-3f42-4694-9072-9976f5f444e2','066d60b9-093c-425d-b930-5b47eb87a353','6761c060-2bd4-4148-8aa0-ed3618128389','96784956-2132-476e-9419-3c597a98559e','91a6a06a-89aa-44e7-8987-9b593b093c0a','1c3e99ef-1ccc-4c3a-8f13-ee8741afe76c','18c641b6-091c-415e-92a5-0a1479b51692','83502f3f-35b3-4553-86e1-e2b772480ec9','8ea32d77-5685-424d-b934-3cfdda4ba242','97b2cc0c-f475-43c4-87fa-7d56ee777242','b9567630-a646-4328-a362-a4895307e344','f4610a5e-48c6-412f-b079-e1472066fce3','185035db-5e0b-450d-a602-2f7bbd83aaab','06177748-e52a-4e19-a8d5-6fb9c4f69d68','36f8bbcc-5aef-4095-9062-41d376dfd5ce','8b378d7b-c10d-4a70-adae-ce8e9adec6d6','19fcf812-786e-4f04-b3c6-76390decaf1d','8b7e8f85-5eeb-45fd-b928-fdb5bd2719a9','6e83b770-3299-4544-b878-ecd83e25c062','15345dfe-7eaa-4662-9075-9fc6e014c7bb','a3bed48c-f8fb-411a-9b3a-fa550d342216','5144c6c3-cfdf-421a-8320-164d49c1a308','c0d7cf1b-383c-4da8-a759-5deb73b2af8b','fd6d9edf-697f-4af5-af12-4d29ea808db5','7cea44c6-c3a8-4b35-b814-ecbc68025873','edf2ee36-bdf4-4934-86e4-bb56a7b57d87','e22b5c53-36a3-4774-9979-db2e8b64c029','45dbfb32-58ef-4d20-a346-57620200632e','26489e4d-5c28-4859-af6e-109d8cea7496','01f1a6fa-ff41-478c-91b2-9a3d5735f407','3ea524fb-7e34-459a-9b66-198264a46935','988f5584-b9f3-4599-a3d9-0f512f04aba7','f2f54a39-5dbf-4223-98d8-a3d65922a830','5421a71d-a98d-489a-9602-3a09d02c503b','234ff367-89d6-4786-9981-71427059cbb5','62a938c7-4edd-4260-9b2a-20d86c78ae65','d994939a-1e7a-452a-a123-dc36da226acb','5a473863-b928-4a1e-bbe8-3b31a0957b6b','896c53d9-a6f6-43c1-88a9-289fd2f7df65','d83cdcca-aa55-493b-9d7e-0b1511f3ca4c','5f36bc43-5b45-48d4-8382-b1b3270d1e3b','007e2ece-a55d-4afa-8331-091d32652fa0','f1d3cb3e-5c0f-48a9-9017-eca69a33dcb3','521505b8-dea4-489c-89c0-33793fabc926','a1f435e0-ba91-44ce-855c-ab99848b6503','f6238d81-cbb2-479d-bd32-5c99b780c923','8c543e3e-cde7-4605-9033-0bd2d3cc4dcd','e3043154-a435-4587-885a-81a453180b00','bcfa27aa-57fa-462c-af6d-e33823239985','6ec40545-a208-4a35-a604-a6582cc9b0d1','423d8a8c-00a8-4c8e-bf7e-00d5bf779fc0','305baa77-79ab-48c4-a052-2e5f4cf2c8a3','87c4d46a-c2e2-4f12-b32f-78afeea161a2','7f4db474-f49b-4895-8cff-20eeb667e98b','29df1441-1f97-4e2b-89e4-36e9c9be54c4','8d7f567b-5b35-4b01-b98d-4536807096f8','69ecca75-ef37-4c50-8e81-b62db5db5d59','5834d32b-7f2c-45e9-8415-f31566ff4177','f3a2395f-0054-4499-bfae-316bd0bab2b3','66d25402-da71-43ba-81cd-ac64f20deaed','14acc55a-ad21-4d4c-8d8a-3bf65c18378d','e018e5ed-f846-413f-84f1-40b5e4ed0db9','e01ac8df-5c21-48e8-9f94-0eb20f2618d5','e8a20c7f-b702-4f07-b94d-369d382e1db1','b09ac94e-0470-4e7f-88a5-d1839c4da986','885943d7-b2b0-441c-a147-c2d2d0223edb','cb67f40c-4b85-4f03-ae74-a8ab3662740e','23f60566-b142-4201-81e9-fd9a2cf64df7','fb59da3a-cf76-406f-a5b9-b73105d349fb','d94838ca-ef44-4db1-8d0b-55206c7581cc','2a8ee258-2af8-410f-967a-9a54f0cf3b26','69c29253-10b5-41bc-b75c-1cb6fdf4f996','e18d44e8-7b6e-4a49-93dd-cbfded1e14d7','f44e2280-3459-4f1d-bbb7-ce1d61910e3e','6ef15d7f-652e-4273-afaf-09fd3b3ba068','b9e4ae70-8cef-4780-9831-d224bfe71063','4ea063df-aef8-40cd-aa55-757aaa555dfe','854464aa-de22-409c-969f-3d21269242cf','4d34d5c4-1329-4f27-a700-1c7fff980978','53f76f6d-daf0-4e3c-a563-8ff370ed3df8','bf0e8403-a13a-4d58-8c5a-1352f6e501e3','55ea3bcd-acaf-435b-acf4-9c555b6a46ec','d73e169d-2bcc-45ef-b7fc-8895c0e2c24b','f8f1fe29-e5f1-42ce-a0cd-8a9e4ff80e4b','8aa2a5d9-900e-4829-91f4-1b482df644f9','67cc73d4-fa7e-443a-a614-9c04ee2db639','9f540636-e8b2-4ad5-a7eb-cc8674323403','2e2e4f99-9a6f-4f8a-8769-33b2a0257de0','6c2eabf4-f085-4604-bd1d-89949db8b942','dc160cf0-4ae9-4836-9474-f4e1d746649d','357f1098-97d5-4d67-9fbe-1957f5bf12c5','6b398d2f-f051-4c82-b6dd-5804add481d3','a665e178-c7bc-4b2c-8d4a-cb786cb5d6bf','79a159f0-9e6b-48d4-b251-5ed220abfdec','c55f3a65-013b-4be9-95d3-339a846a9a65','1932599f-6bff-4bc2-94b5-89ffde0d634c','06d9e39d-5c50-4345-bec0-877b3568b46e','54771237-d904-435c-bb1d-f05ae0ab6912','10beb646-a387-473f-835e-6ab0a1e57137','06606f39-c449-4402-a8f6-e9a3472b3090','93b479bd-55f6-4b25-8350-6b637c7b60a3','f65ba8d4-50d2-4891-9ea6-09c659dade34','bc134a98-4d78-4a82-af4b-0e6d3775374c','46a79008-f233-4f4e-9093-413f974a620a','43fe1d83-28f3-4d34-938b-ee804ab89229','ac865f48-dd02-4469-8a48-dd17280d5189','5bb71ca1-8b4f-454d-a8ab-bcb695de18c9','70f3c6fa-9178-4450-91b1-bb1042902c9e','60605c02-24b4-490c-947e-fcc47be1d7e9','70911567-8cd3-4a67-ada6-771c0e708190','0cd3becb-49bc-48df-ba39-8399c17af07c','0523ddef4e6b41a79be0d82abcfefeeb','960381b8-3864-43f4-bc06-c068c5cd4d86','adf2b920-91d8-4ca7-addd-2534c76a5207','ae3a6a7c-5ca4-40ae-9fbf-1f3c159b869b','d17de7aa-93c8-42d9-a2a2-07a8eeebe857','70057768-22a9-4ca1-ad69-dd49c9dac8c5','ee77b405-9ef7-4a79-bfa4-0249d97f66d5','1494c7a3-921f-4ec8-944b-7bea6bf72de6','e5ab5670-4397-48bb-a965-75234de0dfb7','9c8a731c-d028-408b-89f9-835398edd6ff','d4823fb4-7c49-40fa-ba65-34352db1f12a','18503646-d5b9-40c0-b55c-fa8e5bd6004a','1ac45e3b-8f3f-41f4-b33a-aef3256e00ed','aeff73fe-06b5-4280-892d-b3462e912deb','2081ff1a-12db-43e0-84b0-00b4a83be88e','d19ee821-bfd3-4397-979d-23f87eada468','42d5fb19-0ca2-4693-8b81-caa656c6a935','f7cc93f4-3a82-4980-bb7f-3cc60df290b1','b402faad-7608-4d17-9070-b79fdda44064','a9c2b74d-3d4e-4649-ab7b-e3840f13c818','4122c4a6-64a7-414a-85c5-fa9a314c80e7','dabe4e9d-bb4f-45e3-a32d-c5623669436c','7ad0a767-4688-45c9-ac53-3ce32925f0d7','2e4edf2c-2d9a-428a-82c9-c861ea8bcae0','61410167-1eca-476f-85b9-54ffb0b8876b','1838b6ad-0725-4b4a-b6ae-86088d2ffd01','55b7eadc-90ea-40d5-9251-df33afe63e19','fd5ae64c-71fb-4330-a046-412b32fc3ad3','0c8876d8-b0ab-479d-b507-162f388be61f','5d8d3a38-0343-4900-8b00-d354b5b685eb','fd306914-5605-4bee-883a-18a431c484ea','aa39640f-fb3a-456c-a618-0d0e4a01642f','dd374b0a-edec-4dd9-a91f-615fb39d8c93','7ae46fb1-a182-4c7a-b0ee-6d01f5ac4cd8','0504bf02-0025-478a-b1e5-c84ff609f9f5','b169f8a2-e65c-4dcb-b95c-22866fbf97d8','e086e798-a5c1-4021-a3eb-fa92ac23d422','b6d8a62a-df36-4798-b9ba-20fabe36e436','c0657b1c-72ea-42c2-ab7e-76802ee31004','03e3479a-681a-4343-9de1-7c332d00d465','a7a1923b-dfbc-46f0-ac01-5562ec6d55a2','78737f69-a454-4d6e-91ae-4eb652734b01','8d567c29-c769-496b-961b-f33947c854fe','6baddd0a-709c-453c-8afc-6eb0ddd11dc6','ba034642-ac08-4e1f-a800-9d64dd7d4702','a71a4f23-0318-4172-a512-7c01c7657579','ec0c82b9-b4e9-4c8c-8c05-44a3b6aa08e4','d09b5f39-0ee5-48e5-acbc-09edcb5f749a','4c461840-5047-4ba3-ac9e-39ff1bfd4cd4','806e6136-1d0a-4a2e-b72e-51628d2cde32','f2857616-f188-4f97-8311-aa35de2734ef','203f751a-6783-4258-bfb9-e8575b733820','f4726d42-b14d-4255-9628-ed617a58c38d','9c79d627-5973-4be9-8fe0-8aeadbbda640','8c385013-58f8-465b-b82e-0191adf280cf','cee69e93-9f5c-46b9-bf68-7c734c6cc7d8','5fd60658-3e10-4f65-a3e4-291c91f4c95c','d831e4d7-9eaa-4899-a3cf-b300a5a7013a','903ad0a6-948d-43f3-b598-af1801d43597','d16a8b12-c1dc-482a-88f4-418f031bdf32','58637034-9369-473b-a571-696ae7f29432','f674f60a-3aa1-40a9-9ced-f220f6f95aed','622a532f-e9ac-4c1d-8f97-843c942c80ae','373689ed-38a0-43d1-ba8c-704cf9cd66af','0dd7d59b-39f2-4a51-9d48-8a51f9ff3500','670d11ed-e7a0-4498-a533-25cc0db9086f','29969c33-eb10-4b3a-b2e7-2392d678d6a2','bcd27c3c-7db0-4941-9ebb-71b452657ec3','f5c0037a-7b15-4dd8-a185-3a4d42a8f4df','d7fbd007-10db-4b91-b178-eb545a02cc3a','33749aed-8f84-4840-8d0f-2ddad6d85474','1e02495f-1d5b-48f6-8734-4ed2fbea4958','cdefb394-ea81-411e-892c-fd268f379d60','f948bb61-d9e6-45a0-bac0-863b7757905e','05520e18-b32b-40a7-ae90-f41ce3be2045','a5c33b44-795d-4c51-9e6f-45f7b937a8d2','b131c720-cd4b-4b3e-b329-3633fb8857ff','480f8410-5f6b-448e-b80d-3108d5d13b88','bd4dfad5-8ddb-41e6-88ce-be1f9176f4e6','eeaab251-b78f-4e1b-b2f1-77a62aefb0bf','7ca88f5a-71d6-423e-8895-db4708bdfcbb','6408d5a9-f0bc-4a96-8c69-fe84d26f4302','a347e086-ca21-4ed1-b657-73b270c5e8d6','356056fa-31d5-4a61-8a3e-93e4fb49f21e','22b9112c-5579-45f5-bc21-2cb12ba01c08','53169816-7c73-474b-855a-95cafb7726a1','fb66c9c6-2d2d-4489-b71a-c378e13f9059','08493378-26a2-403c-a9ab-037131d037e1','85a545c5-4c49-4238-bd2b-09e0e4658729','e76b098f-3fb1-4aff-8a2a-6b54da9103ec','ca9ef602-977e-41bd-aa20-fea9015c987c','67b03cb5-f804-4b8e-be90-617aa51d5c70','414bed59-674b-4cfa-a9fa-311e8c6e9481','5dc407ce-3acc-4fb7-8aaf-02ec5346f59a','492a4d31-6fa7-40fc-a706-f6864f5a9281','8c26b9a1-644d-4c9d-bd13-e9ef8abd70a1','72613318-3bd6-43c1-8720-34e3f1cf65c8','4172d9e5-d23b-474e-b9fb-1721d0cb0347','8b3115ec-9150-409c-8c17-b520c2f9ca60','3dd466af-3392-4b1f-b4d9-75ab1c173708','27faacc9-7da5-45a3-b7c4-720fbea066cd','6b519784-8b0b-4294-9f5d-1d42c276b8a0','36de2c95-55e3-46a8-a843-2cb263b55d0b','a3d3a8ae-6eda-4673-99e7-778e9d382273','6a85d44d-a3b1-4a01-8593-8160dc2c04bf','c401b865-1065-47fd-8ce8-d9f3d1bb3acf','b25fc4f3-6180-4737-9e9c-c3ea4cb96189','e81bbe5f-1930-4693-84a8-aaccccb0cc2b','f7048896-f3dd-4620-bd6b-977dd40a8ce7','1da2d503-54fc-4b89-9002-2ba87cd8b237','9df3dd27-97bc-420c-872b-929bc47bdec5','a10b7e4f-4dff-428c-88e3-2eac6ef474b1','961ef014-8920-459f-8028-11824b7805fa','c0c7248c-fa49-4135-bbe7-1107cf7fa9cf','2f8cd9a4-3d0a-46f1-bb3d-018d7f45556a','6fca3ef6-e6f6-4faa-8a52-b87c07fd9573','9a4ad807-aaaa-49ae-b088-6bb24ab90c0f','b0f1f540-a612-489a-8049-2db93b67a00d','f728bf7c-6188-4dfd-ae05-c3445df58863','ba6ef4c6-0088-4ab9-8b69-f160279e2bbe','1f513032-00c1-4b74-9e61-fca9b369a96e','2b187f0c-70b2-42a9-8358-ddf1faad8bf6','e2071532-6dc1-4991-9bac-8eed63c05424','adf23f76-275a-4945-a186-51625c3d5a22','b74028fb-62ea-454b-b3c2-f5ad273de495','2573c290-75d4-4a21-9e9c-e8e71623b188','571c446d-480d-4aed-a1b8-ab6fbbb1cb99','17404c56-8836-4923-8d58-7218af637527','c5fc0f16-1b55-44c1-a3f3-0a5d554da613','d9ace832-8487-494f-a828-104ff7eaa69c','275eb5d0-23f6-407c-9b5b-013160c3595d','3e37383b-737a-4247-b938-a5d93cb24e92','e0d2f0ee-bc56-46ab-8076-2227ec6279be','580e31ba-2116-43dc-af84-c1fc9cb93d18','541f4b54-8bd6-47fa-a320-64d6ebad4906','38064ea4-4a1f-4af8-ab84-bbcf837eebf7','149e55af-4f39-41b3-8047-cbb472e634ac','b8b43757-f6be-4304-a85a-cd8623cb633b','44c05300-6dc6-490e-ba27-4d01a47e54ce','df2055be-83a8-48bc-a34e-f68bb42c057a','d8a8152b-cc86-459d-b482-a16324b4c9b2','35d6a9b1-c2c8-47c9-a3af-13a5b0ab506e','ec83ece9-ee10-4c77-9967-96fa164c4170','c427e58c-59cd-4924-afdc-2ae2c379644e','74ac1b1a-1982-4cd4-aa2e-6982814fcff3','1129a1fd-5154-49e7-bb47-4a3d34339561','8b2380ef-b224-4ff9-97f1-272c628cd711','36e52f6f-df3c-4eb6-9275-61af42a3d4b5','e1d1aa3b-da0f-48c8-988e-d6ad9a0ff420','9641215f-79e0-4c4d-8f92-4d5e1d5c451f','853e7e57-38ee-49dc-a01b-a356fb5488df','5eae6ad8-37cc-4945-a4cb-949fd6fc9d04','a175f722-a34a-49e9-a3b5-1e9cf08926ff','e51e0e4e-c397-40d0-b55c-ce06f33f0467','4c52e41c-6b77-4ec6-a483-67ef853d0464','ea50dc93-ef1e-4fdb-b142-da97807b1f98','9839c48d-5093-4c87-8f1d-db452fa3a536','0bf42359-2593-4719-8e3c-ed5d343e65c7','6ced0aad-805c-4fab-996b-725f1ab5989f','14552e79-70a2-4791-8188-6a16fc30fef0','c6242655-9972-4cb0-8d8d-10ad8c24d2c3','5127010d-bf88-47df-be4f-b909801f3247','92ff0d86-30af-450d-aee2-994134c7db1c','91442954-80d0-4e36-a2a1-d1b10a901f42','8c2afc5f-025d-4afb-9807-f599d3d6ab0c','7efb49ae-1964-43b2-a528-c19acaeaec61','da68f333-438e-4d5d-b642-a193b304911f','a6566f90-f4d8-4883-8d54-88a5e12cea85','e001beeb-4db3-4fa4-9fb6-280ded6c3a57','14199097-9399-4ed8-82ff-743097dfbfeb','653dca54-7d3a-4544-bf9a-9fb8e98f59af','7b8a46a9-8dc8-4bda-91d2-7d422d6eae53','999112af-986f-4b44-939c-5d35f86a8193','29b808b8-294f-4dda-a90f-bf43decc6286','37db88f2-327f-4be0-814a-3439ba2ad8eb','35cdcdcb-598d-4f37-a34c-05050e0658bc','504eb49e-c395-4fad-8402-aca8317f56a6','5a038e7b-4bf2-4248-9cba-c4d7794cbc06','a5369231-a278-49ad-86ec-e0493c67937f','7e5bbf0d-7f46-4ff3-a477-6fb800751473','de8eb499-403c-45d4-b48d-bca312424b33','551ed644-060f-4529-864a-a25d5b611e44','13117da1-8fc2-463a-a89e-f0ef8a853500','f3c83d78-9255-48be-b6d6-87dce936e592','5007c352-e427-492a-99fe-b829dd3e45da','079f971b-2c9d-436b-8ceb-cc5980b0e618','563ab207-f615-4673-abef-1dbc64e98d24','87678b2a-3aa6-42a1-81ee-11298cc9a4f6','dc265a47-fec6-406d-a8dd-1eb62472ae52','8ad9e255-7329-49f9-b422-26db50bc9f3b','6188d97f-b166-45c7-b6a1-931fb6b573c4','4c047fb2-d9fd-4dee-abb1-01d89c2f606f','39da9dfb-4c43-41c4-b0d4-859d6793a609','4f0bef3f-12e8-4516-9b6a-f879e31cc44d','d413b3e4-0173-481d-b5b7-ccef83da7bbc','7994cc07-b7c4-4aa8-b459-833c6facffa4','72fbfd94-cf4a-4dc8-96ad-b6183bb97a87','9473299d-c62e-4ccc-87a3-2c5255886b8b','95298ae5-7400-46d4-891e-2ad66af7fb8a','274b929d-dd62-493c-81e6-7497d90b513d','c408cb7a-df50-4bdb-985c-02db99a45ec2','cf23fff5-1b32-46f9-89b5-882d57189c73','c3425d86-7c61-4892-b989-f2811782be48','e4a23f98-ba3b-484e-b2ee-269e656de359','e9062548-4137-47e3-8b89-d814ec5d0334','6e876e63-4b2f-4e52-b8b5-f26ab22a8323','8ff241c8-1fac-4f77-882d-59090b60097f','ccd27d45-50d7-427c-8578-37fd59006d9d','2976f6ab-1be2-45e7-96f7-cc9709a329b0','5697a3c8-9fc0-4553-a237-b3c39ec75224','46db9fc1-87f3-47e1-8a90-943a0eca3daa','99d8472d-72ab-46b1-a368-cbd4427c40f2','ff89c0ed-5ac2-43e9-a168-820b60c439b9','847875ab-81bf-4763-ba8a-87bfa5a64775','e233b164-e018-4e98-9766-0d38404a8ec0','96733ae1-58e9-4a22-922b-68aa10438a6b','b97493c0-7cae-4f5b-b29d-41962b02e4cc','5e0c45f3-d005-49bc-9d6d-a12f4c571d93','8856ff43-799e-42c4-970d-1cd4a5d9aa3f','e21d4fe7-2d2a-4fde-9dcc-9fbf4c04be51','08d6a2b4-b54c-4e8f-a840-892d5cc4d2c8','2f3fde35-7573-4138-bef4-09562751897b','a36ae477-796c-4add-b30a-668dbde3f810','bf79dea1-900c-4554-bb10-e73cc9c3528d','ec4a2843-a5e6-4133-a988-06e7ff92ba99','4bd1023a-0596-47e5-9d4d-38b9f436a0d5','17c2a971-abbb-40c7-a9a2-513d3d75b821','7117588f-af17-4786-8f82-6564e634a9df','d78f4e15-3bbc-46ee-a2d6-e81f8f46f41a','428e5213-b44d-4699-9372-5c22c7ff5bbb','f2d8913a-e717-4ed5-8853-3a7aa2cb28e8','bd73d8b7-4d98-4b48-acbb-5be699dc9d13','2cbee09d-9a10-4ce2-8151-768e803f82f0','d2ddc9e2-bf0b-4afa-aa61-07f10a12b5f1','830bbf20-913c-4c22-96bd-760440a9abaa','6f283477-35f1-4ce3-a83e-8bbe8d8fc5f4','7eb9a764-f1d2-4715-ae6c-8b086b436cd2','833ef0e2-b142-4b16-8050-0c38a79efe90','054b69f8-404d-485d-9b9b-98609fd86853','29c53283-ef48-46c1-af56-3b2d0660dc3d','180df77b-4eaf-467b-a8a0-07eb821f68fe','adf2f597-67ae-4e4c-99ca-2e44db755886','c7b591f6-e850-4467-8975-c2a421d7d7b0','5c50e581-a3be-4ad9-a4b9-fc997ed5353b','d63f772c-cf04-4395-a81e-c1e41d28cdfc','4b5c9011-68ff-44a6-bc8a-3c60b48046bb','0c8ef0ac-7259-43ef-bc75-b6db880aaaa4','004a831f-b178-4b49-9c1a-d250b6013bbb','b1946df3-1e29-4943-ae02-d623f1312ffe','3c0a9cf6-2439-49a1-8b50-2f47b8a12576','c239f848-5eaa-4f41-9bfd-7b30b9b4c4d5','3bf3484e-e9fe-4e5a-96ab-46f257e5ea02','b3054418-a385-47eb-af63-1681f40fdeb8','1a7e0bc5-3e25-4e7c-877b-65d6900529f6','6b9239a0-cb2b-42cf-90e4-6e63c9b8e6ba','28820692-060c-4ae1-968d-0617c987eaaf','30ecb0d8-301d-407f-93db-1e5107cc9132','1cdb0a37-889f-4c80-b726-4b33f229fc28','1b0d8bf0-5679-40d4-a6e0-4ef8dc5ab3ff','cf8878f3-94fc-4c92-8fb2-ef8967f780cc','a356ae93-471e-4593-b0da-a3b8d69b6838','66d3a31e-510b-4b90-aee1-fc64c451ddea','c74be10a-0297-42bc-8311-614cbfe6cd2a','b5f12523-20d4-45cf-9f4c-1b7bf75afca2','5d681581-f3ef-43ec-9a77-774baafb725f','18b9f9c2-738b-4646-93cd-2f1359aad014']
try: repair_task_files(task_id)
future.result() \ No newline at end of file
except Exception as e:
logger.exception(f"platform_process 执行异常: {e}")
\ No newline at end of file
aidso_geo/models/process.py
View file @ bcf81c1d
...@@ -1466,6 +1466,7 @@ def result_v2(response_content, data): ...@@ -1466,6 +1466,7 @@ def result_v2(response_content, data):
# ---------------------------------产品词----------------------------- # ---------------------------------产品词-----------------------------
if product_map: if product_map:
# 获取所有产品词 # 获取所有产品词
product_list = extract_keywords(product_map) product_list = extract_keywords(product_map)
...@@ -2018,7 +2019,7 @@ if __name__ == '__main__': ...@@ -2018,7 +2019,7 @@ if __name__ == '__main__':
# data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc") # data_list = bh_utils.query_data("select * from geo_commit_task where platform = 'DP' and thinking_enabled = 1 and insertime > 1777564800 and type!='success' order by insertime asc")
data_list = bh_utils.query_data("select * from geo_commit_task where pt = 20260806 and status = 'ING' and platform !='DB'") data_list = bh_utils.query_data("select * from geo_commit_task where reqId in ('fd51b424-e5d0-4423-837b-e56fe8d4fa0f')")
# data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'") # data_list = bh_utils.query_data("select * from geo_commit_task where prompt = '飞鹤和君乐宝奶粉的异同点对比' and platform = 'DB'")
# data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ") # data_list = bh_utils.query_data("select * from geo_commit_task where pt = '20260720' and platform = 'TYQW' and type = 'success' ")
...@@ -2031,7 +2032,7 @@ if __name__ == '__main__': ...@@ -2031,7 +2032,7 @@ if __name__ == '__main__':
i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), []) i["productWordsMap"] = safe_json_loads(i.get("productWordsMap"), [])
type_t = i.get('type') type_t = i.get('type')
type_t = 'stream_batch' type_t = 'stream_batch'
# commit_task(i,'ING') commit_task(i,'ING')
return task_send_queue(i,type_t) return task_send_queue(i,type_t)
# return deepseek_data_process.deepseek_process_original_data(i) # return deepseek_data_process.deepseek_process_original_data(i)
# return platform_process(i) # return platform_process(i)
......
aidso_geo/models/yuanbao_android_data_process.py
View file @ bcf81c1d
...@@ -140,19 +140,21 @@ def yuanbao_android_process_original_data(task_data): ...@@ -140,19 +140,21 @@ def yuanbao_android_process_original_data(task_data):
} }
jd_goods.append(goods_data) jd_goods.append(goods_data)
if json_data.get('replace').get('display') == 'surroundingsV3': if json_data.get('replace').get('display') == 'surroundingsV3':
for media in replace_data.get("multimedias") or []: for media in replace_data.get("multimedias") or []:
if not isinstance(media, dict): if not isinstance(media, dict):
continue continue
tx_data = { if media.get('poiId'):
"title":media.get('title'), tx_data = {
"address":media.get('address'), "title":media.get('title'),
"jumpUrl":media.get('jumpUrl'), "address":media.get('address'),
"poiId":media.get('poiId'), "jumpUrl":media.get('jumpUrl'),
"distance":media.get('distance'), "poiId":media.get('poiId'),
"avgPrice":media.get('avgPrice'), "distance":media.get('distance'),
"starLevel":media.get('starLevel') "avgPrice":media.get('avgPrice'),
} "starLevel":media.get('starLevel')
tx_goods.append(tx_data) }
tx_goods.append(tx_data)
if jd_goods: if jd_goods:
...@@ -199,7 +201,7 @@ if __name__ == '__main__': ...@@ -199,7 +201,7 @@ if __name__ == '__main__':
# yuanbao_android_process_original_data(file_path2) # yuanbao_android_process_original_data(file_path2)
data_list = bh_utils.query_data( data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '0f934ad9-c45e-43eb-93e4-9a0fa5dce5a0' and platform = 'TXYBA'") f"select * from geo_commit_task where taskId = '452cf30b-e8a1-4d94-a495-abb30affc85c' and platform = 'TXYBA'")
# # # # # #
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment