修复

aidso_geo/models/doubao_android_data_process.py

@@ -2,7 +2,7 @@ import json
 import traceback
 
 from aidso_geo.models import spider_save_tos
-from aidso_geo.utils import robot_utils
+from aidso_geo.utils import robot_utils, bh_utils
 from aidso_geo.utils.ai_interface import get_parse_sse_result
 from aidso_geo.utils.tos_utils import get_string_from_tos
 
@@ -299,12 +299,10 @@ def doubao_mobile_process_original_data(data):
 
         url_list = new_url_list
 
-
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions, rich_media_block)
         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
     except Exception as e:
-        # traceback.print_exc()
         parts = file_path.split('/')
         platform = parts[2]
         task_id = parts[1]
@@ -332,12 +330,34 @@ if __name__ == '__main__':
     # poi 21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6 0提及
     # 电商+poi 21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6 0提及
 
-    aa = ['21fd5xxx37f-131b-4ss69axxxxx-9d5c-6ea00da2c0d6']
+    # aa = ['667b7528-5ff4-4ccb-bb98-7d0b5972014d']
     # file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
 
-    for a in aa:
-        file_path = f'geo/{a}/DOUBA/original.text'
-        doubao_mobile_process_original_data(file_path)
-        # print(file_path)
-        # file_path = 'geo/da4ee3sssa2-4c8e-ccc4776-aaaaaaa587-40asas7543ssssssaf6b9d//original.text'
-        # file_path = 'geo/da4ee3sssa2aaaa-4c8e-ccc4776-aaaaaaa587-40asas7543ssssssaf6b9d/DOUBA/original.text'
+    data_list = bh_utils.query_data(
+        f"select * from geo_commit_task  where taskId = '48d5b7e0-49e8-491b-8719-bcb82daee835' and  platform  = 'DOUBA'")
+
+
+
+    # # #
+    # # # # #
+    def handle_item(i):
+        if i.get('comWordsMap'):
+            i['comWordsMap'] = json.loads(i.get('comWordsMap'))
+        if i.get('brandWords'):
+            i['brandWords'] = json.loads(i.get('brandWords'))
+        if i.get('comWords'):
+            i['comWords'] = json.loads(i.get('comWords'))
+        if i.get('keywords'):
+            i['keywords'] = json.loads(i.get('keywords'))
+        if i.get('productWordsMap'):
+            i['productWordsMap'] = json.loads(i.get('productWordsMap'))
+
+        return doubao_mobile_process_original_data(i)
+
+
+    if data_list:
+        for i in data_list:
+            try:
+                handle_item(i)
+            except Exception as e:
+                ...

aidso_geo/models/qianwen_data_process.py

 import json
 import traceback
 
-from aidso_geo.utils import robot_utils
+from aidso_geo.utils import robot_utils, bh_utils
 from aidso_geo.utils.ai_interface import get_parse_sse_result
 from aidso_geo.utils.tos_utils import get_string_from_tos
 from aidso_geo.models import spider_save_tos
@@ -32,7 +32,6 @@ def qianwen_process_original_data(data):
                     data_str = i.split("data:")[1]
 
                     json_data = json.loads(data_str)
-
                 except (IndexError, json.JSONDecodeError):
 
                     continue
@@ -63,7 +62,6 @@ def qianwen_process_original_data(data):
                                         if cn.get('contentType') == 'text':
                                             response_content = cn.get('content')
                     if json_data.get("data"):
-
                         if json_data.get("data").get('status'):
 
                             if json_data.get("data").get('status') == 'complete':
@@ -76,10 +74,11 @@ def qianwen_process_original_data(data):
                                     if cn.get('mime_type') == 'multi_load/iframe' and cn.get('status') == 'complete':
                                         response_content = cn.get('content')
                                         response_content = response_content.replace("[(deep_think)]", "")
+                                        response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
                                         multi_load = cn.get('meta_data').get('multi_load')
                                         if multi_load:
                                             for mu in multi_load:
-                                                if mu.get('type') == 'deep_think':
+                                                if mu.get('type') == 'deep_think' or mu.get('type') == 'multimodal_chat_think':
                                                     if mu.get('content').get('status') =='complete':
                                                         think_content = mu.get('content').get('think_content')
 
@@ -88,15 +87,26 @@ def qianwen_process_original_data(data):
                             for ms in messages:
                                 if ms.get('mime_type') =='multi_load/iframe' and ms.get('status') == 'complete':
                                     response_content = ms.get('content')
+
                                     response_content = response_content.replace("[(deep_think)]", "")
+                                    response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
                                     multi_load = ms.get('meta_data').get('multi_load')
                                     if multi_load:
                                         for mu in multi_load:
-                                            if mu.get('type') == 'deep_think':
+                                            if mu.get('type') == 'deep_think' or mu.get('type') == 'multimodal_chat_think':
                                                 if mu.get('content').get('status') == 'complete':
                                                     think_content = mu.get('content').get('think_content')
+                                            if mu.get('type') == 'ref_source_inline':
+                                                mu_content = mu.get('content')
+                                                if mu_content:
+                                                    if mu_content.get('query_list'):
+                                                        search_keyword.extend(mu_content.get('query_list'))
+                                                    if mu_content.get('list'):
+
+                                                        url_list_batch.extend(mu_content.get('list'))
 
                                 if ms.get('mime_type') =='bar/iframe' and ms.get('status') == 'complete':
+                                    if  ms.get('meta_data').get('sources'):
                                         url_list_batch = ms.get('meta_data').get('sources')[0].get('content').get('list')
                                 if ms.get('mime_type') =='paa/iframe' and ms.get('status') == 'complete':
                                     paas = ms.get('meta_data').get('paas')
@@ -116,6 +126,7 @@ def qianwen_process_original_data(data):
                         "time": url.get('publish_time', ''),
                     }
                 )
+
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions)
         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
@@ -141,17 +152,42 @@ def qianwen_process_original_data(data):
     spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                            suggestions)
     return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
-
+#
 
 if __name__ == '__main__':
 
-    file_path2 = 'geo/21fd5xxx37f-131b-469axxxxx-9d5c-6ea00da2c0d6/TYQW/original.text'
-    # file_path2 = 'geo/jqk/TYQW/2.txt'
-    # /geo/900f0ea6e9a34c95b6b57aa7519a4820/
-    # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
-    qianwen_process_original_data(file_path2)
+    # file_path2 = 'geo/21fd5xxx37f-131b-469axxxxx-9d5c-6ea00da2c0d6/TYQW/original.text'
+    # # file_path2 = 'geo/jqk/TYQW/2.txt'
+    # # /geo/900f0ea6e9a34c95b6b57aa7519a4820/
+    # # file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
+    # qianwen_process_original_data(file_path2)
     # for i in task_id:
-
+    data_list = bh_utils.query_data(f"select * from geo_commit_task  where taskId = '16e53643-b1ac-4566-a0aa-68340576a3ff' and  platform  = 'TYQW'")
+
+    # # #
+    # # #
+    # # # # #
+    def handle_item(i):
+        if i.get('comWordsMap'):
+            i['comWordsMap'] = json.loads(i.get('comWordsMap'))
+        if i.get('brandWords'):
+            i['brandWords'] = json.loads(i.get('brandWords'))
+        if i.get('comWords'):
+            i['comWords'] = json.loads(i.get('comWords'))
+        if i.get('keywords'):
+            i['keywords'] = json.loads(i.get('keywords'))
+        if i.get('productWordsMap'):
+            i['productWordsMap'] = json.loads(i.get('productWordsMap'))
+
+        return qianwen_process_original_data(i)
+
+
+    if data_list:
+        for i in data_list:
+            try:
+                handle_item(i)
+            except Exception as e:
+                ...
     # plan_cot/post  and status =='complete' 思考过程前
     # bar/progress and type = cot 搜索关键词
     # bar/progress and type = sources 引用来源