Merge remote-tracking branch 'origin/master'

aidso_geo/core/feishu_snipaste.py

@@ -1080,9 +1080,9 @@ if __name__ == "__main__":
 
     t1 = threading.Thread(target=scheduler_thread, daemon=True)
     t2 = threading.Thread(target=runner_thread, daemon=True)
-    # t1.start()
-    # t2.start()
-    # t1.join()
-    # t2.join()
-    redis_client = init_redis4()
-    print(redis_client.delete('mt_third_task'))
+    t1.start()
+    t2.start()
+    t1.join()
+    t2.join()
+    # redis_client = init_redis4()
+    # print(redis_client.delete('mt_third_task'))
\ No newline at end of file

aidso_geo/models/doubao_android_data_process.py

@@ -300,36 +300,31 @@ def doubao_mobile_process_original_data(data):
         url_list = new_url_list
 
 
-        print(suggestions)
-        print(search_keyword)
-        print(response_content)
-        print(rich_media_block)
-        print(len(url_list))
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions, rich_media_block)
         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
     except Exception as e:
-        traceback.print_exc()
-    #     parts = file_path.split('/')
-    #     platform = parts[2]
-    #     task_id = parts[1]
-    #     context, quote, suggestion, think, search_word = get_parse_sse_result(platform, task_id)
-    #     if context:
-    #         response_content = context
-    #         url_list = quote
-    #         suggestions = suggestion
-    #         think_content = think
-    #         search_keyword = search_word
-    #         spider_save_tos.process_and_save_files_ai(file_path, search_keyword, url_list, think_content,
-    #                                                   response_content,suggestions)
-    #
-    #         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
-    #     else:
-    #         response_content = "对话信息获取失败:-200"
-    #         robot_utils.feishu_tobot(file_path)
-    # spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
-    #                                        suggestions, rich_media_block)
-    # return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+        # traceback.print_exc()
+        parts = file_path.split('/')
+        platform = parts[2]
+        task_id = parts[1]
+        context, quote, suggestion, think, search_word = get_parse_sse_result(platform, task_id)
+        if context:
+            response_content = context
+            url_list = quote
+            suggestions = suggestion
+            think_content = think
+            search_keyword = search_word
+            spider_save_tos.process_and_save_files_ai(file_path, search_keyword, url_list, think_content,
+                                                      response_content,suggestions)
+
+            return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+        else:
+            response_content = "对话信息获取失败:-200"
+            robot_utils.feishu_tobot(file_path)
+    spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
+                                           suggestions, rich_media_block)
+    return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
 
 
 if __name__ == '__main__':

aidso_geo/models/process.py

@@ -1286,7 +1286,6 @@ def result_v2(response_content, data):
     # -------------------------
     # 获取ai提及词
     ai_word_list = cache_get_ai_brand_list(taskId, platform, response_content, prompt)
-    print(ai_word_list)
     # 获取所有词
     all_word_list = []
     if isinstance(ai_word_list, list):
@@ -1311,16 +1310,9 @@ def result_v2(response_content, data):
     # 获取所有词的排名 word + rank_list
     all_word_rank_list = get_keyword_ranks(response_content, all_word_set_list)
     # 获取所有词的排名 word + rank + count
-    print(all_word_rank_list)
-    print('----')
-    print('----')
-    print('----')
+
     all_keyword_with_rank = convert_rank_data(all_word_rank_list)
-    print(all_keyword_with_rank)
-    print('---')
-    print('---')
-    print('---')
-    print('---')
+
     # 获取所有词的品牌
     all_keyword_with_brand = keyword_map_brand(all_word_list)
 
@@ -1916,41 +1908,41 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
 
 if __name__ == '__main__':
 
-    data_list = bh_utils.query_data(f"select * from geo_commit_task  where reqId = '84db7b12-88e7-4aaa-bc58-8d61f1051934'")
-    # data_list = bh_utils.query_data(query_sql)
-    # print(data_list)
-    # # #
-    # # #
-    # # # # #
-    def handle_item(i):
-        if i.get('comWordsMap'):
-            i['comWordsMap'] = json.loads(i.get('comWordsMap'))
-        if i.get('brandWords'):
-            i['brandWords'] = json.loads(i.get('brandWords'))
-        if i.get('comWords'):
-            i['comWords'] = json.loads(i.get('comWords'))
-        if i.get('keywords'):
-            i['keywords'] = json.loads(i.get('keywords'))
-        if i.get('productWordsMap'):
-            i['productWordsMap'] = json.loads(i.get('productWordsMap'))
-        type_t = i.get('type')
-        # type_t = 'batch'
-
-        # return task_send_queue(i,type_t)
-        return platform_process(i)
-
-    if data_list:
-        with ThreadPoolExecutor(max_workers=30) as executor:
-            futures = [executor.submit(handle_item, i) for i in data_list]
-
-            for future in as_completed(futures):
-                try:
-                    future.result()
-                except Exception as e:
-                    logger.exception(f"platform_process 执行异常: {e}")
-    # PAGE_SIZE =1000
-    # MAX_WORKERS =50
-    # run_data(PAGE_SIZE,MAX_WORKERS)
+    # data_list = bh_utils.query_data(f"select * from geo_commit_task  where reqId = '84db7b12-88e7-4aaa-bc58-8d61f1051934'")
+    # # data_list = bh_utils.query_data(query_sql)
+    # # print(data_list)
+    # # # #
+    # # # #
+    # # # # # #
+    # def handle_item(i):
+    #     if i.get('comWordsMap'):
+    #         i['comWordsMap'] = json.loads(i.get('comWordsMap'))
+    #     if i.get('brandWords'):
+    #         i['brandWords'] = json.loads(i.get('brandWords'))
+    #     if i.get('comWords'):
+    #         i['comWords'] = json.loads(i.get('comWords'))
+    #     if i.get('keywords'):
+    #         i['keywords'] = json.loads(i.get('keywords'))
+    #     if i.get('productWordsMap'):
+    #         i['productWordsMap'] = json.loads(i.get('productWordsMap'))
+    #     type_t = i.get('type')
+    #     # type_t = 'batch'
+    #
+    #     # return task_send_queue(i,type_t)
+    #     return platform_process(i)
+    #
+    # if data_list:
+    #     with ThreadPoolExecutor(max_workers=30) as executor:
+    #         futures = [executor.submit(handle_item, i) for i in data_list]
+    #
+    #         for future in as_completed(futures):
+    #             try:
+    #                 future.result()
+    #             except Exception as e:
+    #                 logger.exception(f"platform_process 执行异常: {e}")
+    PAGE_SIZE =1000
+    MAX_WORKERS =50
+    run_data(PAGE_SIZE,MAX_WORKERS)
     text = """
     
     """

aidso_geo/models/qianwen_android_data_process.py

@@ -91,12 +91,7 @@ def qianwen_android_process_original_data(data):
                             if mime_type == 'multi_load/iframe' and multi_load_type == 'taoassistant_fold_product_feeds' and status == 'complete':
 
                                 for mu in multi_load:
-                                    print(mu)
-                                    print('---')
-                                    print('---')
-                                    print('---')
-                                    print('---')
-                                    print('---')
+
                                     if mu.get('type') == 'taoassistant_fold_product_feeds':
                                         source_seq = mu.get('source_seq')
                                         jump_url = next(
@@ -213,7 +208,6 @@ def qianwen_android_process_original_data(data):
         if url_list_batch:
             url_list = url_list_batch
 
-        print(response_content)
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions, rich_media_block)
         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)