百看更新

aidso_geo/models/baidu_baikan_data_process.py

+import json
+import traceback
+
+from aidso_geo.models import spider_save_tos
+from aidso_geo.utils import robot_utils, bh_utils
+from aidso_geo.utils.ai_interface import get_parse_sse_result
+from aidso_geo.utils.tos_utils import get_string_from_tos
+
+
+def baidu_baikan_data_process_original_data(data):
+    file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
+    """处理文件内容，提取思考内容、响应内容和URL列表"""
+    url_list =[]
+    think_content = ""
+    response_content = ""
+    response_process = ""
+    search_keyword = []
+    is_think = False
+    think_bool = False
+    suggestions = []
+    response_bool = False
+
+    try:
+        original_content = get_string_from_tos(file_path)
+        try:
+            json_content = json.loads(original_content)
+            response_content = json_content.get('copyContent','')
+            citation_list_data = json_content.get('citationListData','')
+            err_code = json_content.get('there_is_no_answer_to_this_prompt','')
+            if err_code =='99':
+                spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
+                                                       suggestions)
+                response_content = "There is no answer to this prompt"
+                return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+
+            if citation_list_data:
+                if citation_list_data.get('type') =='component':
+                    citation_list_data_data = citation_list_data.get('data',{})
+                    if citation_list_data_data:
+                        citation_list_data_data_list = citation_list_data_data.get('referenceList')
+                        if citation_list_data_data_list:
+                            url_list= citation_list_data_data_list
+        except Exception as e:
+            content_list = original_content.split("\n")
+
+            for i in content_list:
+
+                if i.startswith("data:"):
+                        # 提取并解析JSON数据
+                    data_str = i.split("data:")[1]
+
+                    json_data = json.loads(data_str)
+                    data = json_data.get('data',{})
+                    data_message = data.get('message',{})
+                    data_message_content = data_message.get('content',{})
+                    data_message_content_generator = data_message_content.get('generator',{})
+                    data_message_content_generator_component = data_message_content_generator.get('component',{})
+                    data_message_content_generator_data = data_message_content_generator.get('data',{})
+                    if data_message_content_generator_component == 'searchingSteps':
+                        if data_message_content_generator_data:
+                            data_message_content_generator_data_steps = data_message_content_generator_data.get('steps')
+                            if isinstance(data_message_content_generator_data_steps,list):
+                                result = next((item["words"] for item in data_message_content_generator_data_steps if "words" in item), [])
+                                if result:
+                                    search_keyword.extend(result)
+                    elif data_message_content_generator_component == 'markdown-yiyan':
+                        data_message_content_generator_data_value = data_message_content_generator_data.get('value')
+                        if isinstance(data_message_content_generator_data_value,str):
+                            response_content+=data_message_content_generator_data_value
+                    elif data_message_content_generator_component == 'reference':
+                        data_message_content_generator_data_reference_list = data_message_content_generator_data.get('referenceList')
+                        if isinstance(data_message_content_generator_data_reference_list,list):
+                            url_list = data_message_content_generator_data_reference_list
+        spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
+                                               suggestions)
+        return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+    except Exception as e:
+
+        parts = file_path.split('/')
+        platform = parts[2]
+        task_id = parts[1]
+        context, quote, suggestion, think, search_word = get_parse_sse_result(platform, task_id)
+        if context:
+            response_content = context
+            url_list = quote
+            suggestions = suggestion
+            think_content = think
+            search_keyword = search_word
+            spider_save_tos.process_and_save_files_ai(file_path, search_keyword, url_list, think_content,
+                                                      response_content,suggestions)
+
+            return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+        else:
+            response_content = "对话信息获取失败:-200"
+            robot_utils.feishu_tobot(file_path)
+
+
+    spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
+                                           suggestions)
+    return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
+
+if __name__ == '__main__':
+    # file_path2 = 'geo/7ac6f0c7-988d-4c3e-a45b-f317e144b68e/XHSA/original.text'
+    # xiaohongshu_android_process_original_data(file_path2)
+
+
+    # data_list = bh_utils.query_data(f"select * from geo_commit_task  where platform  = 'BK' and taskId = '21fd537f-131b-4wwwww69a-9d5c-6ea00da2c0d6'")
+    data_list = bh_utils.query_data(f"select * from geo_commit_task  where platform  = 'BK' and taskId = '21sssfd537f-131b-4wwwww69a-9dsss5c-6ea00da2c0d6'")
+
+    # # #
+    # # #
+    # # # # #
+    def handle_item(i):
+        if i.get('comWordsMap'):
+            i['comWordsMap'] = json.loads(i.get('comWordsMap'))
+        if i.get('brandWords'):
+            i['brandWords'] = json.loads(i.get('brandWords'))
+        if i.get('comWords'):
+            i['comWords'] = json.loads(i.get('comWords'))
+        if i.get('keywords'):
+            i['keywords'] = json.loads(i.get('keywords'))
+        if i.get('productWordsMap'):
+            i['productWordsMap'] = json.loads(i.get('productWordsMap'))
+        return baidu_baikan_data_process_original_data(i)
+
+    if data_list:
+        for i in data_list:
+            try:
+                handle_item(i)
+            except Exception as e:
+                ...

aidso_geo/models/process.py

@@ -26,7 +26,7 @@ from aidso_geo.utils.ai_utils import ai_get_brand_sentiment_and_mentions, \
 from aidso_geo.models import doubao_data_process, deepseek_data_process, yuanbao_data_process, qianwen_data_process, \
     kimi_data_process, wenxin_data_process, baiduai_data_process, doubao_android_data_process, \
     deepseek_android_data_process, douyinai_data_process, qianwen_android_data_process, yuanbao_android_data_process, \
-    xiaohongshu_android_data_process
+    xiaohongshu_android_data_process, baidu_baikan_data_process
 from aidso_geo.utils import ai_utils, spider_interface, ai_interface, url_utils, robot_utils
 from aidso_geo.utils import tos_utils
 from loguru import logger
@@ -1615,6 +1615,7 @@ def platform_process(data):
         'TYQWA': qianwen_android_data_process.qianwen_android_process_original_data,
         'TXYBA': yuanbao_android_data_process.yuanbao_android_process_original_data,
         'XHSA': xiaohongshu_android_data_process.xiaohongshu_android_process_original_data,
+        'BK': baidu_baikan_data_process.baidu_baikan_data_process_original_data,
     }
 
     try:

aidso_geo/models/spider_save_tos.py

@@ -399,6 +399,22 @@ def qianwen_process_quote(url_list):
         quto_list.append(raw_data)
     return quto_list
 
+def baikan_process_quote(url_list):
+    quto_list = []
+
+    for index, item in enumerate(url_list):
+        raw_data = {
+            "url": item.get('url', ''),
+            "title": item.get('text', ''),
+            "snippet": item.get('abstract', ''),
+            "index": item.get('url_no', ''),
+            "published_at": item.get('publish_time', ''),
+            "site_name": item.get('source', ''),
+            "site_icon": item.get('icon', ''),
+        }
+        quto_list.append(raw_data)
+    return quto_list
+
 
 def kimi_process_quote(url_list):
     quto_list = []
@@ -491,7 +507,8 @@ def save_data_to_tos(target_dir, content, file_name):
             content = yuanbao_android_process_quote(content)
         elif platform == 'XHSA':
             content = xiaohongshu_android_process_quote(content)
-
+        elif platform == 'BK':
+            content = baikan_process_quote(content)
         else:
             content = doubao_process_quote(content)
         task_id = target_dir.split('/')[1]