电商数据

百看平台上线

aidso_geo/config/base_config.py

@@ -70,6 +70,7 @@ class PlatformType(Enum):
     YUANBAOANDROID = "TXYBA"
     XIAOHONGSHUANDROID = "XHSA"
     BAIDUBAIKAN = "BK"
+    BAIDUWENXIN = "BDWX"
 
 
     @classmethod
@@ -138,6 +139,10 @@ class BaseConfig:
         PlatformType.BAIDUBAIKAN.value: {
             "url": f"{base_url}baikan",
             "storage_path": lambda tid: f"geo/{tid}/BK/original.text"
+        },
+        PlatformType.BAIDUWENXIN.value: {
+            "url": f"{base_url}baiduai",
+            "storage_path": lambda tid: f"geo/{tid}/BDWX/original.text"
         }
     }
 
@@ -199,12 +204,25 @@ if __name__ == '__main__':
 'XHSA:geo:stream_batch:list',
 'XHSA:geo:batch:list',
 'geo:task_commit:list']
-    t = init_redis()
+
+
+    def init_redis4():
+        try:
+            redis_client = redis.Redis(
+                host="172.16.0.24",
+                port=6379,
+                db=4,
+                password="aiyingli@@123",
+                socket_timeout=5,
+                decode_responses=True
+            )
+            return redis_client
+        except Exception as e:
+            return None
 
 
     # aa = list(aa)
 
-    t.delete('mt_third_task')
     # for a in aa:
     #     sec_uid = get_sec_id(a)
     #     get_user_info(sec_uid)

aidso_geo/core/feishu_snipaste.py

@@ -14,10 +14,7 @@ from openpyxl.workbook import Workbook
 import os ,sys
 from loguru import logger
 
-from aidso_geo.utils.tos_utils import put_string_to_tos
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(BASE_DIR)
 from aidso_geo.models import spider_save_tos
 from aidso_geo.utils import bh_utils, tos_utils
 
@@ -306,7 +303,6 @@ def doubao_process_original_data(file_path, original_content):
     response_bool = False
 
     file_path_result = os.path.dirname(file_path)
-    print(file_path_result)
     content_list = original_content.split("\n")
     for i in content_list:
 
@@ -332,9 +328,9 @@ def doubao_process_original_data(file_path, original_content):
                     inner_user_ip = extra_json.get('inner_user_ip')
                     inner_pc_version = extra_json.get('inner_pc_version')
                     if inner_user_ip:
-                        put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
+                        tos_utils.put_string_to_tos(f"{file_path_result}/inner_user_ip.txt", inner_user_ip)
                     if inner_pc_version:
-                        put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
+                        tos_utils.put_string_to_tos(f"{file_path_result}/inner_pc_version.txt", inner_pc_version)
             if json_content.get('event_type') == 2001:
 
                 even_data = json.loads(json_content.get('event_data'))

aidso_geo/core/routes/interface.py

@@ -166,37 +166,6 @@ def task_commit():
                 "status": f"缺少必要字段：{', '.join(missing_fields)}",
                 "reqId": req_id
             })
-        # if type == 'stream':
-        #     insert_ok = commit_task(data, 'ING')
-        #     ok = submit_background_task(main_process, data)
-        #     if insert_ok and ok:
-        #         return jsonify({
-        #             "code": 200,
-        #             "msg": 'success',
-        #             "reqId": req_id
-        #         })
-        # else:
-        #     logger.success(f"{data['reqId']}--{platform}--{prompt}--任务提交--{type}")
-        #     ret = redis_client.lpush("geo:task_commit:list",json.dumps(data, ensure_ascii=False))
-        #     if ret and ret > 0:
-        #         resp_cache_key = f"geo:task_check:resp:{req_id}"
-        #         resp = {
-        #             "code": 200,
-        #             "msg": "success",
-        #             "data": {
-        #                 "status": 'ING',
-        #                 "result": {}
-        #             }
-        #         }
-        #         if type == 'stream_batch':
-        #             _cache_set_json(resp_cache_key, resp, 600)
-        #         else:
-        #             _cache_set_json(resp_cache_key, resp, 18000)
-        #         return jsonify({
-        #             "code": 200,
-        #             "msg": "任务已提交",
-        #             "reqId": req_id
-        #         })
         logger.success(f"{data['reqId']}--{platform}--{prompt}--任务提交--{type}")
         ret = redis_client.lpush("geo:task_commit:list",json.dumps(data, ensure_ascii=False))
         if ret and ret > 0:

aidso_geo/models/baidu_baikan_data_process.py

@@ -28,9 +28,9 @@ def baidu_baikan_data_process_original_data(data):
             citation_list_data = json_content.get('citationListData','')
             err_code = json_content.get('there_is_no_answer_to_this_prompt','')
             if err_code =='99':
+                response_content = "There is no answer to this prompt"
                 spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                        suggestions)
-                response_content = "There is no answer to this prompt"
                 return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
 
             if citation_list_data:

aidso_geo/models/doubao_android_data_process.py

@@ -2,13 +2,14 @@ import json
 import traceback
 
 from aidso_geo.models import spider_save_tos
+from aidso_geo.models.eco_data_process import save_eco_data_to_bh
 from aidso_geo.utils import robot_utils, bh_utils
 from aidso_geo.utils.ai_interface import get_parse_sse_result
 from aidso_geo.utils.tos_utils import get_string_from_tos
 
 
-def doubao_mobile_process_original_data(data):
-    file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
+def doubao_mobile_process_original_data(task_data):
+    file_path = f'geo/{task_data["taskId"]}/{task_data["platform"]}/original.text'
     url_list = []
     think_content = ""
     response_content = ""
@@ -19,6 +20,8 @@ def doubao_mobile_process_original_data(data):
     is_think = False
     think_bool = False
     response_bool = False
+    dy_poi_list =[]
+    dy_eco_list =[]
 
     try:
         original_content = get_string_from_tos(file_path)
@@ -195,15 +198,19 @@ def doubao_mobile_process_original_data(data):
                                                                                         for pr in pro_data:
                                                                                             if isinstance(pr,dict):
 
+                                                                                                pid = pr.get('pid', '')
+                                                                                                url = f'https://haohuo.jinritemai.com/ecommerce/trade/detail/index.html?id={pid}&origin_type=604'
                                                                                                 product = {
                                                                                                     'text': pr.get('text', ''),
                                                                                                     'seller_name': pr.get(
                                                                                                         'seller_name', ''),
                                                                                                     'image_url': pr.get(
                                                                                                         'image_url', ''),
-                                                                                                    'pid': pr.get('pid', '')
+                                                                                                    'pid': pid,
+                                                                                                    'jump_url': url,
                                                                                                 }
                                                                                                 media_list.append(product)
+                                                                                                dy_eco_list.extend(media_list)
                                                                                     poi_data = widget_data_data.get(
                                                                                         'poi_list')
 
@@ -291,7 +298,8 @@ def doubao_mobile_process_original_data(data):
 
             url_list_seen.add(key)
             new_url_list.append(item)
-
+        if dy_eco_list:
+            save_eco_data_to_bh(task_data,'dyeco',dy_eco_list)
         url_list = new_url_list
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions, rich_media_block)
@@ -329,7 +337,7 @@ if __name__ == '__main__':
     # file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
 
     data_list = bh_utils.query_data(
-        f"select * from geo_commit_task  where taskId = '6a5a03b63da04b9d809ea9fd27e266f6' and  platform  = 'DOUBA'")
+        f"select * from geo_commit_task  where taskId = '369df725-ef53-4050-8b94-233a007950c3' and  platform  = 'DOUBA'")
 
 
 

aidso_geo/models/eco_data_process.py

+from aidso_geo.utils import bh_utils
+
+
+def qianwen_android_process_taobao_eco(data,eco_list):
+    eco_result = []
+    task_id = data.get('taskId')
+    platform = data.get('platform')
+    req_id = data.get('reqId')
+    prompt = data.get('prompt')
+    for index, item in enumerate(eco_list):
+        if item.get('card_type') =='single_product':
+            raw_data = {
+                "eco_id": item.get('item_id', ''),
+                "eco_url": item.get('auctionURL', '') or item.get('jump_url',''),
+                "eco_title": item.get('title', ''),
+                "eco_index": index+1,
+                "eco_pic": item.get('pic_path', ''),
+                "eco_price": item.get('price', ''),
+                "eco_source": 'taobao',
+                "platform": platform,
+                "task_id": task_id,
+                "req_id": req_id,
+                "prompt": prompt
+            }
+            eco_result.append(raw_data)
+
+    return eco_result
+
+def qianwen_android_process_damai_eco(data,eco_list):
+    eco_result = []
+    task_id = data.get('taskId')
+    platform = data.get('platform')
+    req_id = data.get('reqId')
+    prompt = data.get('prompt')
+    for index, item in enumerate(eco_list):
+        raw_data = {
+            "eco_id": item.get('id', ''),
+            "eco_url": item.get('webURL', '') ,
+            "eco_title": item.get('name', ''),
+            "eco_index": index+1,
+            "eco_pic": item.get('verticalPic', ''),
+            "eco_price": item.get('priceShowText', ''),
+            "eco_source": 'damai',
+            "platform": platform,
+            "task_id": task_id,
+            "req_id": req_id,
+            "prompt": prompt
+        }
+        eco_result.append(raw_data)
+
+    return eco_result
+
+
+def doubao_android_process_douyin_eco(data,eco_list):
+    eco_result = []
+    task_id = data.get('taskId')
+    platform = data.get('platform')
+    req_id = data.get('reqId')
+    prompt = data.get('prompt')
+    for index, item in enumerate(eco_list):
+        raw_data = {
+            "eco_id": item.get('pid', ''),
+            "eco_url": item.get('jump_url', '') ,
+            "eco_title": item.get('text', ''),
+            "eco_index": index+1,
+            "eco_pic": item.get('image_url', ''),
+            "eco_price": item.get('priceShowText', ''),
+            "eco_source": 'dyeco',
+            "platform": platform,
+            "task_id": task_id,
+            "req_id": req_id,
+            "prompt": prompt
+        }
+        eco_result.append(raw_data)
+
+    return eco_result
+def save_eco_data_to_bh(data,eco_type,eco_list):
+
+    platform = data.get('platform')
+
+    eco_result = []
+    if platform == 'TYQWA':
+        if eco_type == 'taobao':
+            eco_result = qianwen_android_process_taobao_eco(data,eco_list)
+        if eco_type == 'damai':
+            eco_result = qianwen_android_process_damai_eco(data, eco_list)
+    if platform == 'DOUBA':
+        if eco_type == 'dypoi':
+            ...
+        if eco_type == 'dyeco':
+            eco_result =doubao_android_process_douyin_eco(data,eco_list)
+            ...
+    bh_utils.insert_data('geo_eco_data',eco_result)
\ No newline at end of file

aidso_geo/models/process.py

@@ -369,6 +369,7 @@ def filter_ai_word_list(ai_word_list_with_rank, brand_words_rank, com_word_list_
         if item.get("brand")
     }
 
+
     com_words_brand_set = {
         normalize_brand(item.get("brand"))
         for item in com_word_list_rank
@@ -1266,7 +1267,28 @@ def add_keyword_type(ai_word_list, brand_words, com_word_list, all_keyword_with_
 
     return result
 
+def reset_rank(all_vos, brand_vos, competitor_vos):
+    """
+    将三个列表中的 rank 重新编号为连续排名（从 1 开始）。
+    rank=0 的数据不参与排序，保持为 0。
+
+    Returns:
+        all_vos, brand_vos, competitor_vos
+    """
+    # 合并所有对象（引用，不会复制）
+    all_items = all_vos + brand_vos + competitor_vos
+
+    # 参与排序的数据
+    ranked_items = [item for item in all_items if item.get("rank", 0) > 0]
 
+    # 按原 rank 排序
+    ranked_items.sort(key=lambda x: x["rank"])
+
+    # 重新编号
+    for new_rank, item in enumerate(ranked_items, start=1):
+        item["rank"] = new_rank
+
+    return all_vos, brand_vos, competitor_vos
 def result_v2(response_content, data):
     data = process_com_map(data)
     reqId = data.get('reqId')
@@ -1309,13 +1331,13 @@ def result_v2(response_content, data):
     all_word_set_list = [str(x).strip() for x in all_word_set_list if str(x).strip()]
     # 获取所有词的排名 word + rank_list
     all_word_rank_list = get_keyword_ranks(response_content, all_word_set_list)
+
     # 获取所有词的排名 word + rank + count
 
     all_keyword_with_rank = convert_rank_data(all_word_rank_list)
 
     # 获取所有词的品牌
     all_keyword_with_brand = keyword_map_brand(all_word_list)
-
     # 追加所有词的品牌
     all_keyword_with_rank_sentiment_brand = keyword_with_brand(all_keyword_with_rank, all_keyword_with_brand)
 
@@ -1325,6 +1347,7 @@ def result_v2(response_content, data):
                                                                                                   com_word_list,
                                                                                                   all_keyword_with_rank_sentiment_brand)
 
+
     # ---------------------------------调整排名-----------------------------
 
     brand_words_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(brand_words_list_with_rank,
@@ -1342,13 +1365,11 @@ def result_v2(response_content, data):
         # 重新计算外层rank count
         com_word_list_with_rank = rebuild_count_and_rank(com_word_list_with_rank)
 
-
     else:
         # list 调整竞争品牌词排名 去除竞争品牌词中ai的排名
         com_word_list_with_rank, ai_word_list_with_rank = merge_brand_rank_by_subset(com_word_list_with_rank,
                                                                                      ai_word_list_with_rank)
     # ---------------------------------调整排名-----------------------------
-
     # 去除all
     ai_word_list_with_rank = filter_ai_word_list(ai_word_list_with_rank, brand_words_list_with_rank,
                                                  com_word_list_with_rank)
@@ -1384,6 +1405,12 @@ def result_v2(response_content, data):
     com_word_list_with_rank = attach_favorability_brand_score(all_sentiment_and_mentions_score, com_word_list_with_rank)
     ai_word_list_with_rank = attach_favorability_score(all_sentiment_and_mentions_score, ai_word_list_with_rank)
 
+
+
+    ai_word_list_with_rank,brand_words_list_with_rank,com_word_list_with_rank=reset_rank(ai_word_list_with_rank,
+                                                                                         brand_words_list_with_rank,
+                                                                                         com_word_list_with_rank)
+
     mentionsList = build_mentions_list(brand_words_list_with_rank, all_sentiment_and_mentions)
     # ---------------------------------情感倾向-----------------------------
 
@@ -1616,6 +1643,7 @@ def platform_process(data):
         'TXYBA': yuanbao_android_data_process.yuanbao_android_process_original_data,
         'XHSA': xiaohongshu_android_data_process.xiaohongshu_android_process_original_data,
         'BK': baidu_baikan_data_process.baidu_baikan_data_process_original_data,
+        'BDWX': baiduai_data_process.baiduai_process_original_data,
     }
 
     try:
@@ -1924,7 +1952,7 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
 
 if __name__ == '__main__':
 
-    data_list = bh_utils.query_data(f"select * from geo_commit_task where status = 'ING' and type !='stream' ")
+    data_list = bh_utils.query_data(f"select * from geo_commit_task where status = 'ING' ")
     # data_list = bh_utils.query_data(query_sql)
     # print(data_list)
     # # #
@@ -1942,10 +1970,10 @@ if __name__ == '__main__':
         if i.get('productWordsMap'):
             i['productWordsMap'] = json.loads(i.get('productWordsMap'))
         type_t = i.get('type')
-        type_t = 'batch'
+        # type_t = 'batch'
         # commit_task(i,'ING')
-        # return task_send_queue(i,type_t)
-        return platform_process(i)
+        return task_send_queue(i,type_t)
+        # return platform_process(i)
     #
 
     if data_list:

aidso_geo/models/qianwen_android_data_process.py

@@ -4,14 +4,15 @@ from asyncio import as_completed
 from concurrent.futures import ThreadPoolExecutor
 
 from aidso_geo.models import spider_save_tos
+from aidso_geo.models.eco_data_process import save_eco_data_to_bh
 from aidso_geo.utils import robot_utils, bh_utils
 from aidso_geo.utils.ai_interface import get_parse_sse_result
 from aidso_geo.utils.tos_utils import get_string_from_tos
 
 
-def qianwen_android_process_original_data(data):
-# def qianwen_android_process_original_data(file_path):
-    file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
+def qianwen_android_process_original_data(task_data):
+
+    file_path = f'geo/{task_data["taskId"]}/{task_data["platform"]}/original.text'
     url_list = []
     url_list_batch = []
     think_content = ""
@@ -23,12 +24,13 @@ def qianwen_android_process_original_data(data):
     think_bool = False
     response_bool = False
     result = []
+    damai_data_list = []
     try:
         json_original = ""
         original_content = get_string_from_tos(file_path)
         try:
-            json_original = json.loads(original_content)
-            sse_json_data = json.loads(json_original.get('sse'))
+            sse_json_data = json.loads(original_content)
+            # sse_json_data = json.loads(json_original.get('sse'))
             response_content = sse_json_data.get('reply_text','')
             think_content = sse_json_data.get('deepthink_text','')
             paa_answer = sse_json_data.get('paa_answer','')
@@ -156,6 +158,7 @@ def qianwen_android_process_original_data(data):
                                     for mu in multi_load:
 
                                         if mu.get('type') == 'taoassistant_fold_product_feeds' :
+
                                             source_seq = mu.get('source_seq')
                                             jump_url = next(
                                                 (
@@ -181,12 +184,13 @@ def qianwen_android_process_original_data(data):
                                                     query = json.loads(query)
                                             if isinstance(query,list):
                                                 search_keyword.extend(query)
-                                            if isinstance(mu.get('content').get('cardData').get('data').get('items'), list):
-                                                for pro in mu.get('content').get('cardData').get('data').get('items'):
+                                            if isinstance(mu.get('content').get('cardData').get('data').get('items'), list) and not taobao_list  :
 
+                                                cateTitle = mu.get('content').get('cardData').get('data').get('cateTitle')
+                                                for pro in mu.get('content').get('cardData').get('data').get('items'):
                                                     price = (pro.get('priceShowWithIcon') or {}).get('price') or pro.get('itemPrice') or ''
                                                     taobao_list.append({
-                                                        "title": pro.get('title'),
+                                                        "title": cateTitle or pro.get('title', ''),
                                                         "shop_name": next(
                                                             (
                                                                 item.get('text', '')
@@ -201,11 +205,15 @@ def qianwen_android_process_original_data(data):
                                                         "jump_url": jump_url,
                                                         "source_seq": source_seq,
                                                         "auctionURL": pro.get('auctionURL'),
+                                                        "item_id": pro.get('item_id'),
+                                                        'card_type':'fold_product'
                                                     })
+
                                         if mu.get('type') == 'taoassistant_single_product':
                                             mu_content = mu.get('content',{})
                                             mu_content_cardData = mu_content.get('cardData',{})
                                             mu_content_cardData_data = mu_content_cardData.get('data',{})
+
                                             if mu_content_cardData_data:
                                                 source_seq = mu_content.get('cardId') or mu_content.get('businessId')
                                                 price = (mu_content_cardData_data.get('priceShowWithIcon') or {}).get('price') or mu_content_cardData_data.get(
@@ -226,6 +234,8 @@ def qianwen_android_process_original_data(data):
                                                     "jump_url": mu_content_cardData_data.get('auctionURL'),
                                                     "source_seq": source_seq,
                                                     "auctionURL": mu_content_cardData_data.get('auctionURL'),
+                                                    "item_id": mu_content_cardData_data.get('item_id'),
+                                                    'card_type': 'single_product'
                                                 })
 
                                 if taobao_list:
@@ -233,7 +243,7 @@ def qianwen_android_process_original_data(data):
                                     damai_str = json.dumps(taobao_list, ensure_ascii=False)
                                     response_content += damai_str
                                     response_content += 'render_ecom_card_widget_taobao_end:\n'
-
+                                    save_eco_data_to_bh(task_data,'taobao',taobao_list)
                                 damai_list = []
                                 if mime_type == 'multi_load/iframe' and multi_load_type == 'damai_shows_list' and multi_load_status == 'complete':
 
@@ -258,14 +268,16 @@ def qianwen_android_process_original_data(data):
                                                         'maxPrice': damai.get('data').get('maxPrice', ''),
                                                         'priceStr': damai.get('data').get('priceStr', ''),
                                                         'showVenueName': cityName + venueName,
-                                                        'source_seq_id': source_seq
+                                                        'source_seq_id': source_seq,
+                                                        'id': damai.get('data').get('id')
                                                     })
+                                                    damai_data_list.extend(damai_list)
+
                                 if damai_list:
                                     response_content += 'render_ecom_card_widget_damai_start:'
                                     damai_str = json.dumps(damai_list, ensure_ascii=False)
                                     response_content += damai_str
                                     response_content += 'render_ecom_card_widget_damai_end:\n'
-
                                 if mime_type == 'multi_load/iframe' and multi_load_type == 'deep_think' and multi_load_status == 'complete':
                                     think_content += multi_load_content.get('think_content')
                                 # 引用来源 回答 视频列表
@@ -295,10 +307,10 @@ def qianwen_android_process_original_data(data):
                                             suggestions.append(pa.get('show_text'))
             if url_list_batch:
                 url_list = url_list_batch
-
+            if damai_data_list:
+                save_eco_data_to_bh(task_data, 'damai', damai_data_list)
         response_content = response_content.replace("[(deep_think)]", "")
         response_content = response_content.replace("[(multimodal_chat_think_1)]", "")
-
         spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
                                                suggestions, rich_media_block)
         return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
@@ -336,7 +348,7 @@ if __name__ == '__main__':
     # qianwen_android_process_original_data(file_path2)
 
 
-    data_list = bh_utils.query_data(f"select * from geo_commit_task  where taskId = '27721b42-3368-4526-b092-d3157aaa617a' and  platform  = 'TYQWA'")
+    data_list = bh_utils.query_data(f"select * from geo_commit_task  where reqId = '4b285be0-c62c-48f7-9d72-2892511c306c' and  platform  = 'TYQWA'")
 
     def handle_item(i):
         if i.get('comWordsMap'):

aidso_geo/models/spider_save_tos.py

@@ -509,6 +509,8 @@ def save_data_to_tos(target_dir, content, file_name):
             content = xiaohongshu_android_process_quote(content)
         elif platform == 'BK':
             content = baikan_process_quote(content)
+        elif platform == 'BDWX':
+            content = baiduai_process_quote(content)
         else:
             content = doubao_process_quote(content)
         task_id = target_dir.split('/')[1]