Skip to content

A
aidso-data
Commit f4a87988 authored by Yaowentong's avatar Yaowentong
Browse files
Options

千问电商数据修复

parent 7feb7340
Hide whitespace changes
Inline Side-by-side
Showing with 39 additions and 59 deletions
aidso_geo/config/base_config.py
View file @ f4a87988
......@@ -190,6 +190,8 @@ if __name__ == '__main__':
'TYQWA:geo:batch:list',
'WXYY:geo:stream_batch:list',
'WXYY:geo:batch:list',
'XHSA:geo:stream_batch:list',
'XHSA:geo:batch:list',
'geo:task_commit:list']
t = init_redis()
......@@ -213,4 +215,3 @@ if __name__ == '__main__':
aidso_geo/core/routes/dashboard.py
View file @ f4a87988
......@@ -3,7 +3,6 @@ from datetime import datetime
from flask import jsonify, Blueprint, render_template, request
import requests
from aidso_geo.config.base_config import init_redis
from aidso_geo.utils import bh_utils
dashboard_app = Blueprint("dashboard", __name__)
......@@ -16,7 +15,7 @@ def queue_status():
platforms = [
"BDAI", "DB", "DOUBA", "DP", "DPA", "DYAI",
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY"
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY", "XHSA"
]
for platform in platforms:
......@@ -83,56 +82,6 @@ def get_req_trend():
}), 500
@dashboard_app.route("/api/third/usage/latest", methods=["GET"])
def third_usage_latest():
latest_rows = bh_utils.query_data("""
select sync_time
from geo_third_usage_snapshot
where sync_time is not null and sync_time != ''
order by sync_time desc
limit 1
""")
if latest_rows is None:
return jsonify({
"code": 500,
"msg": "query latest sync_time failed",
"data": []
}), 500
if not latest_rows:
return jsonify({
"code": 200,
"msg": "ok",
"sync_time": "",
"data": []
})
sync_time = latest_rows[0].get("sync_time")
data = bh_utils.query_data("""
select channel, pt, daily_limit, total_limit, today_used, total_used,
today_remain, total_remain, sync_time, insertime
from geo_third_usage_snapshot
where sync_time = %s
order by channel asc
""", (sync_time,))
if data is None:
return jsonify({
"code": 500,
"msg": "query third usage snapshot failed",
"sync_time": sync_time,
"data": []
}), 500
return jsonify({
"code": 200,
"msg": "ok",
"sync_time": sync_time,
"data": data
})
def normalize_date_to_datetime(value, default_time):
date_value = value.replace("T", " ").strip()[:10]
datetime.strptime(date_value, "%Y-%m-%d")
......
aidso_geo/core/static/dashboard/queue_monitor.js
View file @ f4a87988
......@@ -2,8 +2,11 @@ const API_URL = "/api/queue/status";
const STATS_API_URL = "/api/queue/platformStats";
const platforms = [
"BDAI", "DB", "DOUBA", "DP", "DPA", "DYAI",
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY"
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY", "XHSA"
];
const PLATFORM_NAME_MAP = {
XHSA: "小红书手机版"
};
let currentPanel = "queue";
let loadedPanels = {queue: false, trend: false};
let trendChartMode = "platform";
......@@ -123,12 +126,16 @@ const API_URL = "/api/queue/status";
return getStatsItems(responseData)
.map(item => ({
time: normalizeStatDate(item["时间"] || item.time || item.date || item.begin || ""),
platform: item["平台"] || item.platform || item.name || "未知平台",
platform: getPlatformDisplayName(item["平台"] || item.platform || item.name || "未知平台"),
count: Number(item["数量"] || item.count || item.value || 0)
}))
.filter(item => item.time && Number.isFinite(item.count));
}
function getPlatformDisplayName(platform) {
return PLATFORM_NAME_MAP[String(platform || "").trim()] || platform;
}
function normalizeStatDate(value) {
return String(value || "").slice(0, 10);
}
......
aidso_geo/models/doubao_android_data_process.py
View file @ f4a87988
......@@ -34,8 +34,6 @@ def doubao_mobile_process_original_data(data):
try:
json_content = json.loads(payload)
except (IndexError, json.JSONDecodeError):
continue
......@@ -330,7 +328,7 @@ if __name__ == '__main__':
# file_path = 'geo/c7eb465e-f385-4aa2-89c4-a7cf11897f45/KIMI/1(1).txt'
data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '8f605073-841a-4e8f-ac4b-1a686d5e9c4f' and platform = 'DOUBA'")
f"select * from geo_commit_task where taskId = '27721b42-3368-4526-b092-d3157aaa617a' and platform = 'DOUBA'")
......
aidso_geo/models/qianwen_android_data_process.py
View file @ f4a87988
......@@ -155,7 +155,7 @@ def qianwen_android_process_original_data(data):
if mime_type == 'multi_load/iframe' and (multi_load_type == 'taoassistant_fold_product_feeds' or multi_load_type == 'taoassistant_single_product') and status == 'complete':
for mu in multi_load:
if mu.get('type') in ('taoassistant_fold_product_feeds' ,'taoassistant_single_product'):
if mu.get('type') == 'taoassistant_fold_product_feeds' :
source_seq = mu.get('source_seq')
jump_url = next(
(
......@@ -202,6 +202,31 @@ def qianwen_android_process_original_data(data):
"source_seq": source_seq,
"auctionURL": pro.get('auctionURL'),
})
if mu.get('type') == 'taoassistant_single_product':
mu_content = mu.get('content',{})
mu_content_cardData = mu_content.get('cardData',{})
mu_content_cardData_data = mu_content_cardData.get('data',{})
if mu_content_cardData_data:
source_seq = mu_content.get('cardId') or mu_content.get('businessId')
price = (mu_content_cardData_data.get('priceShowWithIcon') or {}).get('price') or mu_content_cardData_data.get(
'itemPrice') or ''
taobao_list.append({
"title": mu_content_cardData_data.get('title'),
"shop_name": next(
(
item.get('text', '')
for item in
mu_content_cardData_data.get('structuredShopInfo', {}).get('infoList', [])
if item.get('sourceType') == 'shop_name'
),
''
),
"pic_path": mu_content_cardData_data.get('pic_path', ''),
"price": price,
"jump_url": mu_content_cardData_data.get('auctionURL'),
"source_seq": source_seq,
"auctionURL": mu_content_cardData_data.get('auctionURL'),
})
if taobao_list:
response_content += 'render_ecom_card_widget_taobao_start:'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment