Skip to content

A
aidso-data
Commit 27ac512a authored by Yaowentong's avatar Yaowentong
Browse files
Options

后台健全

parent fc2c7163
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 4030 additions and 422 deletions
aidso_geo/core/feishu_snipaste_v3.py
View file @ 27ac512a
This diff is collapsed.
aidso_geo/core/interface.py
View file @ 27ac512a
......@@ -6,6 +6,11 @@ from aidso_geo.core.routes.feishu_interface import feishu_app
from aidso_geo.core.routes.third_interface import third_app
app = Flask(__name__)
app.secret_key = "aidso-dashboard-session-2026-v3-fixed-key"
app.config.update(
SESSION_COOKIE_HTTPONLY=True,
SESSION_COOKIE_SAMESITE="Lax",
)
app.json.ensure_ascii = False
app.register_blueprint(line_app)
......@@ -14,4 +19,4 @@ app.register_blueprint(feishu_app)
app.register_blueprint(dashboard_app)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8086)
\ No newline at end of file
app.run(host='0.0.0.0', port=8086)
aidso_geo/core/meituan_process.py
View file @ 27ac512a
......@@ -16,6 +16,7 @@ KEEP_FIELDS = (
"type",
"thinkingEnabled",
"channel",
"is_share"
)
......@@ -47,12 +48,12 @@ def get_result_api(task_data):
platform = task_data.get("platform",'')
channel = task_data.get("channel") or ""
task_type = task_data.get("type",'')
url = f"http://172.16.1.223:8086/api/geo/task_check?reqId={req_id}"
try:
response = requests.get(url)
response_data = response.json()
print(response.text)
data = response_data.get("data") or {}
if response_data.get("code") == 200 and data.get("status") == "success":
......@@ -90,9 +91,9 @@ def task_check():
if __name__ == "__main__":
logger.info("调度 启动")
scheduler = BlockingScheduler(timezone="Asia/Shanghai")
# task_check()
scheduler.add_job(
task_commit,
trigger="interval",
......
aidso_geo/core/routes/dashboard.py
View file @ 27ac512a
This diff is collapsed.
aidso_geo/core/routes/third_interface.py
View file @ 27ac512a
......@@ -502,12 +502,57 @@ def mt_task_commit():
if type not in {"0", "1"}:
return err(400, f"platform[{idx}].type must be '0' or '1'")
is_share = str(item.get("is_share", "0"))
if is_share not in {"0", "1"}:
return err(
400,
f"platform[{idx}].is_share must be '0' or '1'"
)
normalized.append({
"name": name,
"thinkingEnabled": thinking_enabled,
"type": type
"type": type,
"is_share": is_share
})
share_requested_platforms = {
item["name"]
for item in normalized
if item["is_share"] == "1"
}
if share_requested_platforms:
try:
share_platforms_raw = tos_utils.get_string_from_tos(
"geo_config/share_platforms.json"
)
share_platforms = json.loads(share_platforms_raw)
except Exception:
traceback.print_exc()
return err(500, "分享平台配置读取失败", 500)
if not isinstance(share_platforms, list):
return err(
500,
"分享平台配置格式错误，必须是平台数组",
500
)
supported_share_platforms = {
str(platform).strip()
for platform in share_platforms
if str(platform).strip()
}
unsupported_share_platforms = sorted(
share_requested_platforms - supported_share_platforms
)
if unsupported_share_platforms:
return err(
400,
"平台不支持分享: "
+ ", ".join(unsupported_share_platforms)
)
platform_costs = parse_platform_costs(auth_info.get("platform_costs"))
thinking_costs = parse_thinking_costs(auth_info.get("thinking_costs"))
......@@ -563,6 +608,7 @@ def mt_task_commit():
"platform": name,
"thinkingEnabled": p["thinkingEnabled"],
"type": type_map[p["type"]],
"is_share": p["is_share"],
"insertime": now_ts,
"status": "ING",
"channel": channel
......@@ -660,7 +706,8 @@ def mt_get_result():
"think": "think.txt",
"context": "context.txt",
"suggestions": "suggestions.txt",
"rich_media_block": "rich_media_block.txt"
"rich_media_block": "rich_media_block.txt",
"share_url": "share_url.text"
}
result = []
......@@ -797,4 +844,3 @@ def mt_get_usage():
# "history_used_list": history_userd_list
})
aidso_geo/core/static/dashboard/dashboard_login.css 0 → 100644
View file @ 27ac512a
* {
box-sizing: border-box;
}
:root {
--primary: #7138f4;
--primary-dark: #5824d6;
--ink: #282735;
--muted: #7b788a;
--line: #e2deea;
}
body {
margin: 0;
color: var(--ink);
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC",
"Microsoft YaHei", Arial, sans-serif;
background: #eef3ff;
}
.login-page {
display: grid;
min-height: 100vh;
grid-template-columns: minmax(420px, 1.08fr) minmax(430px, 0.92fr);
}
.login-visual {
position: relative;
display: flex;
align-items: center;
overflow: hidden;
padding: clamp(50px, 7vw, 110px);
background:
radial-gradient(circle at 22% 18%, rgba(255, 255, 255, 0.22), transparent 26%),
radial-gradient(circle at 84% 84%, rgba(193, 164, 255, 0.3), transparent 30%),
linear-gradient(145deg, #6322eb 0%, #7f3fff 54%, #985fff 100%);
}
.login-visual::before,
.login-visual::after {
content: "";
position: absolute;
border: 1px solid rgba(255, 255, 255, 0.18);
border-radius: 50%;
}
.login-visual::before {
width: 520px;
height: 520px;
top: -270px;
right: -180px;
}
.login-visual::after {
width: 360px;
height: 360px;
bottom: -205px;
left: -130px;
}
.visual-content {
position: relative;
z-index: 1;
max-width: 580px;
color: #fff;
}
.visual-badge {
display: inline-flex;
padding: 7px 11px;
border: 1px solid rgba(255, 255, 255, 0.32);
border-radius: 999px;
background: rgba(255, 255, 255, 0.1);
font-size: 11px;
font-weight: 800;
letter-spacing: 0.15em;
}
.visual-content h1 {
margin: 24px 0 15px;
font-size: clamp(42px, 5.2vw, 74px);
font-weight: 850;
letter-spacing: -0.055em;
line-height: 1.05;
}
.visual-content p {
max-width: 500px;
margin: 0;
color: rgba(255, 255, 255, 0.78);
font-size: clamp(15px, 1.25vw, 19px);
line-height: 1.8;
}
.visual-grid {
display: grid;
width: min(100%, 470px);
grid-template-columns: repeat(3, 1fr);
gap: 12px;
margin-top: 52px;
}
.visual-grid span {
height: 74px;
border: 1px solid rgba(255, 255, 255, 0.17);
border-radius: 14px;
background: rgba(255, 255, 255, 0.08);
box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1);
}
.visual-grid span:nth-child(2),
.visual-grid span:nth-child(4),
.visual-grid span:nth-child(6) {
background: rgba(255, 255, 255, 0.15);
}
.login-panel {
display: grid;
min-width: 0;
place-items: center;
padding: 42px;
background:
radial-gradient(circle at 100% 0%, rgba(113, 56, 244, 0.08), transparent 28%),
#f7f8fc;
}
.login-card {
width: min(100%, 440px);
padding: 42px;
border: 1px solid rgba(226, 222, 234, 0.9);
border-radius: 22px;
background: rgba(255, 255, 255, 0.96);
box-shadow: 0 22px 55px rgba(58, 45, 88, 0.1);
}
.login-logo {
display: block;
width: 240px;
max-width: 78%;
height: auto;
margin-bottom: 42px;
}
.login-heading h2 {
margin: 0;
color: #242230;
font-size: 28px;
font-weight: 850;
letter-spacing: -0.025em;
}
.login-heading p {
margin: 9px 0 30px;
color: var(--muted);
font-size: 14px;
}
.login-field {
display: grid;
gap: 8px;
margin-bottom: 20px;
color: #565267;
font-size: 13px;
font-weight: 750;
}
.input-shell {
position: relative;
display: flex;
align-items: center;
}
.input-shell input {
width: 100%;
height: 50px;
padding: 0 14px 0 44px;
border: 1px solid var(--line);
border-radius: 11px;
outline: none;
background: #fff;
color: #292635;
font: inherit;
font-size: 14px;
transition: border-color 0.18s ease, box-shadow 0.18s ease;
}
.input-shell input::placeholder {
color: #aaa7b2;
}
.input-shell input:focus {
border-color: var(--primary);
box-shadow: 0 0 0 4px rgba(113, 56, 244, 0.11);
}
.input-icon {
position: absolute;
z-index: 1;
left: 17px;
width: 14px;
height: 14px;
pointer-events: none;
}
.user-icon {
border: 2px solid #9994a5;
border-radius: 50%;
}
.user-icon::after {
content: "";
position: absolute;
width: 18px;
height: 9px;
top: 12px;
left: -4px;
border: 2px solid #9994a5;
border-bottom: 0;
border-radius: 12px 12px 0 0;
}
.lock-icon {
top: 18px;
border: 2px solid #9994a5;
border-radius: 3px;
}
.lock-icon::before {
content: "";
position: absolute;
width: 8px;
height: 8px;
top: -9px;
left: 1px;
border: 2px solid #9994a5;
border-bottom: 0;
border-radius: 8px 8px 0 0;
}
.login-error {
margin: -3px 0 17px;
padding: 10px 12px;
border: 1px solid #fecaca;
border-radius: 9px;
background: #fff4f4;
color: #b42323;
font-size: 13px;
}
.login-button {
width: 100%;
height: 50px;
margin-top: 5px;
border: 0;
border-radius: 11px;
background: linear-gradient(135deg, #8247ff, #6825ed);
color: #fff;
cursor: pointer;
font-size: 15px;
font-weight: 800;
box-shadow: 0 11px 22px rgba(113, 56, 244, 0.24);
transition: transform 0.18s ease, box-shadow 0.18s ease;
}
.login-button:hover {
transform: translateY(-1px);
box-shadow: 0 14px 28px rgba(113, 56, 244, 0.3);
}
.login-footer {
margin: 27px 0 0;
color: #aaa7b2;
font-size: 11px;
text-align: center;
}
@media (max-width: 900px) {
.login-page {
grid-template-columns: 1fr;
}
.login-visual {
min-height: 260px;
padding: 50px 34px;
}
.visual-content h1 {
margin-top: 16px;
font-size: 42px;
}
.visual-grid {
display: none;
}
.login-panel {
padding: 34px 20px;
}
}
@media (max-width: 520px) {
.login-visual {
min-height: 220px;
padding: 38px 24px;
}
.visual-content h1 {
font-size: 34px;
}
.visual-content p {
font-size: 13px;
}
.login-card {
padding: 30px 24px;
border-radius: 18px;
}
.login-logo {
width: 210px;
margin-bottom: 34px;
}
}
aidso_geo/core/templates/dashboard_login.html 0 → 100644
View file @ 27ac512a
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AIDSO 数据监控 - 登录</title>
<link rel="stylesheet" href="{{ url_for('static', filename='dashboard/dashboard_login.css') }}">
</head>
<body>
<main class="login-page">
<section class="login-visual">
<div class="visual-content">
<div class="visual-badge">AIDSO DATA</div>
<h1>数据监控中心</h1>
<p>统一查看队列状态、历史任务趋势与三方渠道用量。</p>
<div class="visual-grid" aria-hidden="true">
<span></span><span></span><span></span>
<span></span><span></span><span></span>
</div>
</div>
</section>
<section class="login-panel">
<div class="login-card">
<img
class="login-logo"
src="{{ url_for('static', filename='dashboard/aidso_logo.png') }}"
alt="AIDSO 爱搜"
>
<div class="login-heading">
<h2>登录数据监控</h2>
<p>请输入账号和密码进入系统</p>
</div>
<form method="post" action="{{ url_for('dashboard.dashboard_login') }}">
<label class="login-field">
<span>账号</span>
<span class="input-shell">
<i class="input-icon user-icon" aria-hidden="true"></i>
<input
type="text"
name="username"
autocomplete="username"
placeholder="请输入账号"
required
autofocus
>
</span>
</label>
<label class="login-field">
<span>密码</span>
<span class="input-shell">
<i class="input-icon lock-icon" aria-hidden="true"></i>
<input
type="password"
name="password"
autocomplete="current-password"
placeholder="请输入密码"
required
>
</span>
</label>
{% if error_message %}
<div class="login-error" role="alert">{{ error_message }}</div>
{% endif %}
<button class="login-button" type="submit">登录</button>
</form>
<p class="login-footer">AIDSO 数据监控</p>
</div>
</section>
</main>
</body>
</html>
aidso_geo/core/templates/queue_monitor.html
View file @ 27ac512a
......@@ -2,7 +2,7 @@
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>GEO 队列监控面板</title>
<title>AIDSO 数据监控</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="{{ url_for('static', filename='dashboard/queue_monitor.css') }}">
......@@ -11,20 +11,36 @@
<body>
<div class="layout">
<aside class="sidebar">
<button class="nav-btn active" id="queueNavBtn" onclick="showPanel('queue')">积压查看</button>
<button class="nav-btn" id="trendNavBtn" onclick="showPanel('trend')">历史任务趋势</button>
<div class="brand">
<img
class="brand-logo"
src="{{ url_for('static', filename='dashboard/aidso_logo.png') }}"
alt="AIDSO"
>
</div>
<div class="nav-label">数据看板</div>
<nav class="nav-list">
<button class="nav-btn active" id="queueNavBtn" onclick="showPanel('queue')">积压查看</button>
<button class="nav-btn" id="trendNavBtn" onclick="showPanel('trend')">历史任务趋势</button>
<button class="nav-btn" id="thirdNavBtn" onclick="showPanel('third')">三方用量分析</button>
</nav>
<div class="sidebar-foot">
<span class="health-dot"></span>
<span>数据服务运行中</span>
</div>
</aside>
<main class="main">
<div class="header">
<div>
<div class="title" id="pageTitle">GEO 队列监控面板</div>
<div class="title" id="pageTitle">队列积压监控</div>
<div class="desc" id="pageDesc">查看各平台 stream_batch / batch 队列积压情况</div>
</div>
<div class="right">
<span class="last-time" id="lastTime">未刷新</span>
<button class="btn" onclick="refreshCurrentPanel()">刷新</button>
<a class="logout-link" href="{{ url_for('dashboard.dashboard_logout') }}">退出</a>
</div>
</div>
......@@ -48,11 +64,11 @@
</div>
</div>
<div class="section-head">
<div class="section-title">队列积压详情</div>
<div class="section-note">Redis 队列长度</div>
</div>
<div class="table-wrap">
<div class="section-head panel-card-head">
<div class="section-title">队列积压详情</div>
<div class="section-note">Redis 队列长度</div>
</div>
<table>
<thead>
<tr>
......@@ -69,11 +85,11 @@
</section>
<section class="panel" id="trendPanel">
<div class="section-head">
<div class="section-title">历史任务趋势</div>
<div class="section-note" id="statsNote">等待加载</div>
</div>
<div class="trend-panel">
<div class="section-head panel-card-head">
<div class="section-title">历史任务概览</div>
<div class="section-note" id="statsNote">等待加载</div>
</div>
<div class="filter-row">
<div class="quick-range">
<button class="quick-btn active" id="range3Btn" onclick="setQuickRange(3)">近3天</button>
......@@ -100,6 +116,111 @@
</div>
</section>
<section class="panel" id="thirdPanel">
<div class="third-panel">
<div class="section-head panel-card-head">
<div class="section-title">三方用量概览</div>
<div class="section-note" id="thirdStatsNote">等待加载</div>
</div>
<div class="filter-row">
<div class="quick-range">
<button class="quick-btn active" id="thirdRange3Btn" onclick="setThirdQuickRange(3)">近3天</button>
<button class="quick-btn" id="thirdRange7Btn" onclick="setThirdQuickRange(7)">近7天</button>
<button class="quick-btn" id="thirdRange30Btn" onclick="setThirdQuickRange(30)">近30天</button>
</div>
<div class="field">
<label for="thirdBeginTime">开始时间</label>
<input id="thirdBeginTime" type="date">
</div>
<div class="field">
<label for="thirdEndTime">结束时间</label>
<input id="thirdEndTime" type="date">
</div>
<div class="field">
<label for="thirdChannelFilter">渠道</label>
<select id="thirdChannelFilter">
<option value="">全部渠道</option>
</select>
</div>
<div class="field">
<label for="thirdPlatformFilter">平台</label>
<select id="thirdPlatformFilter">
<option value="">全部平台</option>
</select>
</div>
</div>
<div class="third-metrics">
<div class="third-metric-card"><span>三方任务总量</span><strong id="thirdTotalMetric">0</strong></div>
<div class="third-metric-card"><span>日均任务量</span><strong id="thirdDailyAverageMetric">0</strong></div>
<div class="third-metric-card"><span>三方任务占比</span><strong id="thirdRatioMetric">0%</strong></div>
<div class="third-metric-card"><span>三方分享任务</span><strong id="thirdShareMetric">0</strong></div>
<div class="third-metric-card"><span>当前启用渠道</span><strong id="thirdEnabledMetric">0</strong></div>
</div>
<div class="third-dashboard-grid">
<section class="third-section-card">
<div class="third-card-head">
<strong>渠道用量排行</strong>
<span>按任务量降序</span>
</div>
<div id="thirdRanking"></div>
</section>
<section class="third-section-card">
<div class="third-card-head">
<strong>三方每日用量趋势</strong>
<div class="third-trend-switch">
<button class="active" id="thirdTrendChannelBtn" onclick="setThirdTrendMode('channel')">按渠道</button>
<button id="thirdTrendTotalBtn" onclick="setThirdTrendMode('total')">每日总量</button>
</div>
</div>
<div class="third-trend-chart" id="thirdTrendChart"></div>
</section>
</div>
<section class="third-section-card">
<div class="third-card-head">
<strong>渠道 × 平台用量矩阵</strong>
<span>颜色越深代表任务量越高</span>
</div>
<div class="third-table-scroll">
<table class="third-matrix-table" id="thirdMatrixTable"></table>
</div>
</section>
<section class="third-section-card">
<div class="third-card-head">
<strong>渠道详细数据</strong>
<span>仅展示当前启用渠道</span>
</div>
<div class="third-table-scroll">
<table class="third-detail-table">
<thead>
<tr>
<th>渠道</th>
<th>任务量</th>
<th>三方用量占比</th>
<th>分享任务</th>
<th>平台数</th>
<th>最近使用</th>
<th>日额度</th>
<th>总额度</th>
</tr>
</thead>
<tbody id="thirdDetailBody"></tbody>
</table>
</div>
</section>
<div class="trend-loading-mask" id="thirdLoadingMask">
<div class="loading-box">
<span class="loading-spinner"></span>
<span>加载中</span>
</div>
</div>
</div>
</section>
<div class="error" id="errorBox"></div>
</main>
</div>
......
aidso_geo/models/douyinai_data_process.py
View file @ 27ac512a
import json
from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils
from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
......@@ -31,6 +31,7 @@ def douyin_ai_process_original_data(data):
# 提取并解析JSON数据
data_str = item.split("data:")[1]
json_data = json.loads(data_str)
except (IndexError, json.JSONDecodeError):
continue
......@@ -117,6 +118,30 @@ def douyin_ai_process_original_data(data):
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
if __name__ == '__main__':
file_path = "geo/8004e38c-37da-4e82-b08f-453ccdf0d661/DYAI/original.text"
douyin_ai_process_original_data(file_path)
data_list = bh_utils.query_data(
f"select * from geo_commit_task where taskId = '1cfb92e0-b0ae-40e5-b198-342d079dea0a' and platform = 'DYAI'")
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
return douyin_ai_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/models/spider_save_tos.py
View file @ 27ac512a
......@@ -265,12 +265,12 @@ def qianwen_android_process_quote(url_list):
content = url.get('content')
if content:
if isinstance(content.get('list'),list):
for c in content.get('list'):
for index2,c in enumerate(content.get('list'),start=1):
raw_data = {
"url": c.get('url', ''),
"title": c.get('title', ''),
"snippet": c.get('summary', ''),
"index": index,
"index": index2,
"published_at": c.get('publish_time', ''),
"site_name": c.get('name', ''),
"site_icon": c.get('icon', ''),
......@@ -624,6 +624,3 @@ def process_and_save_files_ai(file_path, search_keyword, url_list, think_content
for content, file_name in data_config:
save_data_to_tos_ai(target_dir, content, file_name)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment