Skip to content

A
aidso-data
Commit 7d6df9f1 authored by Yaowentong's avatar Yaowentong
Browse files
Options

修复数据

parent f051142d
Expand all Show whitespace changes
Inline Side-by-side
Showing with 25097 additions and 146 deletions
aidso_geo/core/hua.py
View file @ 7d6df9f1
......@@ -319,10 +319,10 @@ if __name__ == "__main__":
aidso["prompt_id"] = aidso_excel_data.get('prompt_id','')
aidso["layer"] = aidso_excel_data.get('layer','')
aidso["subcat"] = aidso_excel_data.get('subcat','')
# write_list_dict_to_excel(
# data_list=aidso_result,
# output_path="/Users/yaowentong/Desktop/aidso_result_v2.xlsx"
# )
write_list_dict_to_excel(
data_list=aidso_result,
output_path="/Users/yaowentong/Desktop/aidso_result_v2.xlsx"
)
#
all_file = f"/Users/yaowentong/Desktop/aidso_result_v2.txt"
with open(all_file, "w", encoding="utf-8") as f:
......
aidso_geo/core/interface.py
View file @ 7d6df9f1
from flask import Flask
from aidso_geo.core.routes.dashboard import dashboard_app
from aidso_geo.core.routes.interface import line_app
from aidso_geo.core.routes.feishu_interface import feishu_app
from aidso_geo.core.routes.third_interface import third_app
......@@ -9,6 +11,7 @@ app.json.ensure_ascii = False
app.register_blueprint(line_app)
app.register_blueprint(third_app)
app.register_blueprint(feishu_app)
app.register_blueprint(dashboard_app)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8086)
\ No newline at end of file
aidso_geo/core/routes/dashboard.py 0 → 100644
View file @ 7d6df9f1
from flask import jsonify,Blueprint,render_template
from aidso_geo.config.base_config import init_redis
dashboard_app = Blueprint("dashboard", __name__)
redis_client = init_redis()
@dashboard_app.route("/api/queue/status", methods=["GET"])
def queue_status():
data = {}
platforms = [
"BDAI", "DB", "DOUBA", "DP", "DPA", "DYAI",
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY"
]
for platform in platforms:
stream_key = f"{platform}:geo:stream_batch:list"
batch_key = f"{platform}:geo:batch:list"
data[stream_key] = redis_client.llen(stream_key)
data[batch_key] = redis_client.llen(batch_key)
data["geo:task_commit:list"] = redis_client.llen("geo:task_commit:list")
return jsonify(data)
@dashboard_app.route("/queue/monitor", methods=["GET"])
def queue_monitor_page():
return render_template("queue_monitor.html")
\ No newline at end of file
aidso_geo/core/routes/third_interface.py
View file @ 7d6df9f1
......@@ -661,6 +661,94 @@ def mt_get_result():
}
}), 200
@third_app.route('/open/mt/search_prompt', methods=['POST'])
def mt_search_commit():
auth_header = request.headers.get('Authorization')
if not auth_header:
return err(401, "缺少Authorization请求头", 401)
auth_info = get_auth_info_by_token(auth_header)
if not auth_info:
return err(401, "Authorization参数错误", 401)
if int(auth_info.get("status", 0)) != 1:
return err(401, "Authorization已禁用", 401)
channel = auth_info["channel"]
data = request.get_json(silent=True)
if not isinstance(data, dict):
return err(400, "invalid json body (must be an object)")
keyword = str(data.get("keyword", "")).strip()
if not keyword:
return err(400, "missing required field: keyword")
limit = data.get("limit", 100)
try:
limit = int(limit)
except Exception:
return err(400, "limit must be int")
if limit <= 0:
return err(400, "limit must be > 0")
limit = min(limit, 100)
try:
query_sql = """
SELECT
prompt,
reqId,
taskId,
platform,
status,
insertime
FROM geo_third_task_data
WHERE channel = %(channel)s
AND prompt LIKE %(keyword)s
ORDER BY insertime DESC
LIMIT %(limit)s
"""
rows = bh_utils.query_data(query_sql, {
"channel": channel,
"keyword": f"%{keyword}%",
"limit": limit
}) or []
except Exception as e:
return jsonify({
"code": 500,
"data": None,
"msg": f"查询失败：{str(e)}"
}), 500
result_map = {}
for row in rows:
prompt = row.get("prompt", "")
if not prompt:
continue
if prompt not in result_map:
result_map[prompt] = {
"prompt": prompt,
"reqList": []
}
result_map[prompt]["reqList"].append({
"reqId": row.get("reqId", ""),
"taskId": row.get("taskId", ""),
"platform": row.get("platform", ""),
"status": row.get("status", ""),
"insertTime": row.get("insertime", 0)
})
return ok({
"keyword": keyword,
"list": list(result_map.values())
})
@third_app.route('/open/mt/get_usage', methods=['GET'])
def mt_get_usage():
......@@ -680,12 +768,14 @@ def mt_get_usage():
total_limit = int(auth_info.get("total_limit", 0))
_, today_used, _ = get_today_limit_info(channel, daily_limit)
_, total_used, _ = get_total_limit_info(channel, total_limit)
_, total_used, total_remain = get_total_limit_info(channel, total_limit)
history_userd_list = get_history_used_list(channel)
return ok({
"today_used": today_used,
"total_used": total_used
"total_used": total_used,
"total_remain":total_remain
# "history_used_list": history_userd_list
})
aidso_geo/core/templates/queue_monitor.html 0 → 100644
View file @ 7d6df9f1
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>Geo 队列监控面板</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<style>
* {
box-sizing: border-box;
}
body {
margin: 0;
padding: 24px;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Microsoft YaHei", Arial, sans-serif;
background: #f3f5f9;
color: #1f2937;
}
.header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 24px;
}
.title {
font-size: 26px;
font-weight: 700;
}
.desc {
margin-top: 6px;
font-size: 14px;
color: #6b7280;
}
.right {
display: flex;
align-items: center;
gap: 12px;
}
.last-time {
font-size: 13px;
color: #6b7280;
}
.btn {
border: none;
border-radius: 8px;
background: #2563eb;
color: #fff;
padding: 9px 16px;
cursor: pointer;
font-size: 14px;
}
.btn:hover {
background: #1d4ed8;
}
.summary {
display: grid;
grid-template-columns: repeat(4, 1fr);
gap: 16px;
margin-bottom: 24px;
}
.card {
background: #fff;
border-radius: 14px;
padding: 18px;
box-shadow: 0 6px 18px rgba(15, 23, 42, 0.06);
}
.card-name {
color: #6b7280;
font-size: 14px;
margin-bottom: 10px;
}
.card-value {
font-size: 30px;
font-weight: 700;
}
.table-wrap {
background: #fff;
border-radius: 14px;
overflow: hidden;
box-shadow: 0 6px 18px rgba(15, 23, 42, 0.06);
}
table {
width: 100%;
border-collapse: collapse;
}
thead {
background: #f9fafb;
}
th, td {
padding: 14px 16px;
border-bottom: 1px solid #edf0f5;
text-align: left;
font-size: 14px;
}
th {
font-weight: 600;
color: #374151;
}
.platform {
font-weight: 700;
}
.num {
font-weight: 700;
}
.num-ok {
color: #16a34a;
}
.num-warn {
color: #f59e0b;
}
.num-danger {
color: #dc2626;
}
.status {
display: inline-block;
padding: 4px 10px;
border-radius: 999px;
font-size: 12px;
font-weight: 600;
}
.status-ok {
background: #dcfce7;
color: #166534;
}
.status-warn {
background: #fef3c7;
color: #92400e;
}
.status-danger {
background: #fee2e2;
color: #991b1b;
}
.error {
display: none;
margin-top: 16px;
padding: 12px 14px;
border-radius: 8px;
background: #fee2e2;
color: #991b1b;
font-size: 14px;
}
@media (max-width: 900px) {
.summary {
grid-template-columns: repeat(2, 1fr);
}
.header {
flex-direction: column;
align-items: flex-start;
gap: 12px;
}
}
@media (max-width: 600px) {
body {
padding: 14px;
}
.summary {
grid-template-columns: 1fr;
}
th, td {
padding: 10px;
font-size: 13px;
}
}
</style>
</head>
<body>
<div class="header">
<div>
<div class="title">Geo 队列监控面板</div>
<div class="desc">查看各平台 stream_batch / batch 队列积压情况</div>
</div>
<div class="right">
<span class="last-time" id="lastTime">未刷新</span>
<button class="btn" onclick="loadQueueStatus()">刷新</button>
</div>
</div>
<div class="summary">
<div class="card">
<div class="card-name">总积压</div>
<div class="card-value" id="totalCount">0</div>
</div>
<div class="card">
<div class="card-name">stream_batch 总数</div>
<div class="card-value" id="streamBatchCount">0</div>
</div>
<div class="card">
<div class="card-name">batch 总数</div>
<div class="card-value" id="batchCount">0</div>
</div>
<div class="card">
<div class="card-name">提交队列</div>
<div class="card-value" id="commitCount">0</div>
</div>
</div>
<div class="table-wrap">
<table>
<thead>
<tr>
<th>平台</th>
<th>stream_batch</th>
<th>batch</th>
<th>总积压</th>
<th>状态</th>
</tr>
</thead>
<tbody id="queueTableBody"></tbody>
</table>
</div>
<div class="error" id="errorBox"></div>
<script>
const API_URL = "/api/queue/status";
const platforms = [
"BDAI", "DB", "DOUBA", "DP", "DPA", "DYAI",
"KIMI", "TXYB", "TXYBA", "TYQW", "TYQWA", "WXYY"
];
function getNumClass(num) {
if (num === 0) {
return "num-ok";
}
if (num < 500) {
return "num-warn";
}
return "num-danger";
}
function getStatus(total) {
if (total === 0) {
return {
text: "正常",
className: "status status-ok"
};
}
if (total < 500) {
return {
text: "轻微积压",
className: "status status-warn"
};
}
return {
text: "严重积压",
className: "status status-danger"
};
}
function renderData(data) {
const tbody = document.getElementById("queueTableBody");
let totalCount = 0;
let streamBatchCount = 0;
let batchCount = 0;
tbody.innerHTML = "";
platforms.forEach(platform => {
const streamKey = `${platform}:geo:stream_batch:list`;
const batchKey = `${platform}:geo:batch:list`;
const streamValue = Number(data[streamKey] || 0);
const batchValue = Number(data[batchKey] || 0);
const platformTotal = streamValue + batchValue;
streamBatchCount += streamValue;
batchCount += batchValue;
totalCount += platformTotal;
const status = getStatus(platformTotal);
const tr = document.createElement("tr");
tr.innerHTML = `
<td class="platform">${platform}</td>
<td class="num ${getNumClass(streamValue)}">${streamValue}</td>
<td class="num ${getNumClass(batchValue)}">${batchValue}</td>
<td class="num ${getNumClass(platformTotal)}">${platformTotal}</td>
<td><span class="${status.className}">${status.text}</span></td>
`;
tbody.appendChild(tr);
});
const commitCount = Number(data["geo:task_commit:list"] || 0);
document.getElementById("totalCount").innerText = totalCount;
document.getElementById("streamBatchCount").innerText = streamBatchCount;
document.getElementById("batchCount").innerText = batchCount;
document.getElementById("commitCount").innerText = commitCount;
document.getElementById("lastTime").innerText = "最后刷新：" + new Date().toLocaleString();
}
async function loadQueueStatus() {
const errorBox = document.getElementById("errorBox");
try {
errorBox.style.display = "none";
errorBox.innerText = "";
const res = await fetch(API_URL, {
method: "GET",
cache: "no-store"
});
if (!res.ok) {
throw new Error("接口请求失败，状态码：" + res.status);
}
const data = await res.json();
renderData(data);
} catch (e) {
errorBox.style.display = "block";
errorBox.innerText = "加载失败：" + e.message;
}
}
loadQueueStatus();
setInterval(loadQueueStatus, 30000);
</script>
</body>
</html>
\ No newline at end of file
aidso_geo/models/baiduai_data_process.py
View file @ 7d6df9f1
......@@ -14,7 +14,8 @@ def thinking_enabled_fragments(json_data):
return search_word
def baiduai_process_original_data(file_path):
def baiduai_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list = []
rich_media_block = []
......
aidso_geo/models/deepseek_android_data_process.py
View file @ 7d6df9f1
......@@ -6,7 +6,8 @@ from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def deepseek_android_process_original_data(file_path):
def deepseek_android_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list = ""
think_content = ""
......
aidso_geo/models/deepseek_data_process.py
View file @ 7d6df9f1
......@@ -15,7 +15,8 @@ def thinking_enabled_fragments(json_data):
return search_word
def deepseek_process_original_data(file_path):
def deepseek_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list = ""
think_content = ""
......
aidso_geo/models/doubao_data_process.py
View file @ 7d6df9f1
......@@ -7,7 +7,8 @@ from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def doubao_process_original_data(file_path):
def doubao_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = ""
think_content = ""
response_content = ""
......@@ -35,6 +36,7 @@ def doubao_process_original_data(file_path):
try:
json_content = json.loads(payload)
except (IndexError, json.JSONDecodeError):
continue
......@@ -181,7 +183,7 @@ if __name__ == '__main__':
# 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35'
# ]
# for task in task_id_list:
task = '1d35ff25-760d-43c4-a24f-210d74e82ffe'
task = 'a090ebc2-8f74-4533-915c-f7352658c42e'
file_path = f'geo/{task}/DB/original.text'
doubao_process_original_data(file_path)
......
aidso_geo/models/douyinai_data_process.py
View file @ 7d6df9f1
......@@ -6,7 +6,8 @@ from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def douyin_ai_process_original_data(file_path):
def douyin_ai_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list = []
think_content = ""
......
aidso_geo/models/kimi_data_process.py
View file @ 7d6df9f1
......@@ -8,7 +8,8 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
from aidso_geo.models import spider_save_tos
def kimi_process_original_data(file_path):
def kimi_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = []
think_content = ""
response_content = ""
......
aidso_geo/models/process.py
View file @ 7d6df9f1
......@@ -1286,7 +1286,7 @@ def result_v2(response_content, data):
# -------------------------
# 获取ai提及词
ai_word_list = cache_get_ai_brand_list(taskId, platform, response_content, prompt)
print(ai_word_list)
# 获取所有词
all_word_list = []
if isinstance(ai_word_list, list):
......@@ -1311,8 +1311,16 @@ def result_v2(response_content, data):
# 获取所有词的排名 word + rank_list
all_word_rank_list = get_keyword_ranks(response_content, all_word_set_list)
# 获取所有词的排名 word + rank + count
print(all_word_rank_list)
print('----')
print('----')
print('----')
all_keyword_with_rank = convert_rank_data(all_word_rank_list)
print(all_keyword_with_rank)
print('---')
print('---')
print('---')
print('---')
# 获取所有词的品牌
all_keyword_with_brand = keyword_map_brand(all_word_list)
......@@ -1625,32 +1633,32 @@ def platform_process(data):
response_content = None
#——————————————###################
has_original = check_file_in_tos(original_path)
has_context = check_file_in_tos(context_path)
# 重新跑逻辑 当俩个都有走下面
if has_original and has_context:
process_func = PLATFORM_PROCESS_MAP.get(platform)
if process_func:
_, _, _, _, response_content, _ = process_func(original_path)
# 2. 只有 context：直接读 context
elif has_context:
response_content = tos_utils.get_string_from_tos(context_path)
# 3. 只有 original：跑平台解析逻辑
elif has_original:
process_func = PLATFORM_PROCESS_MAP.get(platform)
if process_func:
_, _, _, _, response_content, _ = process_func(original_path)
#——————————————###################
# if check_file_in_tos(context_path):
# #——————————————###################
# has_original = check_file_in_tos(original_path)
# has_context = check_file_in_tos(context_path)
# # 重新跑逻辑 当俩个都有走下面
# if has_original and has_context:
# process_func = PLATFORM_PROCESS_MAP.get(platform)
# if process_func:
# _, _, _, _, response_content, _ = process_func(original_path)
# # 2. 只有 context：直接读 context
# elif has_context:
# response_content = tos_utils.get_string_from_tos(context_path)
# else:
#
# # 3. 只有 original：跑平台解析逻辑
# elif has_original:
# process_func = PLATFORM_PROCESS_MAP.get(platform)
# if process_func:
# _, _, _, _, response_content, _ = process_func(original_path)
# #——————————————###################
if check_file_in_tos(context_path):
response_content = tos_utils.get_string_from_tos(context_path)
else:
process_func = PLATFORM_PROCESS_MAP.get(platform)
if process_func:
_, _, _, _, response_content, _ = process_func(data)
if response_content:
result_v2(response_content, data)
else:
......@@ -1871,7 +1879,8 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
query_sql = f"""
SELECT *
FROM geo_commit_task
WHERE insertime < {last_insertime} and status ='SUCCESS'
WHERE insertime < {last_insertime}
AND status = 'SUCCESS'
ORDER BY insertime DESC
LIMIT {PAGE_SIZE}
"""
......@@ -1882,8 +1891,13 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
logger.success("处理结束")
break
# 关键：处理之前先保存本批次最后一条的原始 insertime
next_last_insertime = int(data_list[-1].get("insertime"))
logger.success(
f"本批次查询到 {len(data_list)} 条，last_insertime={last_insertime}"
f"本批次查询到 {len(data_list)} 条，"
f"当前 last_insertime={last_insertime}, "
f"下一批 next_last_insertime={next_last_insertime}"
)
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
......@@ -1895,45 +1909,50 @@ def run_data(PAGE_SIZE,MAX_WORKERS):
except Exception as e:
logger.exception(f"platform_process 执行异常: {e}")
# 用本批次最后一条的 insertime 推进游标
last_insertime = int(data_list[-1].get("insertime"))
# 用处理前保存的原始游标推进
last_insertime = next_last_insertime
logger.success(f"本批次处理完成，更新 last_insertime={last_insertime}")
if __name__ == '__main__':
# data_list = bh_utils.query_data(f"select * from geo_commit_task where status ='PROCESSING' ")
# # data_list = bh_utils.query_data(query_sql)
# # print(data_list)
# # # #
# # # #
# # # # # #
# def handle_item(i):
# if i.get('comWordsMap'):
# i['comWordsMap'] = json.loads(i.get('comWordsMap'))
# if i.get('brandWords'):
# i['brandWords'] = json.loads(i.get('brandWords'))
# if i.get('comWords'):
# i['comWords'] = json.loads(i.get('comWords'))
# if i.get('keywords'):
# i['keywords'] = json.loads(i.get('keywords'))
# if i.get('productWordsMap'):
# i['productWordsMap'] = json.loads(i.get('productWordsMap'))
# type_t = i.get('type')
# # type_t = 'batch'
#
# # return task_send_queue(i,type_t)
# return platform_process(i)
#
# if data_list:
# with ThreadPoolExecutor(max_workers=30) as executor:
# futures = [executor.submit(handle_item, i) for i in data_list]
#
# for future in as_completed(futures):
# try:
# future.result()
# except Exception as e:
# logger.exception(f"platform_process 执行异常: {e}")
PAGE_SIZE =10000
MAX_WORKERS =30
run_data(PAGE_SIZE,MAX_WORKERS)
data_list = bh_utils.query_data(f"select * from geo_commit_task where reqId = '84db7b12-88e7-4aaa-bc58-8d61f1051934'")
# data_list = bh_utils.query_data(query_sql)
# print(data_list)
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
type_t = i.get('type')
# type_t = 'batch'
# return task_send_queue(i,type_t)
return platform_process(i)
if data_list:
with ThreadPoolExecutor(max_workers=30) as executor:
futures = [executor.submit(handle_item, i) for i in data_list]
for future in as_completed(futures):
try:
future.result()
except Exception as e:
logger.exception(f"platform_process 执行异常: {e}")
# PAGE_SIZE =1000
# MAX_WORKERS =50
# run_data(PAGE_SIZE,MAX_WORKERS)
text = """
"""
# keywords = []
# get_keyword_ranks()
aidso_geo/models/qianwen_android_data_process.py
View file @ 7d6df9f1
import json
import traceback
from asyncio import as_completed
from concurrent.futures import ThreadPoolExecutor
from aidso_geo.models import spider_save_tos
from aidso_geo.utils import robot_utils
from aidso_geo.utils import robot_utils, bh_utils
from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def qianwen_android_process_original_data(file_path):
def qianwen_android_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = []
url_list_batch = []
think_content = ""
......@@ -88,6 +91,12 @@ def qianwen_android_process_original_data(file_path):
if mime_type == 'multi_load/iframe' and multi_load_type == 'taoassistant_fold_product_feeds' and status == 'complete':
for mu in multi_load:
print(mu)
print('---')
print('---')
print('---')
print('---')
print('---')
if mu.get('type') == 'taoassistant_fold_product_feeds':
source_seq = mu.get('source_seq')
jump_url = next(
......@@ -203,6 +212,8 @@ def qianwen_android_process_original_data(file_path):
suggestions.append(pa.get('show_text'))
if url_list_batch:
url_list = url_list_batch
print(response_content)
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
......@@ -231,6 +242,42 @@ def qianwen_android_process_original_data(file_path):
if __name__ == '__main__':
#https://tcdn.aidso.com/geo/6fede9d0-ce6c-4c97-bcb8-43c499fc24e5/TYQWA/think.txt?secret=1104ed37df6abefbb2b454530b06515f0f59505d5751504e524855
# 6fede9d0-ce6c-4c97-bcb8-43c499fc24e5
file_path2 = 'geo/0a2ed990b43a4f389ecdc14d185a3866/TYQWA/original.text'
# file_path2 = 'geo/ef586bf6-55d8-4e2a-8fb0-5fd1beb5bf7c/TYQWA/original.text'
qianwen_android_process_original_data(file_path2)
# qianwen_android_process_original_data(file_path2)
data_list = bh_utils.query_data(f"select * from geo_commit_task where taskId = '0a2ed990b43a4f389ecdc14d185a3866' and platform = 'TYQWA'")
# data_list = bh_utils.query_data(query_sql)
# print(data_list)
# # #
# # #
# # # # #
def handle_item(i):
if i.get('comWordsMap'):
i['comWordsMap'] = json.loads(i.get('comWordsMap'))
if i.get('brandWords'):
i['brandWords'] = json.loads(i.get('brandWords'))
if i.get('comWords'):
i['comWords'] = json.loads(i.get('comWords'))
if i.get('keywords'):
i['keywords'] = json.loads(i.get('keywords'))
if i.get('productWordsMap'):
i['productWordsMap'] = json.loads(i.get('productWordsMap'))
type_t = i.get('type')
# type_t = 'batch'
# return task_send_queue(i,type_t)
return qianwen_android_process_original_data(i)
if data_list:
for i in data_list:
try:
handle_item(i)
except Exception as e:
...
aidso_geo/models/qianwen_data_process.py
View file @ 7d6df9f1
......@@ -7,7 +7,8 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
from aidso_geo.models import spider_save_tos
def qianwen_process_original_data(file_path):
def qianwen_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = []
url_list_batch = []
think_content = ""
......@@ -144,7 +145,7 @@ def qianwen_process_original_data(file_path):
if __name__ == '__main__':
file_path2 = 'geo/a5136478-a267-4824-84a9-08a57e1290ec/TYQW/original.text'
file_path2 = 'geo/21fd5xxx37f-131b-469axxxxx-9d5c-6ea00da2c0d6/TYQW/original.text'
# file_path2 = 'geo/jqk/TYQW/2.txt'
# /geo/900f0ea6e9a34c95b6b57aa7519a4820/
# file_path3 = 'geo/51a7ee04-711c-4cf0-9d4c-4b523fba7037/TYQW/original.text'
......
aidso_geo/models/wenxin_data_process.py
View file @ 7d6df9f1
......@@ -7,7 +7,8 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
from aidso_geo.models import spider_save_tos
def wenxin_process_original_data(file_path):
def wenxin_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = ""
think_content = ""
response_content = ""
......
aidso_geo/models/xiaohongshu_android_data_process.py
View file @ 7d6df9f1
......@@ -7,7 +7,8 @@ from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def xiaohongshu_android_process_original_data(file_path):
def xiaohongshu_android_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list =[]
think_content = ""
......
aidso_geo/models/yuanbao_android_data_process.py
View file @ 7d6df9f1
......@@ -6,7 +6,8 @@ from aidso_geo.utils.ai_interface import get_parse_sse_result
from aidso_geo.utils.tos_utils import get_string_from_tos
def yuanbao_android_process_original_data(file_path):
def yuanbao_android_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
"""处理文件内容，提取思考内容、响应内容和URL列表"""
url_list =[]
think_content = ""
......
aidso_geo/models/yuanbao_data_process.py
View file @ 7d6df9f1
......@@ -7,7 +7,8 @@ from aidso_geo.utils.tos_utils import get_string_from_tos
from aidso_geo.models import spider_save_tos
def yuanbao_process_original_data(file_path):
def yuanbao_process_original_data(data):
file_path = f'geo/{data["taskId"]}/{data["platform"]}/original.text'
url_list = []
think_content = ""
response_process = ""
......
third_live/1.json 0 → 100644
View file @ 7d6df9f1
{
'uniqpid': '102775237',
'reason': '氨基酸成分，可卸属性',
'activityIdBase64': 'MTI0MTE2MDc3',
'short_title_c2c': '',
'clickTrace': 'query:%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90;nid:643872234510;cat_id:50011977;seller_id:719647565;seller_type:0;src:mainse;recType:null;rn:7dcf1a0fd37aa7ee1b2fc85a543e306a;c_flag:false;client:iphone;sp_rank_features:;wlsort:98;price:38;sort:null;tpp_bucket:82;catepgoryp:50011977 1801 0;gray:0;grayBucket:81;channelSrp:qwenApp;servicePlanCode:idsByOrder;srp:MainSrp;skuEngV2:F',
'extraParams': [
{
'value': 'taobaoSearch',
'key': 'xxc'
},
{
'value': '%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90',
'key': 'detailAlgoParam'
},
{
'value': '0000JIq5Mck0LYPLMNqrdaBX3ftWuJZhQMTTgbZfuxjA4OU',
'key': 'mi_id'
},
{
'value': '4988814839437',
'key': 'skuId'
},
{
'value': '3',
'key': 'skuPriceType'
},
{
'value': '3820',
'key': 'upStreamPrice'
}
],
'imageInfo': [
{
'imageUrl': 'https://gw.alicdn.com/img/bao/uploaded/i4/i1/719647565/O1CN017qqWvi25kpEGZLOBE_!!719647565.jpg',
'fieldTemplate': 'image'
}
],
'summaryTipsColor': '',
'tItemType': 'ms_tb-webb-ai-assistant_qwen-search-m3',
'price': '53.00',
'realSales': '100+人付款',
'sameCount': '94',
'shopInfo': {
'pbshowshopinfo': 'true',
'shopInfoList': [
'进店'
],
'url': 'https://shop.m.taobao.com/shop/shop_index.htm?shop_navi=allitems&upText=%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90&item_id=643872234510',
'shopInfoColor': '#333333'
},
'hiddenSet': {},
'priceColor': '#000000',
'listTwoLineTitle': 'true',
'itemPriceSign': '',
'fullSpan': 'false',
'priceShow': {
'unit': '¥',
'price': '38.20',
'preText': ''
},
'itemId': '643872234510',
'similarURL': 'http://h5.m.taobao.com/app/searchsimilar/www/tbsimilar/index.html?from=tbsearch&showtype=similar&liantiao=wsearch_pre_http_gray&scm=20140662.search&vm=nw&nid=643872234510&q=%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90&searchToken=3d283b62c42455595828ceb5770e31880507776f6177d28d8e2e3c85378fd989',
'g9': 'false',
'dItemType': 'nt_auction_2019',
'isB2c': '0',
'productLaunch': '',
'iconList': 'commonPreTitleActivity,guanfanglijian2023_nonactivity,thbyf,gongyibaobei',
'locType': '',
'structuredUSPInfo': [
{
'icon': 'https://gw.alicdn.com/imgextra/i3/O1CN01lJ98uI28BVfSRC6Nt_!!6000000007894-2-tps-47-38.png',
'type': 'taowiseSellPoint'
}
],
'priceShowWithIcon': {
'iconHeight': '',
'iconWidth': '',
'priceColor': '#ff5000',
'showOriginPrice': 'false',
'index': 1.0,
'preText': '',
'originPrice': '￥45',
'unit': '¥',
'suffixColor': '#ff5000',
'price': '38.20',
'suffixText': '券后价',
'domClass': 'umpcouponprice',
'hiddenPriceUnderline': 'true',
'iconUrl': ''
},
'wfTwoLineTitle': 'true',
'priceWithRate': '45.00',
'nidlong': 643872234510.0,
'title': '温和不刺激氨基酸洁面瑷尔博士',
'utLogMap': {
'shop': 'look_back_user_num;shop_name;into_shop',
'summary_price': '45.00',
'price_desc': '券后价',
'umpDirect': 'true',
'title': 'showTitle',
's_id': '4988814839437',
'list_param': '可以卸防晒的氨基酸洗面奶有哪些推荐_98_7dcf1a0fd37aa7ee1b2fc85a543e306a',
'x_object_type_search': 'item',
'm_sp': '1',
'xui_ump': 'true',
'sku_p_t': '3',
'p_t': '2',
'saleText': '100+人付款',
'provcity': '辽宁 沈阳',
'pic_source': 'sp',
'x_object_id': '643872234510'
},
'result': 'T',
'auctionURL': 'http://a.m.taobao.com/i643872234510.htm?&ttid=201200%40qwen_iphone_10.4.0&sid=eb7fd95ddac4274e8f55c68f30453b3e',
'itemCollect': 'false',
'leafCategory': '50011977',
'isP4p': 'false',
'xsearchFullspan': 'false',
'detailBaseUrl': '',
'similarSameUrl': '//market.m.taobao.com/app/nx3/similar_and_same?q=%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90&prern=7dcf1a0fd37aa7ee1b2fc85a543e306a&m=newsimilarsame&isP4p=false&n=10&nid=643872234510&searchToken=3d283b62c42455595828ceb5770e31880507776f6177d28d8e2e3c85378fd989&hasSimilar=true&graphnid=643872234510&hasSame=true&uniqpid=102775237&selected=similar&category=50011977&sellerId=719647565&from=longpress&channelSrp=search&spm=a2141.7631557.0.0&appId=18601',
'item_id': '643872234510',
'exposureInfo': '643872234510#0#0#0',
'cardType': 'item',
'pic_path': 'https://gw.alicdn.com/img/bao/uploaded/i4/i1/719647565/O1CN017qqWvi25kpEGZLOBE_!!719647565.jpg',
'structuredShopInfo': {
'pbshowshopinfo': True,
'bgColor': '#f7f7f7',
'paddingRight': '6px',
'infoList': [
{
'color': '#FF7C40',
'sourceType': 'look_back_user_num',
'text': '回头客2千',
'hiddenType': 'all'
},
{
'color': '#666666',
'sourceType': 'shop_name',
'text': '喜乐汪记 原喵记',
'hiddenType': 'suffix'
},
{
'color': '#333333',
'sourceType': 'into_shop',
'text': '进店',
'hiddenType': 'all'
}
],
'guideColor': '#333333',
'radius': '10.5px',
'paddingLeft': '6px',
'url': 'https://shop.m.taobao.com/shop/shop_index.htm?shop_navi=allitems&upText=%E5%8F%AF%E4%BB%A5%E5%8D%B8%E9%98%B2%E6%99%92%E7%9A%84%E6%B0%A8%E5%9F%BA%E9%85%B8%E6%B4%97%E9%9D%A2%E5%A5%B6%E6%9C%89%E5%93%AA%E4%BA%9B%E6%8E%A8%E8%8D%90&item_id=643872234510'
},
'icons': [
{
'alias': 'commonPreTitleActivity',
'showType': '0',
'domClass': 'commonPreTitleActivity',
'type': 'img',
'iconStyle': {
'light': {
'img': 'https://gw.alicdn.com/imgextra/i4/O1CN013NswFg1IapZ0z3ptR_!!6000000000910-2-tps-240-56.png',
'width': '240',
'height': '56'
}
},
'group': '1'
},
{
'innerIconGroup': 'guanfanglijian2023',
'alias': 'guanfanglijian2023_nonactivity',
'showType': '0',
'domClass': 'guanfanglijian2023_nonactivity',
'text': '官方立减15%',
'type': 'text',
'group': '2'
},
{
'innerIconGroup': 'zengyunfeixiangroup',
'alias': 'thbyf',
'showType': '0',
'domClass': 'thbyf',
'text': '退货宝',
'type': 'text',
'group': '2'
},
{
'alias': 'gongyibaobei',
'showType': '0',
'domClass': 'gongyibaobei',
'text': '公益宝贝',
'type': 'text',
'group': '2'
}
],
'userId': '719647565',
'pltSimilarUrl': 'https://h5.m.taobao.com/tusou/image_editor/index.html?picurl=http%3A%2F%2Fg.search2.alicdn.com%2Fimg%2Fbao%2Fuploaded%2Fi4%2Fi1%2F719647565%2FO1CN017qqWvi25kpEGZLOBE_%21%21719647565.jpg_600x600q90.jpg&pssource=zszxs&photofrom=zszxs&item_id=643872234510&seller_id=719647565',
'showLongTitle': 'true',
'similarCount': '1',
'localPrice': ''
}
\ No newline at end of file
third_live/2.json 0 → 100644
View file @ 7d6df9f1
This diff is collapsed.
third_live/test.py 0 → 100644
View file @ 7d6df9f1
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment