Skip to content

A
aidso-data
Commit 8f636361 authored by Yaowentong's avatar Yaowentong
Browse files
Options

舆情追踪修复

parent e8391d82
Hide whitespace changes
Inline Side-by-side
Showing with 56 additions and 42 deletions
aidso_geo/models/doubao_data_process.py
View file @ 8f636361
...@@ -81,38 +81,39 @@ def doubao_process_original_data(file_path): ...@@ -81,38 +81,39 @@ def doubao_process_original_data(file_path):
if message_data.get('content_type') == 2002: if message_data.get('content_type') == 2002:
suggestions = suggestions + json.loads(message_data.get('content')).get('suggestions') suggestions = suggestions + json.loads(message_data.get('content')).get('suggestions')
else: else:
if json_content.get("patch_op"): if json_content.get("patch_op"):
if json_content.get('patch_op')[0].get("patch_object") == 111: if json_content.get('patch_op')[0].get("patch_object") == 111:
if json_content.get('patch_op')[0].get("patch_value").get("tts_content"): if json_content.get('patch_op')[0].get("patch_value").get("tts_content"):
response_content += json_content.get('patch_op')[0].get("patch_value").get("tts_content") response_content += json_content.get('patch_op')[0].get("patch_value").get("tts_content")
if json_content.get('patch_op')[0].get("patch_object") == 1: if json_content.get('patch_op')[0].get("patch_object") == 1:
if json_content.get('patch_op')[0].get("patch_value",{}).get("content_block"):
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if json_content.get('patch_op')[0].get("patch_value",{}).get("content_block")[0].get(
"content").get("search_query_result_block"): "content",{}).get("search_query_result_block",{}):
search_keyword = json_content.get('patch_op')[0].get("patch_value").get("content_block")[ search_keyword = json_content.get('patch_op')[0].get("patch_value").get("content_block")[
0].get("content").get("search_query_result_block").get("queries") 0].get("content").get("search_query_result_block").get("queries")
url_list = json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( url_list = json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"content").get("search_query_result_block").get("results") "content").get("search_query_result_block").get("results")
if is_think == True and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if is_think == True and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"content").get("search_query_result_block").get("results") is not None: "content").get("search_query_result_block").get("results") is not None:
think_content += "\n\n" think_content += "\n\n"
think_content += "**" think_content += "**"
think_content+=json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( think_content+=json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"content").get("search_query_result_block").get("summary") "content").get("search_query_result_block").get("summary")
think_content += "**" think_content += "**"
think_content += "\n\n" think_content += "\n\n"
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get( "block_type") ==10000 and json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"parent_id") and len(json_content.get('patch_op'))>1: "parent_id") and len(json_content.get('patch_op'))>1:
is_think = True is_think = True
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get("content").get('text_block').get("text"): if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get("content").get('text_block').get("text"):
think_content+=json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get("content").get("text_block").get("text") think_content+=json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get("content").get("text_block").get("text")
continue
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") == 10040:
is_think = False
continue continue
if json_content.get('patch_op')[0].get("patch_value").get("content_block")[0].get(
"block_type") == 10040:
is_think = False
continue
if json_content.get('patch_op')[0].get("patch_object") == 50: if json_content.get('patch_op')[0].get("patch_object") == 50:
for sug in json.loads( for sug in json.loads(
json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")): json_content.get('patch_op')[0].get("patch_value").get("ext").get("sp_v2")):
...@@ -133,12 +134,10 @@ def doubao_process_original_data(file_path): ...@@ -133,12 +134,10 @@ def doubao_process_original_data(file_path):
response_content = content_block[0].get("content").get("text_block").get("text") response_content = content_block[0].get("content").get("text_block").get("text")
suggestions = list(set(suggestions)) suggestions = list(set(suggestions))
spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content, spider_save_tos.process_and_save_files(file_path, search_keyword, url_list, think_content, response_content,
suggestions,rich_media_block) suggestions,rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e: except Exception as e:
traceback.print_exc()
parts = file_path.split('/') parts = file_path.split('/')
platform = parts[2] platform = parts[2]
task_id = parts[1] task_id = parts[1]
...@@ -164,8 +163,26 @@ def doubao_process_original_data(file_path): ...@@ -164,8 +163,26 @@ def doubao_process_original_data(file_path):
if __name__ == '__main__': if __name__ == '__main__':
file_path = 'geo/50adc4fd-40de-4352-bb63-649207718036/DB/original.text'
# task_id_list = [
# '8629188a-46c9-429f-8aa6-d342110326f0', 'ba138c4d-2bb4-45e3-bda3-ecf80553ac89',
# 'e178ee36-b9ca-4f1f-802b-83b5e4b820f0', '52efa2d8-616f-48b9-b656-59833fbb91b8',
# '769570cf-c385-4ed7-8e28-f57b5f89d34b', 'cac5bbdb-2145-44a0-9b81-eb20b286d278',
# '7d8ad6ad-c784-407e-9e5f-127c15de5c9e', '6a839541-e3f3-4cd6-a33d-366b3c3297b9',
# '86bd6c56-18d4-41c4-a75a-8d15da5ca589', 'd7e86e4b-521b-4f6f-a34d-ccc7fa629e09',
# '5069cfbb-1150-4bf6-a967-698640c2ce8f', '8bc46734-3469-4681-8a0c-f048169e9486',
# '0480d9c5-0ca9-4c5f-a5c9-dfc4db3b5511', '34cfbfae-cf25-49e6-80a9-199d31813bf2',
# '1d35ff25-760d-43c4-a24f-210d74e82ffe', 'f1d7ca94-7ddf-4548-9b5f-8890d39e7b6c',
# '6fb6c767-54bf-4872-ad60-c52223238d41', '785bf969-4645-4ca7-a9c9-6a5f104a7894',
# 'e52a3c9c-720a-46c5-98f5-efb3e9d8c597', 'ccc66d28-8843-4ea0-a3b0-2700c655260b',
# '3aeee8d2-3086-48c6-a9cb-e29f4a244e34', '67660404-bac4-4bb4-80bb-4b78fd713af0',
# '4b0c64e1-8393-4638-ab98-6b14c579912c', '9217ad06-413a-4161-9be8-6150ffe7b4eb',
# 'e9b27490-6ff0-48da-91a5-3dbbb8494c1d', 'b92b318c71d54c399ed033a722c06a35'
# ]
# for task in task_id_list:
task = '1d35ff25-760d-43c4-a24f-210d74e82ffe'
file_path = f'geo/{task}/DB/original.text'
doubao_process_original_data(file_path) doubao_process_original_data(file_path)
......
aidso_geo/models/qianwen_android_data_process.py
View file @ 8f636361
...@@ -33,9 +33,6 @@ def qianwen_android_process_original_data(file_path): ...@@ -33,9 +33,6 @@ def qianwen_android_process_original_data(file_path):
json_data = json.loads(data_str) json_data = json.loads(data_str)
# #
# print('----')
# print('----')
except (IndexError, json.JSONDecodeError): except (IndexError, json.JSONDecodeError):
continue continue
...@@ -113,7 +110,8 @@ def qianwen_android_process_original_data(file_path): ...@@ -113,7 +110,8 @@ def qianwen_android_process_original_data(file_path):
'' ''
) )
if isinstance(query,str): if isinstance(query,str):
query = json.loads(query) if query:
query = json.loads(query)
if isinstance(query,list): if isinstance(query,list):
search_keyword.extend(query) search_keyword.extend(query)
if isinstance(mu.get('content').get('cardData').get('data').get('items'), list): if isinstance(mu.get('content').get('cardData').get('data').get('items'), list):
...@@ -209,7 +207,6 @@ def qianwen_android_process_original_data(file_path): ...@@ -209,7 +207,6 @@ def qianwen_android_process_original_data(file_path):
suggestions, rich_media_block) suggestions, rich_media_block)
return (file_path, search_keyword, url_list, think_content, response_content, suggestions) return (file_path, search_keyword, url_list, think_content, response_content, suggestions)
except Exception as e: except Exception as e:
# traceback.print_exc()
parts = file_path.split('/') parts = file_path.split('/')
platform = parts[2] platform = parts[2]
task_id = parts[1] task_id = parts[1]
...@@ -234,6 +231,6 @@ def qianwen_android_process_original_data(file_path): ...@@ -234,6 +231,6 @@ def qianwen_android_process_original_data(file_path):
if __name__ == '__main__': if __name__ == '__main__':
file_path2 = 'geo/e7b374d8-60c3-4658-98a2-64c72b6c521b/TYQWA/original.text' file_path2 = 'geo/0a2ed990b43a4f389ecdc14d185a3866/TYQWA/original.text'
# file_path2 = 'geo/ef586bf6-55d8-4e2a-8fb0-5fd1beb5bf7c/TYQWA/original.text' # file_path2 = 'geo/ef586bf6-55d8-4e2a-8fb0-5fd1beb5bf7c/TYQWA/original.text'
qianwen_android_process_original_data(file_path2) qianwen_android_process_original_data(file_path2)
aidso_geo/models/spider_save_tos.py
View file @ 8f636361
...@@ -141,13 +141,13 @@ def doubao_process_quote(url_list): ...@@ -141,13 +141,13 @@ def doubao_process_quote(url_list):
for item in url_list: for item in url_list:
raw_data = { raw_data = {
"url": item.get('text_card').get('url', ''), "url": item.get('text_card',{}).get('url', '') or item.get('video_card',{}).get('main_site_url'),
"title": item.get('text_card').get('title', ''), "title": item.get('text_card',{}).get('title', '') or item.get('video_card',{}).get('video_captions'),
"snippet": item.get('text_card').get('summary', ''), "snippet": item.get('text_card',{}).get('summary', '') or item.get('video_card',{}).get('summary',''),
"index": item.get('text_card').get('index', ''), "index": item.get('text_card',{}).get('index', '') or item.get('video_card',{}).get('index', ''),
"published_at": item.get('text_card').get('publish_time_second', ''), "published_at": item.get('text_card',{}).get('publish_time_second', '') or item.get('video_card',{}).get('publish_time_second', ''),
"site_name": item.get('text_card').get('sitename', ''), "site_name": item.get('text_card',{}).get('sitename', '') or item.get('video_card',{}).get('source_app_name', ''),
"site_icon": item.get('text_card').get('logo_url', ''), "site_icon": item.get('text_card',{}).get('logo_url', '') or item.get('video_card',{}).get('source_app_icon', ''),
} }
quto_list.append(raw_data) quto_list.append(raw_data)
return quto_list return quto_list
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment