45 lines
1.5 KiB
Python
45 lines
1.5 KiB
Python
"""A股新闻采集 - AkShare stock_news_em"""
|
|
import warnings, logging, sqlite3
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
import pandas as pd
|
|
from src.storage.db import get_conn
|
|
import akshare as ak
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
|
log = logging.getLogger(__name__)
|
|
|
|
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
|
|
|
|
def fetch_news():
|
|
df = ak.stock_news_em(symbol='A股')
|
|
log.info(f'获取到 {len(df)} 行，列: {df.columns.tolist()}')
|
|
# 列名标准化
|
|
cn_map = {
|
|
'关键词': 'keywords',
|
|
'股票代码': 'ts_code',
|
|
'新闻标题': 'title',
|
|
'新闻内容': 'content',
|
|
'发布时间': 'pub_date',
|
|
'文章来源': 'source',
|
|
'新闻链接': 'url',
|
|
}
|
|
df = df.rename(columns=cn_map)
|
|
# 只保留表里有的列
|
|
db_cols = ['title', 'content', 'pub_date', 'source', 'url', 'keywords']
|
|
for c in db_cols:
|
|
if c not in df.columns:
|
|
df[c] = ''
|
|
df = df[['id'] + [c for c in db_cols if c in df.columns]] if 'id' in df.columns else df[db_cols]
|
|
df['inserted_at'] = datetime.now().isoformat()
|
|
if 'id' not in df.columns:
|
|
df.insert(0, 'id', range(1, len(df) + 1))
|
|
conn = get_conn()
|
|
df.to_sql('news_cn', conn, if_exists='replace', index=False)
|
|
conn.close()
|
|
log.info(f'写入 news_cn: {len(df)} 行')
|
|
return df
|
|
|
|
if __name__ == '__main__':
|
|
fetch_news()
|