fix: fetcher 数据库路径可移植化(db.py get_conn 统一,去除 Windows 硬编码)

This commit is contained in:
lookt
2026-09-15 21:25:23 +08:00
parent d03b4c0089
commit dee0e798f3
6 changed files with 46 additions and 9 deletions
+3 -2
View File
@@ -12,8 +12,9 @@ import pandas as pd
warnings.filterwarnings('ignore')
DB = r'C:' + chr(92) + 'Users' + chr(92) + 'lookt' + chr(92) + 'a_stock_timeline' + chr(92) + 'data' + chr(92) + 'a_stock.db'
OUT = r'E:' + chr(92) + 'Data' + chr(92) + 'skills' + chr(92) + 'a-stock-timeline-patterns' + chr(92) + 'references'
import os
DB = os.environ.get('MINE_DB', 'data/a_stock.db')
OUT = os.environ.get('MINE_OUT_DIR', '../skills-out/references')
IDX_NAMES = {'sh000001': '上证指数', 'sz399001': '深证成指', 'sh000300': '沪深300',
'sz399006': '创业板指', 'sh000688': '科创50'}
+4 -2
View File
@@ -1,13 +1,15 @@
"""交易日历采集 - AkShare tool_trade_date_hist_sina"""
import warnings, logging, sqlite3
from datetime import datetime
from pathlib import Path
import pandas as pd
from src.storage.db import get_conn
import akshare as ak
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
log = logging.getLogger(__name__)
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
def fetch_calendar():
df = ak.tool_trade_date_hist_sina()
@@ -17,7 +19,7 @@ def fetch_calendar():
df['trade_date'] = pd.to_datetime(df['trade_date'])
df['is_trade_day'] = df['trade_date'].dt.dayofweek.apply(lambda x: 0 if x >= 5 else 1)
df['trade_date'] = df['trade_date'].dt.strftime('%Y-%m-%d')
conn = sqlite3.connect(DB)
conn = get_conn()
df.to_sql('trade_calendar', conn, if_exists='replace', index=False)
conn.close()
log.info(f'写入 trade_calendar: {len(df)}')
+4 -2
View File
@@ -1,13 +1,15 @@
"""A股新闻采集 - AkShare stock_news_em"""
import warnings, logging, sqlite3
from datetime import datetime
from pathlib import Path
import pandas as pd
from src.storage.db import get_conn
import akshare as ak
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
log = logging.getLogger(__name__)
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
def fetch_news():
df = ak.stock_news_em(symbol='A股')
@@ -32,7 +34,7 @@ def fetch_news():
df['inserted_at'] = datetime.now().isoformat()
if 'id' not in df.columns:
df.insert(0, 'id', range(1, len(df) + 1))
conn = sqlite3.connect(DB)
conn = get_conn()
df.to_sql('news_cn', conn, if_exists='replace', index=False)
conn.close()
log.info(f'写入 news_cn: {len(df)}')
+2 -1
View File
@@ -4,6 +4,7 @@ import warnings
import logging
from urllib.parse import urlparse
from pathlib import Path
import pandas as pd
import requests
@@ -13,7 +14,7 @@ warnings.filterwarnings('ignore')
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
log = logging.getLogger(__name__)
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
# 出站请求域名白名单(SSRF 防护:仅 http + 白名单主机,禁重定向跟随)
_ALLOWED_HOSTS = {'data.10jqka.com.cn'}
+4 -2
View File
@@ -1,13 +1,15 @@
"""外盘指数采集 - AkShare index_us_stock_sina"""
import warnings, logging, sqlite3
from datetime import datetime
from pathlib import Path
import pandas as pd
from src.storage.db import get_conn
import akshare as ak
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
log = logging.getLogger(__name__)
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
# 要采集的外盘指数
INDICES = [
@@ -17,7 +19,7 @@ INDICES = [
]
def fetch_global_index():
conn = sqlite3.connect(DB)
conn = get_conn()
results = []
for code, symbol, name in INDICES:
try: