fix: fetcher 数据库路径可移植化(db.py get_conn 统一,去除 Windows 硬编码)
This commit is contained in:
@@ -12,8 +12,9 @@ import pandas as pd
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
DB = r'C:' + chr(92) + 'Users' + chr(92) + 'lookt' + chr(92) + 'a_stock_timeline' + chr(92) + 'data' + chr(92) + 'a_stock.db'
|
||||
OUT = r'E:' + chr(92) + 'Data' + chr(92) + 'skills' + chr(92) + 'a-stock-timeline-patterns' + chr(92) + 'references'
|
||||
import os
|
||||
DB = os.environ.get('MINE_DB', 'data/a_stock.db')
|
||||
OUT = os.environ.get('MINE_OUT_DIR', '../skills-out/references')
|
||||
|
||||
IDX_NAMES = {'sh000001': '上证指数', 'sz399001': '深证成指', 'sh000300': '沪深300',
|
||||
'sz399006': '创业板指', 'sh000688': '科创50'}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
"""交易日历采集 - AkShare tool_trade_date_hist_sina"""
|
||||
import warnings, logging, sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
from src.storage.db import get_conn
|
||||
import akshare as ak
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
|
||||
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
|
||||
|
||||
def fetch_calendar():
|
||||
df = ak.tool_trade_date_hist_sina()
|
||||
@@ -17,7 +19,7 @@ def fetch_calendar():
|
||||
df['trade_date'] = pd.to_datetime(df['trade_date'])
|
||||
df['is_trade_day'] = df['trade_date'].dt.dayofweek.apply(lambda x: 0 if x >= 5 else 1)
|
||||
df['trade_date'] = df['trade_date'].dt.strftime('%Y-%m-%d')
|
||||
conn = sqlite3.connect(DB)
|
||||
conn = get_conn()
|
||||
df.to_sql('trade_calendar', conn, if_exists='replace', index=False)
|
||||
conn.close()
|
||||
log.info(f'写入 trade_calendar: {len(df)} 行')
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
"""A股新闻采集 - AkShare stock_news_em"""
|
||||
import warnings, logging, sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
from src.storage.db import get_conn
|
||||
import akshare as ak
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
|
||||
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
|
||||
|
||||
def fetch_news():
|
||||
df = ak.stock_news_em(symbol='A股')
|
||||
@@ -32,7 +34,7 @@ def fetch_news():
|
||||
df['inserted_at'] = datetime.now().isoformat()
|
||||
if 'id' not in df.columns:
|
||||
df.insert(0, 'id', range(1, len(df) + 1))
|
||||
conn = sqlite3.connect(DB)
|
||||
conn = get_conn()
|
||||
df.to_sql('news_cn', conn, if_exists='replace', index=False)
|
||||
conn.close()
|
||||
log.info(f'写入 news_cn: {len(df)} 行')
|
||||
|
||||
@@ -4,6 +4,7 @@ import warnings
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
@@ -13,7 +14,7 @@ warnings.filterwarnings('ignore')
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
|
||||
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
|
||||
|
||||
# 出站请求域名白名单(SSRF 防护:仅 http + 白名单主机,禁重定向跟随)
|
||||
_ALLOWED_HOSTS = {'data.10jqka.com.cn'}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
"""外盘指数采集 - AkShare index_us_stock_sina"""
|
||||
import warnings, logging, sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
from src.storage.db import get_conn
|
||||
import akshare as ak
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB = r'C:\Users\lookt\a_stock_timeline\data\a_stock.db'
|
||||
DB = str(Path(__file__).resolve().parent.parent.parent / 'data' / 'a_stock.db')
|
||||
|
||||
# 要采集的外盘指数
|
||||
INDICES = [
|
||||
@@ -17,7 +19,7 @@ INDICES = [
|
||||
]
|
||||
|
||||
def fetch_global_index():
|
||||
conn = sqlite3.connect(DB)
|
||||
conn = get_conn()
|
||||
results = []
|
||||
for code, symbol, name in INDICES:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user