royy/chinese-equity-quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: class-based alpha factory + month-partitioned data pipeline

Browse Source 
Replace the old signal/strategy/backtest modules with a decoupled
data → alpha → combo pipeline (parquet between phases, .pq extension).

Alphas:
- BaseAlpha + @register_alpha factory/plugin registry; one file per
  built-in (reversal, reversal_vol, momentum); external alphas via
  --alpha-module. Alphas are z-scored position weights, not predictors.

Data:
- baostock primary / akshare fallback, treated consistently.
- New --universe all (~5000 A-shares via query_all_stock, filtered).
- login-once batch downloader; empty-string OHLCV coerced to NaN.
- Month-partitioned dataset {output_dir}/{universe}/month=YYYY-MM/*.pq
  with chunked durability flushes; --data-path is the dataset dir.

CLI logs at INFO by default (--log-level) so progress is visible.
Docs (README, CLAUDE.md) updated incl. pipeline diagram and roadmap
TODOs for portfolio construction / backtest / paper trading.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Yuxuan Yan
2026-06-09 14:07:07 +08:00
parent 769cf25daa
commit 1caa63faeb
54 changed files with 1640 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/downloader.py
+80 -28
View File
@@ -1,14 +1,16 @@
"""Unified data downloader: akshare primary, baostock fallback."""
"""Unified data downloader: baostock primary, akshare fallback."""
import logging
from datetime import date, datetime
from typing import Optional
from typing import Iterable, Iterator, Optional, Tuple
import pandas as pd
import akshare as ak
import baostock as bs
logger = logging.getLogger(__name__)
BAOSTOCK_FREQ_MAP = {"d": "d", "w": "w", "m": "m"} # baostock only supports daily
# Map the adjust argument to baostock's adjustflag codes.
_BAOSTOCK_ADJUST = {"qfq": "2", "hfq": "1", "": "3", "none": "3"}
_BAOSTOCK_FIELDS = "date,open,high,low,close,volume,amount"
_OHLCV = ["open", "high", "low", "close", "volume", "amount"]
def _download_akshare(symbol: str, start: str, end: str, adjust: str = "qfq") -> Optional[pd.DataFrame]:
@@ -44,8 +46,8 @@ def _download_akshare(symbol: str, start: str, end: str, adjust: str = "qfq") ->
return None
def _download_baostock(symbol: str, start: str, end: str, frequency: str = "d") -> Optional[pd.DataFrame]:
"""Download daily bars from baostock as fallback."""
def _download_baostock(symbol: str, start: str, end: str, adjust: str = "qfq") -> Optional[pd.DataFrame]:
"""Download daily bars from baostock (primary source)."""
try:
bs.login()
# baostock format: sh.600000
@@ -55,8 +57,8 @@ def _download_baostock(symbol: str, start: str, end: str, frequency: str = "d")
fields="date,open,high,low,close,volume,amount",
start_date=start,
end_date=end,
frequency=frequency,
adjustflag="2", # qfq
frequency="d",
adjustflag=_BAOSTOCK_ADJUST.get(adjust, "2"),
)
if rs.error_code != "0":
logger.warning(f"baostock error for {symbol}: {rs.error_msg}")
@@ -64,22 +66,22 @@ def _download_baostock(symbol: str, start: str, end: str, frequency: str = "d")
data_list = []
while rs.next():
data_list.append(rs.get_row_data())
bs.logout()
if not data_list:
return None
df = pd.DataFrame(data_list, columns=["date", "open", "high", "low", "close", "volume", "amount"])
df[["open", "high", "low", "close", "volume", "amount"]] = df[
["open", "high", "low", "close", "volume", "amount"]
].astype(float)
].apply(pd.to_numeric, errors="coerce")
df["symbol"] = symbol
return df[["symbol", "date", "open", "high", "low", "close", "volume", "amount"]]
except Exception as e:
logger.warning(f"baostock download failed for {symbol}: {e}")
return None
finally:
try:
bs.logout()
except Exception:
pass
return None
def download_daily(
@@ -90,24 +92,24 @@ def download_daily(
source: str = "auto",
) -> pd.DataFrame:
"""
Download daily OHLCV data. Tries akshare first, falls back to baostock.
Download daily OHLCV data. Tries baostock first, falls back to akshare.
Args:
symbol: Stock symbol like 'sh600000' or 'sz000001'
start: Start date 'YYYY-MM-DD'
end: End date 'YYYY-MM-DD'
adjust: 'qfq' (forward-adjusted), 'hfq' (backward), '' (none)
source: 'auto' (akshare then baostock fallback), 'akshare' only,
or 'baostock' only
source: 'auto' (baostock then akshare fallback), 'baostock' only,
or 'akshare' only
Returns:
DataFrame with columns: symbol, date, open, high, low, close, volume, amount
"""
df = None
if source in ("akshare", "auto"):
if source in ("baostock", "auto"):
df = _download_baostock(symbol, start, end, adjust)
if df is None and source in ("akshare", "auto"):
df = _download_akshare(symbol, start, end, adjust)
if df is None and source in ("baostock", "auto"):
df = _download_baostock(symbol, start, end)
if df is None or df.empty:
raise RuntimeError(f"Failed to download data for {symbol} from {start} to {end}")
@@ -117,18 +119,68 @@ def download_daily(
return df
def download_batch(
symbols: list[str],
def download_daily_batch(
symbols: Iterable[str],
start: str,
end: str,
adjust: str = "qfq",
) -> dict[str, pd.DataFrame]:
"""Download daily data for multiple symbols. Returns {symbol: DataFrame}."""
results = {}
for sym in symbols:
akshare_fallback: bool = True,
) -> Iterator[Tuple[str, Optional[pd.DataFrame]]]:
"""Download many symbols under a single baostock session.
Logging into baostock once per call (instead of per symbol) is the dominant
speed-up when fetching thousands of symbols. Yields ``(symbol, df)`` as each
symbol completes so callers can stream results to disk; ``df`` is ``None``
when both sources fail. Each ``df`` has the same 8 columns as
:func:`download_daily`.
Args:
symbols: Internal-form symbols (``sh600000`` / ``sz000001``).
start, end: ``YYYY-MM-DD`` bounds.
adjust: ``qfq`` / ``hfq`` / ``''``.
akshare_fallback: Retry a failed symbol through akshare before yielding
``None``.
"""
flag = _BAOSTOCK_ADJUST.get(adjust, "2")
bs.login()
try:
for symbol in symbols:
df: Optional[pd.DataFrame] = None
try:
code = f"{symbol[:2]}.{symbol[2:]}"
rs = bs.query_history_k_data_plus(
code=code, fields=_BAOSTOCK_FIELDS,
start_date=start, end_date=end,
frequency="d", adjustflag=flag,
)
if rs.error_code == "0":
rows = []
while rs.next():
rows.append(rs.get_row_data())
if rows:
df = pd.DataFrame(rows, columns=["date", *_OHLCV])
# Suspended-trading days come back as empty strings;
# coerce to NaN rather than crashing the whole symbol.
df[_OHLCV] = df[_OHLCV].apply(pd.to_numeric, errors="coerce")
df["symbol"] = symbol
df = df[["symbol", "date", *_OHLCV]]
else:
logger.warning("baostock error for %s: %s", symbol, rs.error_msg)
except Exception as e:
logger.warning("baostock download failed for %s: %s", symbol, e)
if (df is None or df.empty) and akshare_fallback:
df = _download_akshare(symbol, start, end, adjust)
if df is not None and not df.empty:
df["date"] = pd.to_datetime(df["date"])
df = df.sort_values("date").reset_index(drop=True)
yield symbol, df
else:
yield symbol, None
finally:
try:
results[sym] = download_daily(sym, start, end, adjust)
logger.info(f"Downloaded {sym}: {len(results[sym])} bars")
except Exception as e:
logger.error(f"Failed {sym}: {e}")
return results
bs.logout()
except Exception:
pass