chinese-equity-quant/data/downloader.py

"""Unified data downloader: baostock primary, akshare fallback."""
import logging
from typing import Iterable, Iterator, Optional, Tuple
import pandas as pd
import akshare as ak
import baostock as bs

logger = logging.getLogger(__name__)

# Map the adjust argument to baostock's adjustflag codes.
_BAOSTOCK_ADJUST = {"qfq": "2", "hfq": "1", "": "3", "none": "3"}

# Richer field set requested by the batch downloader. On top of OHLCV+amount we
# pull baostock's preclose, turnover rate, daily % change, trade/ST status, and
# the four valuation ratios, then derive a daily VWAP (amount / volume).
_BATCH_FIELDS = (
    "date,open,high,low,close,preclose,volume,amount,turn,pctChg,"
    "tradestatus,isST,peTTM,pbMRQ,psTTM,pcfNcfTTM"
)
# Every batch field except ``date`` is numeric (flags included: 0/1 strings).
_BATCH_NUMERIC = [
    "open", "high", "low", "close", "preclose", "volume", "amount",
    "turn", "pctChg", "tradestatus", "isST",
    "peTTM", "pbMRQ", "psTTM", "pcfNcfTTM",
]
# Output column order; ``vwap`` is derived (inserted right after ``amount``).
_BATCH_COLUMNS = [
    "symbol", "date",
    "open", "high", "low", "close", "preclose", "volume", "amount", "vwap",
    "turn", "pctChg", "tradestatus", "isST",
    "peTTM", "pbMRQ", "psTTM", "pcfNcfTTM",
]


class _SessionLost(Exception):
    """baostock reported the session was dropped (``用户未登录``)."""


def _download_akshare(symbol: str, start: str, end: str, adjust: str = "qfq") -> Optional[pd.DataFrame]:
    """Download daily bars from akshare. Returns DataFrame with OHLCV columns."""
    try:
        # symbol format: 'sh600000' in akshare stock_zh_a_hist expects raw code like '600000'
        # strip exchange prefix for akshare
        raw = symbol.replace("sh", "").replace("sz", "")
        df = ak.stock_zh_a_hist(
            symbol=raw,
            period="daily",
            start_date=start,
            end_date=end,
            adjust=adjust,
        )
        if df is None or df.empty:
            return None
        # Standardize columns
        col_map = {
            "日期": "date",
            "开盘": "open",
            "最高": "high",
            "最低": "low",
            "收盘": "close",
            "成交量": "volume",
            "成交额": "amount",
        }
        df = df.rename(columns=col_map)
        df["symbol"] = symbol
        return df[["symbol", "date", "open", "high", "low", "close", "volume", "amount"]]
    except Exception as e:
        logger.warning(f"akshare download failed for {symbol}: {e}")
        return None


def _download_baostock(symbol: str, start: str, end: str, adjust: str = "qfq") -> Optional[pd.DataFrame]:
    """Download daily bars from baostock (primary source)."""
    try:
        bs.login()
        # baostock format: sh.600000
        code = f"{symbol[:2]}.{symbol[2:]}"
        rs = bs.query_history_k_data_plus(
            code=code,
            fields="date,open,high,low,close,volume,amount",
            start_date=start,
            end_date=end,
            frequency="d",
            adjustflag=_BAOSTOCK_ADJUST.get(adjust, "2"),
        )
        if rs.error_code != "0":
            logger.warning(f"baostock error for {symbol}: {rs.error_msg}")
            return None
        data_list = []
        while rs.next():
            data_list.append(rs.get_row_data())
        if not data_list:
            return None
        df = pd.DataFrame(data_list, columns=["date", "open", "high", "low", "close", "volume", "amount"])
        df[["open", "high", "low", "close", "volume", "amount"]] = df[
            ["open", "high", "low", "close", "volume", "amount"]
        ].apply(pd.to_numeric, errors="coerce")
        df["symbol"] = symbol
        return df[["symbol", "date", "open", "high", "low", "close", "volume", "amount"]]
    except Exception as e:
        logger.warning(f"baostock download failed for {symbol}: {e}")
        return None
    finally:
        try:
            bs.logout()
        except Exception:
            pass


def download_daily(
    symbol: str,
    start: str,
    end: str,
    adjust: str = "qfq",
    source: str = "auto",
) -> pd.DataFrame:
    """
    Download daily OHLCV data. Tries baostock first, falls back to akshare.

    Args:
        symbol: Stock symbol like 'sh600000' or 'sz000001'
        start: Start date 'YYYY-MM-DD'
        end: End date 'YYYY-MM-DD'
        adjust: 'qfq' (forward-adjusted), 'hfq' (backward), '' (none)
        source: 'auto' (baostock then akshare fallback), 'baostock' only,
            or 'akshare' only

    Returns:
        DataFrame with columns: symbol, date, open, high, low, close, volume, amount
    """
    df = None
    if source in ("baostock", "auto"):
        df = _download_baostock(symbol, start, end, adjust)
    if df is None and source in ("akshare", "auto"):
        df = _download_akshare(symbol, start, end, adjust)

    if df is None or df.empty:
        raise RuntimeError(f"Failed to download data for {symbol} from {start} to {end}")

    df["date"] = pd.to_datetime(df["date"])
    df = df.sort_values("date").reset_index(drop=True)
    return df


def download_daily_batch(
    symbols: Iterable[str],
    start: str,
    end: str,
    adjust: str = "qfq",
    akshare_fallback: bool = False,
    relogin_every: int = 200,
) -> Iterator[Tuple[str, Optional[pd.DataFrame]]]:
    """Download many symbols, keeping a baostock session alive across the run.

    Logging in once (instead of per symbol) is the dominant speed-up for
    thousands of symbols, but baostock drops a session after a while
    (subsequent queries return ``用户未登录``). So we refresh the session every
    ``relogin_every`` symbols and also re-login + retry once whenever a query
    reports the session is gone. Yields ``(symbol, df)`` as each symbol
    completes; ``df`` is ``None`` when no data is available. Each ``df`` has the
    same 8 columns as :func:`download_daily`.

    Args:
        symbols: Internal-form symbols (``sh600000`` / ``sz000001``).
        start, end: ``YYYY-MM-DD`` bounds.
        adjust: ``qfq`` / ``hfq`` / ``''``.
        akshare_fallback: Retry a failed symbol through akshare. Off by default
            because akshare is unreliable on the deployment network and each
            failed attempt is slow; baostock + re-login is the fast path.
        relogin_every: Proactively refresh the baostock session every N symbols.
    """
    flag = _BAOSTOCK_ADJUST.get(adjust, "2")

    def _relogin() -> None:
        try:
            bs.logout()
        except Exception:
            pass
        bs.login()

    def _fetch(symbol: str) -> Optional[pd.DataFrame]:
        """One baostock query; returns df, or None (no data), or raises _SessionLost."""
        code = f"{symbol[:2]}.{symbol[2:]}"
        rs = bs.query_history_k_data_plus(
            code=code, fields=_BATCH_FIELDS,
            start_date=start, end_date=end, frequency="d", adjustflag=flag,
        )
        if rs.error_code != "0":
            if "未登录" in (rs.error_msg or ""):
                raise _SessionLost(rs.error_msg)
            logger.warning("baostock error for %s: %s", symbol, rs.error_msg)
            return None
        rows = []
        while rs.next():
            rows.append(rs.get_row_data())
        if not rows:
            return None
        df = pd.DataFrame(rows, columns=["date", *_BATCH_NUMERIC])
        # Suspended-trading days come back as empty strings; coerce to NaN
        # rather than crashing the whole symbol.
        df[_BATCH_NUMERIC] = df[_BATCH_NUMERIC].apply(pd.to_numeric, errors="coerce")
        # Daily VWAP = turnover (yuan) / shares; NaN when no volume (suspended).
        df["vwap"] = (df["amount"] / df["volume"]).where(df["volume"] > 0)
        df["symbol"] = symbol
        return df[_BATCH_COLUMNS]

    bs.login()
    try:
        for i, symbol in enumerate(symbols):
            if i and relogin_every and i % relogin_every == 0:
                _relogin()

            df: Optional[pd.DataFrame] = None
            for attempt in (1, 2):
                try:
                    df = _fetch(symbol)
                    break
                except _SessionLost:
                    if attempt == 1:
                        _relogin()  # session dropped — refresh and retry once
                        continue
                    logger.warning("baostock session lost for %s after relogin", symbol)
                except Exception as e:
                    logger.warning("baostock download failed for %s: %s", symbol, e)
                    break

            if (df is None or df.empty) and akshare_fallback:
                df = _download_akshare(symbol, start, end, adjust)

            if df is not None and not df.empty:
                df["date"] = pd.to_datetime(df["date"])
                df = df.sort_values("date").reset_index(drop=True)
                yield symbol, df
            else:
                yield symbol, None
    finally:
        try:
            bs.logout()
        except Exception:
            pass