diff --git a/CLAUDE.md b/CLAUDE.md index 4071d0f..e6e0c47 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,7 @@ An **alpha** is a signed cross-sectional **position weight**: positive = long, n ## Parquet schema contracts `pipeline/common/schema.py` defines the column contracts that are the *only* interface between phases. Any new phase or alpha must conform: -- `DATA_COLUMNS` (data output): `symbol_id, symbol_name, date, open, high, low, close, volume, amount` +- `DATA_COLUMNS` (data output): `symbol_id, symbol_name, date, open, high, low, close, preclose, volume, amount, vwap, turn, pctChg, tradestatus, isST, peTTM, pbMRQ, psTTM, pcfNcfTTM` (`vwap` = `amount/volume` is a raw-price daily VWAP, *not* on the adjusted OHLC scale under qfq/hfq). The richer fields are fetched only by the **batch** path (`download_daily_batch` → `download_universe`); single-symbol `download_daily` keeps the legacy 8-column schema that `tests/test_downloader.py` pins. - `ALPHA_COLUMNS` (alpha output): `symbol_id, date, alpha_name, weight` - `COMBO_COLUMNS` (combo output): `symbol_id, date, combo_name, weight` diff --git a/README.md b/README.md index 96fbb05..8825153 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,11 @@ supplies it and the unrelated `--lookback`/`--vol-window` defaults are ignored. The column contracts in `pipeline/common/schema.py` are the only interface between phases (data is stored long/tidy): -- **data** (`DATA_COLUMNS`): `symbol_id, symbol_name, date, open, high, low, close, volume, amount` +- **data** (`DATA_COLUMNS`): `symbol_id, symbol_name, date, open, high, low, close, preclose, volume, amount, vwap, turn, pctChg, tradestatus, isST, peTTM, pbMRQ, psTTM, pcfNcfTTM` + (`vwap` = `amount / volume` — a **raw**-price daily VWAP, *not* on the adjusted + OHLC scale under `qfq`/`hfq`; `turn` is turnover %, `pctChg` daily % change, + `tradestatus`/`isST` are 0/1 flags, and `peTTM`/`pbMRQ`/`psTTM`/`pcfNcfTTM` are + baostock valuation ratios.) - **alpha** (`ALPHA_COLUMNS`): `symbol_id, date, alpha_name, weight` - **combo** (`COMBO_COLUMNS`): `symbol_id, date, combo_name, weight` @@ -258,6 +262,10 @@ execution modeling. The following phases are planned but not built yet: richer P&L / risk attribution than `alpha eval`. - [ ] **Forward / paper trading** — run the same construction logic on live daily data, track simulated fills and a running P&L without real capital. +- [ ] **Intraday / microstructure data** — bid/ask prices & sizes, mid-price, + and intraday VWAP. These need a tick / L1–L2 quote feed (typically a paid or + brokerage data tier); the free daily sources here only expose daily bars, so + this is a separate data phase rather than extra columns on the daily schema. Until these land, treat `alpha eval` as a fast sanity check on a weight series, not a performance estimate. diff --git a/data/downloader.py b/data/downloader.py index 2c15734..29d2786 100644 --- a/data/downloader.py +++ b/data/downloader.py @@ -9,8 +9,27 @@ logger = logging.getLogger(__name__) # Map the adjust argument to baostock's adjustflag codes. _BAOSTOCK_ADJUST = {"qfq": "2", "hfq": "1", "": "3", "none": "3"} -_BAOSTOCK_FIELDS = "date,open,high,low,close,volume,amount" -_OHLCV = ["open", "high", "low", "close", "volume", "amount"] + +# Richer field set requested by the batch downloader. On top of OHLCV+amount we +# pull baostock's preclose, turnover rate, daily % change, trade/ST status, and +# the four valuation ratios, then derive a daily VWAP (amount / volume). +_BATCH_FIELDS = ( + "date,open,high,low,close,preclose,volume,amount,turn,pctChg," + "tradestatus,isST,peTTM,pbMRQ,psTTM,pcfNcfTTM" +) +# Every batch field except ``date`` is numeric (flags included: 0/1 strings). +_BATCH_NUMERIC = [ + "open", "high", "low", "close", "preclose", "volume", "amount", + "turn", "pctChg", "tradestatus", "isST", + "peTTM", "pbMRQ", "psTTM", "pcfNcfTTM", +] +# Output column order; ``vwap`` is derived (inserted right after ``amount``). +_BATCH_COLUMNS = [ + "symbol", "date", + "open", "high", "low", "close", "preclose", "volume", "amount", "vwap", + "turn", "pctChg", "tradestatus", "isST", + "peTTM", "pbMRQ", "psTTM", "pcfNcfTTM", +] class _SessionLost(Exception): @@ -163,7 +182,7 @@ def download_daily_batch( """One baostock query; returns df, or None (no data), or raises _SessionLost.""" code = f"{symbol[:2]}.{symbol[2:]}" rs = bs.query_history_k_data_plus( - code=code, fields=_BAOSTOCK_FIELDS, + code=code, fields=_BATCH_FIELDS, start_date=start, end_date=end, frequency="d", adjustflag=flag, ) if rs.error_code != "0": @@ -176,12 +195,14 @@ def download_daily_batch( rows.append(rs.get_row_data()) if not rows: return None - df = pd.DataFrame(rows, columns=["date", *_OHLCV]) + df = pd.DataFrame(rows, columns=["date", *_BATCH_NUMERIC]) # Suspended-trading days come back as empty strings; coerce to NaN # rather than crashing the whole symbol. - df[_OHLCV] = df[_OHLCV].apply(pd.to_numeric, errors="coerce") + df[_BATCH_NUMERIC] = df[_BATCH_NUMERIC].apply(pd.to_numeric, errors="coerce") + # Daily VWAP = turnover (yuan) / shares; NaN when no volume (suspended). + df["vwap"] = (df["amount"] / df["volume"]).where(df["volume"] > 0) df["symbol"] = symbol - return df[["symbol", "date", *_OHLCV]] + return df[_BATCH_COLUMNS] bs.login() try: diff --git a/pipeline/common/schema.py b/pipeline/common/schema.py index f13d90a..85bea8d 100644 --- a/pipeline/common/schema.py +++ b/pipeline/common/schema.py @@ -4,15 +4,26 @@ from typing import Final # Required columns for data parquet files (daily bars, alternative data, etc.) DATA_COLUMNS: Final[list[str]] = [ - "symbol_id", # str: internal code like 'sh600000' - "symbol_name", # str: stock name like '浦发银行' - "date", # date - "open", # float64 - "high", # float64 - "low", # float64 - "close", # float64 - "volume", # float64 (shares) - "amount", # float64 (turnover in yuan) + "symbol_id", # str: internal code like 'sh600000' + "symbol_name", # str: stock name like '浦发银行' + "date", # date + "open", # float64 + "high", # float64 + "low", # float64 + "close", # float64 + "preclose", # float64: previous trading day's close + "volume", # float64 (shares) + "amount", # float64 (turnover in yuan, raw/unadjusted) + "vwap", # float64: daily VWAP = amount / volume. NB raw price scale — + # NOT comparable with adjusted OHLC under qfq/hfq. + "turn", # float64: turnover rate (%) + "pctChg", # float64: daily % change + "tradestatus", # int: 1 = traded, 0 = suspended + "isST", # int: 1 = ST/special-treatment, 0 = normal + "peTTM", # float64: trailing-12m P/E + "pbMRQ", # float64: P/B (most recent quarter) + "psTTM", # float64: trailing-12m P/S + "pcfNcfTTM", # float64: P/CF (net cash flow, TTM) ] # Required columns for alpha parquet files.