refactor: class-based alpha factory + month-partitioned data pipeline
Replace the old signal/strategy/backtest modules with a decoupled
data → alpha → combo pipeline (parquet between phases, .pq extension).
Alphas:
- BaseAlpha + @register_alpha factory/plugin registry; one file per
built-in (reversal, reversal_vol, momentum); external alphas via
--alpha-module. Alphas are z-scored position weights, not predictors.
Data:
- baostock primary / akshare fallback, treated consistently.
- New --universe all (~5000 A-shares via query_all_stock, filtered).
- login-once batch downloader; empty-string OHLCV coerced to NaN.
- Month-partitioned dataset {output_dir}/{universe}/month=YYYY-MM/*.pq
with chunked durability flushes; --data-path is the dataset dir.
CLI logs at INFO by default (--log-level) so progress is visible.
Docs (README, CLAUDE.md) updated incl. pipeline diagram and roadmap
TODOs for portfolio construction / backtest / paper trading.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Yuxuan Yan
@@ -0,0 +1,180 @@
|
||||
"""Tests for pipeline alpha computation and combination (no network)."""
|
||||
import textwrap
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from pipeline.alpha.base import BaseAlpha
|
||||
from pipeline.alpha.compute import compute_alpha, evaluate_alpha
|
||||
from pipeline.alpha.registry import (
|
||||
available_alphas,
|
||||
get_alpha,
|
||||
load_alpha_module,
|
||||
register_alpha,
|
||||
)
|
||||
from pipeline.combo.combine import combine_alphas, _equal_weight
|
||||
from pipeline.common.schema import ALPHA_COLUMNS, COMBO_COLUMNS
|
||||
|
||||
|
||||
def _make_data(n_days: int = 30, symbols=("sh600000", "sz000001", "sh600519")) -> pd.DataFrame:
|
||||
"""Build a synthetic long-format DATA_COLUMNS frame with deterministic prices."""
|
||||
dates = pd.date_range("2024-01-01", periods=n_days)
|
||||
rng = np.random.default_rng(0)
|
||||
frames = []
|
||||
for i, sym in enumerate(symbols):
|
||||
# Distinct drift per symbol so the cross-section is non-degenerate.
|
||||
close = 100.0 + i * 5 + np.cumsum(rng.standard_normal(n_days))
|
||||
frames.append(pd.DataFrame({
|
||||
"symbol_id": sym,
|
||||
"symbol_name": sym,
|
||||
"date": dates,
|
||||
"open": close,
|
||||
"high": close,
|
||||
"low": close,
|
||||
"close": close,
|
||||
"volume": 1_000.0,
|
||||
"amount": 1_000.0 * close,
|
||||
}))
|
||||
return pd.concat(frames, ignore_index=True)
|
||||
|
||||
|
||||
def test_compute_alpha_schema_and_naming():
|
||||
alpha = compute_alpha(_make_data(), "rev5", "reversal", lookback=5)
|
||||
assert list(alpha.columns) == ALPHA_COLUMNS
|
||||
assert (alpha["alpha_name"] == "rev5").all()
|
||||
|
||||
|
||||
def test_reversal_sign_matches_negative_trailing_return():
|
||||
# Cross-sectional z-score preserves the sign relative to the cross-section,
|
||||
# so the stock with the most negative trailing return ranks highest.
|
||||
data = _make_data()
|
||||
alpha = compute_alpha(data, "rev5", "reversal", lookback=5)
|
||||
close = data.pivot_table(index="date", columns="symbol_id", values="close").sort_index()
|
||||
raw = -close.pct_change(5)
|
||||
last = raw.index[-1]
|
||||
expected_top = raw.loc[last].idxmax()
|
||||
got = alpha[alpha["date"] == last].set_index("symbol_id")["weight"].idxmax()
|
||||
assert got == expected_top
|
||||
|
||||
|
||||
def test_weights_are_cross_sectional_zscore():
|
||||
# Each date's weights are a z-score, so the per-date mean is ~0.
|
||||
alpha = compute_alpha(_make_data(), "rev5", "reversal", lookback=5)
|
||||
per_date_mean = alpha.groupby("date")["weight"].mean().abs()
|
||||
assert (per_date_mean < 1e-9).all()
|
||||
|
||||
|
||||
def test_evaluate_alpha_keys():
|
||||
data = _make_data()
|
||||
alpha = compute_alpha(data, "rev5", "reversal", lookback=5)
|
||||
metrics = evaluate_alpha(alpha, data)
|
||||
for key in ("cumulative_return", "sharpe_annual", "turnover_annual",
|
||||
"max_drawdown", "hit_rate", "n_dates"):
|
||||
assert key in metrics
|
||||
|
||||
|
||||
def test_equal_weight_is_mean_of_alphas():
|
||||
data = _make_data()
|
||||
a = compute_alpha(data, "rev", "reversal", lookback=5)
|
||||
b = compute_alpha(data, "mom", "momentum", lookback=5)
|
||||
combo = _equal_weight([a, b])
|
||||
# reversal = -momentum before z-scoring, but after independent per-date
|
||||
# z-scoring they are exact negatives, so the equal-weight mean is ~0.
|
||||
assert combo["weight"].abs().max() < 1e-9
|
||||
|
||||
|
||||
def test_combine_alphas_schema(tmp_path):
|
||||
data = _make_data()
|
||||
a_path = tmp_path / "a.pq"
|
||||
b_path = tmp_path / "b.pq"
|
||||
compute_alpha(data, "rev", "reversal", lookback=5).to_parquet(a_path, index=False)
|
||||
compute_alpha(data, "revvol", "reversal_vol", lookback=5, vol_window=10).to_parquet(b_path, index=False)
|
||||
combo = combine_alphas([str(a_path), str(b_path)], "eq", method="equal_weight")
|
||||
assert list(combo.columns) == COMBO_COLUMNS
|
||||
assert (combo["combo_name"] == "eq").all()
|
||||
|
||||
|
||||
# --- registry / factory -----------------------------------------------------
|
||||
|
||||
def test_builtins_are_registered():
|
||||
assert {"reversal", "reversal_vol", "momentum"} <= set(available_alphas())
|
||||
|
||||
|
||||
def test_get_alpha_filters_unaccepted_params():
|
||||
# reversal only accepts lookback; passing vol_window too must not error.
|
||||
alpha = get_alpha("reversal", lookback=7, vol_window=99)
|
||||
assert alpha.name == "reversal"
|
||||
assert alpha.lookback == 7
|
||||
assert not hasattr(alpha, "vol_window")
|
||||
|
||||
|
||||
def test_get_alpha_unknown_raises():
|
||||
with pytest.raises(KeyError):
|
||||
get_alpha("does_not_exist")
|
||||
|
||||
|
||||
def test_register_duplicate_name_raises():
|
||||
available_alphas() # ensure built-ins loaded
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
@register_alpha
|
||||
class Dup(BaseAlpha):
|
||||
name = "reversal"
|
||||
|
||||
def signal(self, close):
|
||||
return close
|
||||
|
||||
|
||||
def test_register_rejects_non_basealpha():
|
||||
with pytest.raises(TypeError):
|
||||
register_alpha(object) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# --- base class --------------------------------------------------------------
|
||||
|
||||
def test_to_weights_are_per_date_zscore():
|
||||
class _Const(BaseAlpha):
|
||||
name = "_const_test"
|
||||
|
||||
def signal(self, close):
|
||||
return close # arbitrary finite signal
|
||||
|
||||
close = _make_data().pivot_table(index="date", columns="symbol_id", values="close")
|
||||
weights = _Const().weights(close.sort_index())
|
||||
# Each date demeaned to ~0.
|
||||
assert (weights.mean(axis=1).abs() < 1e-9).all()
|
||||
|
||||
|
||||
# --- external plugin loading -------------------------------------------------
|
||||
|
||||
def test_load_external_alpha_module(tmp_path):
|
||||
module_path = tmp_path / "my_external_alpha.py"
|
||||
module_path.write_text(textwrap.dedent('''
|
||||
import pandas as pd
|
||||
from pipeline.alpha.base import BaseAlpha
|
||||
from pipeline.alpha.registry import register_alpha
|
||||
|
||||
@register_alpha
|
||||
class ExternalDemoAlpha(BaseAlpha):
|
||||
name = "external_demo"
|
||||
|
||||
def __init__(self, span: int = 3):
|
||||
self.span = span
|
||||
|
||||
def signal(self, close: pd.DataFrame) -> pd.DataFrame:
|
||||
return -close.pct_change(self.span)
|
||||
'''))
|
||||
|
||||
load_alpha_module(str(module_path))
|
||||
assert "external_demo" in available_alphas()
|
||||
|
||||
# The factory forwards the external alpha's own param (`span`).
|
||||
instance = get_alpha("external_demo", span=4, lookback=99)
|
||||
assert instance.span == 4
|
||||
|
||||
# And it works end-to-end through compute_alpha.
|
||||
result = compute_alpha(_make_data(), "ext", "external_demo", span=4)
|
||||
assert list(result.columns) == ALPHA_COLUMNS
|
||||
assert (result["alpha_name"] == "ext").all()
|
||||
|
||||
@@ -11,8 +11,22 @@ def test_download_single_stock():
|
||||
assert df["close"].notna().all()
|
||||
|
||||
|
||||
def test_download_baostock_fallback():
|
||||
"""Test baostock works as secondary source."""
|
||||
def test_download_baostock_primary():
|
||||
"""baostock is the primary source for 'auto'."""
|
||||
df = download_daily("sz000001", "2024-06-01", "2024-06-15", source="baostock")
|
||||
assert df is not None
|
||||
assert len(df) > 0
|
||||
|
||||
|
||||
def test_download_akshare_fallback():
|
||||
"""akshare works as the secondary source when reachable.
|
||||
|
||||
akshare is the fallback precisely because it is unreliable on some
|
||||
networks; skip rather than fail when it cannot be reached.
|
||||
"""
|
||||
try:
|
||||
df = download_daily("sh600000", "2024-01-01", "2024-01-31", source="akshare")
|
||||
except RuntimeError as e:
|
||||
pytest.skip(f"akshare unreachable on this network: {e}")
|
||||
assert df is not None
|
||||
assert len(df) > 0
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
"""Tests for cross-sectional IC evaluation."""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from eval.metrics import evaluate_cross_sectional
|
||||
|
||||
|
||||
def test_cross_sectional_keys_present():
|
||||
dates = pd.date_range("2024-01-01", periods=10)
|
||||
cols = ["a", "b", "c"]
|
||||
rng = np.random.default_rng(0)
|
||||
signals = pd.DataFrame(rng.standard_normal((10, 3)), index=dates, columns=cols)
|
||||
returns = pd.DataFrame(rng.standard_normal((10, 3)), index=dates, columns=cols)
|
||||
res = evaluate_cross_sectional(signals, returns)
|
||||
for key in (
|
||||
"ic_mean", "ic_std", "ir", "rank_ic_mean", "rank_ic_std",
|
||||
"rank_ir", "hit_rate", "n_periods",
|
||||
):
|
||||
assert key in res
|
||||
|
||||
|
||||
def test_perfect_signal_has_positive_rank_ic():
|
||||
# When the signal equals next-period returns, rank IC should be ~1 each day.
|
||||
dates = pd.date_range("2024-01-01", periods=8)
|
||||
cols = ["a", "b", "c"]
|
||||
rng = np.random.default_rng(42)
|
||||
returns = pd.DataFrame(rng.standard_normal((8, 3)), index=dates, columns=cols)
|
||||
signals = returns.copy() # perfect foresight
|
||||
res = evaluate_cross_sectional(signals, returns)
|
||||
assert res["rank_ic_mean"] > 0.99
|
||||
assert res["hit_rate"] == 1.0
|
||||
assert res["n_periods"] == 8
|
||||
|
||||
|
||||
def test_inverted_signal_has_negative_rank_ic():
|
||||
dates = pd.date_range("2024-01-01", periods=6)
|
||||
cols = ["a", "b", "c"]
|
||||
rng = np.random.default_rng(7)
|
||||
returns = pd.DataFrame(rng.standard_normal((6, 3)), index=dates, columns=cols)
|
||||
signals = -returns # perfectly wrong
|
||||
res = evaluate_cross_sectional(signals, returns)
|
||||
assert res["rank_ic_mean"] < -0.99
|
||||
|
||||
|
||||
def test_single_stock_falls_back_to_rolling():
|
||||
dates = pd.date_range("2024-01-01", periods=40)
|
||||
rng = np.random.default_rng(1)
|
||||
signals = pd.DataFrame({"a": rng.standard_normal(40)}, index=dates)
|
||||
returns = pd.DataFrame({"a": rng.standard_normal(40)}, index=dates)
|
||||
res = evaluate_cross_sectional(signals, returns)
|
||||
# Rolling fallback still yields the standard metric keys.
|
||||
assert "rank_ic_mean" in res
|
||||
assert res["n_periods"] > 0
|
||||
@@ -1,21 +0,0 @@
|
||||
import pytest
|
||||
from backtest.config import BacktestConfig
|
||||
from backtest.runner import BacktestRunner
|
||||
from strategies.reversal import FiveDayReversal
|
||||
|
||||
|
||||
def test_reversal_smoke():
|
||||
"""Smoke test: run a minimal reversal backtest and check results exist."""
|
||||
config = BacktestConfig(
|
||||
symbols=["sh600000"],
|
||||
start_date="2024-01-01",
|
||||
end_date="2024-03-31",
|
||||
initial_cash=100_000,
|
||||
)
|
||||
runner = BacktestRunner(config)
|
||||
results = runner.run(FiveDayReversal)
|
||||
assert results is not None
|
||||
assert len(results) == 1
|
||||
# Check analyzers exist
|
||||
sharpe = results[0].analyzers.sharpe.get_analysis()
|
||||
assert "sharperatio" in sharpe
|
||||
@@ -1,21 +0,0 @@
|
||||
import pytest
|
||||
from backtest.config import BacktestConfig
|
||||
from backtest.runner import BacktestRunner
|
||||
from strategies.base import SmaCross
|
||||
|
||||
|
||||
def test_backtest_smoke():
|
||||
"""Smoke test: run a minimal backtest and check results exist."""
|
||||
config = BacktestConfig(
|
||||
symbols=["sh600000"],
|
||||
start_date="2024-01-01",
|
||||
end_date="2024-03-31",
|
||||
initial_cash=100_000,
|
||||
)
|
||||
runner = BacktestRunner(config)
|
||||
results = runner.run(SmaCross)
|
||||
assert results is not None
|
||||
assert len(results) == 1
|
||||
# Check analyzers exist
|
||||
sharpe = results[0].analyzers.sharpe.get_analysis()
|
||||
assert "sharperatio" in sharpe
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Tests for alpha signal computation."""
|
||||
import pandas as pd
|
||||
|
||||
from signals.reversal import ReversalSignal
|
||||
|
||||
|
||||
def _make_df(closes):
|
||||
return pd.DataFrame({"close": closes})
|
||||
|
||||
|
||||
def test_reversal_name():
|
||||
assert ReversalSignal(lookback=5).name == "reversal_5d"
|
||||
assert ReversalSignal(lookback=10).name == "reversal_10d"
|
||||
|
||||
|
||||
def test_reversal_is_negative_trailing_return():
|
||||
# Monotonically rising prices -> negative (bearish) reversal signal.
|
||||
df = _make_df([10.0, 11.0, 12.0, 13.0, 14.0, 15.0])
|
||||
sig = ReversalSignal(lookback=5).compute(df)
|
||||
# First 5 values are NaN (insufficient lookback).
|
||||
assert sig.iloc[:5].isna().all()
|
||||
# 15/10 - 1 = 0.5 return -> signal = -0.5
|
||||
assert abs(sig.iloc[5] - (-0.5)) < 1e-9
|
||||
|
||||
|
||||
def test_reversal_oversold_is_positive():
|
||||
# Falling prices -> positive (bullish) reversal signal.
|
||||
df = _make_df([20.0, 18.0, 16.0, 14.0, 12.0, 10.0])
|
||||
sig = ReversalSignal(lookback=5).compute(df)
|
||||
assert sig.iloc[5] > 0
|
||||
# 10/20 - 1 = -0.5 -> signal = +0.5
|
||||
assert abs(sig.iloc[5] - 0.5) < 1e-9
|
||||
|
||||
|
||||
def test_reversal_output_length_matches_input():
|
||||
df = _make_df([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])
|
||||
sig = ReversalSignal(lookback=3).compute(df)
|
||||
assert len(sig) == len(df)
|
||||
Reference in New Issue
Block a user