From 528620b2712e3eba56aec801a63760cae1f48728 Mon Sep 17 00:00:00 2001 From: Yuxuan Yan Date: Tue, 16 Jun 2026 21:10:30 +0800 Subject: [PATCH] Raise coverage threshold to 95% and expand test coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml: fail_under 80 → 95 - test_alpha: +79 lines - test_cli_workflow: +226 lines - test_derived: +121 lines - test_downloader_contracts: +169 lines - test_features: +16 lines - test_minute_downloader: +81 lines - test_portfolio: +208 lines --- pyproject.toml | 2 +- tests/test_alpha.py | 79 ++++++++++ tests/test_cli_workflow.py | 226 +++++++++++++++++++++++++++++ tests/test_derived.py | 121 ++++++++++++++- tests/test_downloader_contracts.py | 169 +++++++++++++++++++++ tests/test_features.py | 16 ++ tests/test_minute_downloader.py | 81 +++++++++++ tests/test_portfolio.py | 208 ++++++++++++++++++++++++++ 8 files changed, 898 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 342b772..eddada8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ source = [ ] [tool.coverage.report] -fail_under = 80 +fail_under = 95 show_missing = true skip_covered = false omit = [ diff --git a/tests/test_alpha.py b/tests/test_alpha.py index 4ac21f2..3332e02 100644 --- a/tests/test_alpha.py +++ b/tests/test_alpha.py @@ -191,6 +191,15 @@ def test_combine_single_alpha_is_identity(tmp_path): assert (combo["combo_name"] == "rev_combo").all() +def test_combine_alphas_rejects_unknown_method(tmp_path): + data = _make_data() + alpha_path = tmp_path / "alpha.pq" + compute_alpha(data, "rev", "reversal", lookback=5).to_parquet(alpha_path, index=False) + + with pytest.raises(ValueError, match="Unknown combo method"): + combine_alphas([str(alpha_path)], "bad_combo", method="does_not_exist") + + # --- registry / factory ----------------------------------------------------- def test_builtins_are_registered(): @@ -205,6 +214,22 @@ def test_get_alpha_filters_unaccepted_params(): assert not hasattr(alpha, "vol_window") +def test_get_alpha_forwards_kwargs_to_flexible_alpha(): + @register_alpha + class _FlexibleAlpha(BaseAlpha): + name = "_flexible_alpha_kwargs" + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def signal(self, close): + return close + + alpha = get_alpha("_flexible_alpha_kwargs", decay=0.5, label="demo") + + assert alpha.kwargs == {"decay": 0.5, "label": "demo"} + + def test_get_alpha_unknown_raises(): with pytest.raises(KeyError): get_alpha("does_not_exist") @@ -227,6 +252,31 @@ def test_register_rejects_non_basealpha(): register_alpha(object) # type: ignore[arg-type] +def test_register_rejects_empty_alpha_name(): + with pytest.raises(ValueError, match="non-empty"): + @register_alpha + class NoNameAlpha(BaseAlpha): + def signal(self, close): + return close + + +def test_load_alpha_module_error_paths(tmp_path, monkeypatch): + missing_path = tmp_path / "missing_alpha.py" + with pytest.raises(FileNotFoundError): + load_alpha_module(str(missing_path)) + + bad_path = tmp_path / "bad_alpha.py" + bad_path.write_text("x = 1\n") + monkeypatch.setattr( + "pipeline.alpha.registry.importlib.util.spec_from_file_location", + lambda *args, **kwargs: None, + ) + with pytest.raises(ImportError, match="Cannot load alpha module"): + load_alpha_module(str(bad_path)) + + load_alpha_module("math") + + # --- base class -------------------------------------------------------------- def test_to_weights_are_per_date_zscore(): @@ -242,6 +292,20 @@ def test_to_weights_are_per_date_zscore(): assert (weights.mean(axis=1).abs() < 1e-9).all() +def test_base_alpha_default_signal_and_repr(): + alpha = BaseAlpha() + alpha.example = 3 + + with pytest.raises(NotImplementedError, match="signal"): + alpha.signal(pd.DataFrame({"x": [1.0]})) + with pytest.raises(NotImplementedError, match="signal"): + alpha.signal_from_data( + pd.DataFrame({"symbol_id": ["sh600000"]}), + pd.DataFrame({"sh600000": [1.0]}), + ) + assert repr(alpha) == "BaseAlpha(example=3)" + + # --- external plugin loading ------------------------------------------------- def test_load_external_alpha_module(tmp_path): @@ -502,3 +566,18 @@ def test_compute_alpha_rejects_feature_path_collision_with_daily_data(tmp_path): "reversal", feature_paths=[str(close_collision_path)], ) + + +def test_evaluate_alpha_empty_when_signal_dates_not_on_market_calendar(): + data = _make_data(n_days=3) + alpha = pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [pd.Timestamp("2030-01-01")], + "alpha_name": ["future"], + "weight": [1.0], + }) + + metrics = evaluate_alpha(alpha, data) + + assert metrics["n_dates"] == 0 + assert metrics["cumulative_return"] == 0.0 diff --git a/tests/test_cli_workflow.py b/tests/test_cli_workflow.py index a8f4d06..79406c5 100644 --- a/tests/test_cli_workflow.py +++ b/tests/test_cli_workflow.py @@ -5,10 +5,13 @@ from __future__ import annotations import textwrap from pathlib import Path +import click import pandas as pd from click.testing import CliRunner from cli import cli +import pipeline.derived.cli as derived_cli +import pipeline.features.cli as features_cli from tests.helpers import ( make_generated_daily_bars, make_generated_derived_features, @@ -426,6 +429,61 @@ def test_cli_error_paths_are_clear_for_bad_user_inputs(tmp_path): ]) assert "Symbol 'sh999999' not found" in alphaview_missing_symbol.output + alphaview_missing_column = _invoke_error(runner, [ + "alphaview", + "--data-path", str(daily_path), + "--alpha-path", str(positions_path), + "--symbol", "sh600000", + "--columns", "close,missing_bar_col", + ]) + assert "Bar columns not found: missing_bar_col" in alphaview_missing_column.output + + alphaview_alpha = pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [daily_bars["date"].min()], + "alpha_name": ["toy_alpha"], + "weight": [1.0], + }) + alphaview_alpha_path = tmp_path / "alphaview_alpha.pq" + alphaview_alpha.to_parquet(alphaview_alpha_path, index=False) + alphaview_empty_range = _invoke_error(runner, [ + "alphaview", + "--data-path", str(daily_path), + "--alpha-path", str(alphaview_alpha_path), + "--symbol", "sh600000", + "--start-date", "2030-01-01", + ]) + assert "No rows in the requested date range" in alphaview_empty_range.output + + empty_combo_paths = _invoke_ok(runner, [ + "combo", "combine", + "--alpha-paths", " , ", + "--combo-name", "empty", + "--output-dir", str(tmp_path / "combos"), + ]) + assert "requires at least 1 path" in empty_combo_paths.output + + +def test_cli_parser_helpers_cover_string_coercion_and_bad_params(): + assert derived_cli._parse_params(("n=7", "scale=2.5", "label=demo")) == { + "n": 7, + "scale": 2.5, + "label": "demo", + } + assert features_cli._parse_params(("n=7", "scale=2.5", "label=demo")) == { + "n": 7, + "scale": 2.5, + "label": "demo", + } + + for module in (derived_cli, features_cli): + try: + module._parse_params(("not-an-assignment",)) + except click.BadParameter as exc: + assert "--param must be name=value" in str(exc) + else: + raise AssertionError("expected BadParameter") + def test_cli_list_and_legacy_feature_paths(tmp_path): runner = CliRunner() @@ -519,6 +577,174 @@ def test_cli_list_and_legacy_feature_paths(tmp_path): assert "Unknown feature-type" in unknown_feature.output +def test_cli_shortcuts_and_external_module_loading(tmp_path): + runner = CliRunner() + daily_bars = make_generated_daily_bars(n_sessions=8, include_missing=False) + minute_bars = make_generated_minute_bars(daily_bars) + daily_path = tmp_path / "daily_bars.pq" + minute_path = tmp_path / "minute_bars.pq" + daily_bars.to_parquet(daily_path, index=False) + minute_bars.to_parquet(minute_path, index=False) + + alpha_module = tmp_path / "listed_alpha.py" + alpha_module.write_text(textwrap.dedent(""" + import pandas as pd + from pipeline.alpha.base import BaseAlpha + from pipeline.alpha.registry import register_alpha + + @register_alpha + class ListedAlpha(BaseAlpha): + name = "listed_alpha_cli" + + def __init__(self, scale: float = 1.0, label: str = "x"): + self.scale = scale + self.label = label + + def signal(self, close: pd.DataFrame) -> pd.DataFrame: + return close.pct_change(1, fill_method=None) * self.scale + """)) + derived_module = tmp_path / "listed_derived.py" + derived_module.write_text(textwrap.dedent(""" + import pandas as pd + from pipeline.derived.base import BaseDerivedData + from pipeline.derived.registry import register_derived + + @register_derived + class ListedDerived(BaseDerivedData): + name = "listed_derived_cli" + + def __init__(self, scale: float = 1.0): + self.scale = scale + + def compute(self, daily=None, minute=None) -> pd.DataFrame: + out = daily[["symbol_id", "date", "close"]].copy() + out["listed_value"] = out.pop("close") * self.scale + return out + """)) + derived_compute_module = tmp_path / "computed_derived.py" + derived_compute_module.write_text(textwrap.dedent(""" + import pandas as pd + from pipeline.derived.base import BaseDerivedData + from pipeline.derived.registry import register_derived + + @register_derived + class ComputedDerived(BaseDerivedData): + name = "computed_derived_cli" + + def __init__(self, scale: float = 1.0): + self.scale = scale + + def compute(self, daily=None, minute=None) -> pd.DataFrame: + out = daily[["symbol_id", "date", "close"]].copy() + out["computed_value"] = out.pop("close") * self.scale + return out + """)) + feature_module = tmp_path / "listed_feature.py" + feature_module.write_text(textwrap.dedent(""" + import pandas as pd + from pipeline.features.base import BaseFeature + from pipeline.features.registry import register_feature + + @register_feature + class ListedFeature(BaseFeature): + name = "listed_feature_cli" + + def compute(self, daily=None, minute=None) -> pd.DataFrame: + out = minute[["symbol_id", "date", "close"]].copy() + out["date"] = pd.to_datetime(out["date"]).dt.normalize() + out = out.groupby(["symbol_id", "date"], as_index=False)["close"].mean() + return out.rename(columns={"close": "listed_feature_value"}) + """)) + feature_compute_module = tmp_path / "computed_feature.py" + feature_compute_module.write_text(textwrap.dedent(""" + import pandas as pd + from pipeline.features.base import BaseFeature + from pipeline.features.registry import register_feature + + @register_feature + class ComputedFeature(BaseFeature): + name = "computed_feature_cli" + + def compute(self, daily=None, minute=None) -> pd.DataFrame: + out = minute[["symbol_id", "date", "close"]].copy() + out["date"] = pd.to_datetime(out["date"]).dt.normalize() + out = out.groupby(["symbol_id", "date"], as_index=False)["close"].mean() + return out.rename(columns={"close": "computed_feature_value"}) + """)) + + alpha_list = _invoke_ok(runner, [ + "alpha", "list", + "--alpha-module", str(alpha_module), + ]) + assert "listed_alpha_cli" in alpha_list.output + + alpha_dir = tmp_path / "alphas" + reversal = _invoke_ok(runner, [ + "alpha", "reversal", + "--data-path", str(daily_path), + "--output-dir", str(alpha_dir), + "--lookback", "3", + ]) + reversal_vol = _invoke_ok(runner, [ + "alpha", "reversal-vol", + "--data-path", str(daily_path), + "--output-dir", str(alpha_dir), + "--lookback", "3", + "--vol-window", "3", + ]) + external_alpha = _invoke_ok(runner, [ + "alpha", "compute", + "--data-path", str(daily_path), + "--alpha-type", "listed_alpha_cli", + "--alpha-name", "listed_alpha_run", + "--param", "scale=2.5", + "--param", "label=demo", + "--output-dir", str(alpha_dir), + ]) + + assert "Saved alpha:" in reversal.output + assert "Saved alpha:" in reversal_vol.output + assert "Saved alpha:" in external_alpha.output + assert (alpha_dir / "reversal_3d.pq").exists() + assert (alpha_dir / "reversal_vol_3d_3d.pq").exists() + assert (alpha_dir / "listed_alpha_run.pq").exists() + + derived_list = _invoke_ok(runner, [ + "derived", "list", + "--derived-module", str(derived_module), + ]) + assert "listed_derived_cli" in derived_list.output + derived_dir = tmp_path / "derived_external" + derived_compute = _invoke_ok(runner, [ + "derived", "compute", + "--daily-path", str(daily_path), + "--derived-module", str(derived_compute_module), + "--derived-type", "computed_derived_cli", + "--derived-name", "listed_derived_run", + "--param", "scale=3", + "--output-dir", str(derived_dir), + ]) + assert "Saved derived data:" in derived_compute.output + assert (derived_dir / "listed_derived_run.pq").exists() + + feature_list = _invoke_ok(runner, [ + "feature", "list", + "--feature-module", str(feature_module), + ]) + assert "listed_feature_cli" in feature_list.output + feature_dir = tmp_path / "features_external" + feature_compute = _invoke_ok(runner, [ + "feature", "compute", + "--minute-path", str(minute_path), + "--feature-module", str(feature_compute_module), + "--feature-type", "computed_feature_cli", + "--feature-name", "listed_feature_run", + "--output-dir", str(feature_dir), + ]) + assert "Saved feature:" in feature_compute.output + assert (feature_dir / "listed_feature_run.pq").exists() + + def test_cli_pqcat_row_modes(tmp_path): runner = CliRunner() daily_bars = make_generated_daily_bars(n_sessions=3, include_missing=False) diff --git a/tests/test_derived.py b/tests/test_derived.py index 5e20223..2f9f446 100644 --- a/tests/test_derived.py +++ b/tests/test_derived.py @@ -9,8 +9,14 @@ from click.testing import CliRunner from cli import cli from pipeline.alpha.compute import join_feature_frames -from pipeline.derived.compute import compute_derived, validate_derived_frame -from pipeline.derived.registry import available_derived, get_derived, load_derived_module +from pipeline.derived.base import BaseDerivedData +from pipeline.derived.compute import compute_derived, read_derived_frame, validate_derived_frame +from pipeline.derived.registry import ( + available_derived, + get_derived, + load_derived_module, + register_derived, +) def _daily_bars() -> pd.DataFrame: @@ -88,6 +94,27 @@ def test_validate_derived_frame_rejects_non_numeric_values(): })) +def test_validate_derived_frame_rejects_missing_value_columns(): + with pytest.raises(ValueError, match="at least one value column"): + validate_derived_frame(pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [pd.Timestamp("2024-01-02")], + })) + + +def test_read_derived_frame_rejects_empty_csv(tmp_path): + empty_csv = tmp_path / "empty.csv" + empty_csv.write_text("") + + with pytest.raises(ValueError, match="CSV input is empty"): + read_derived_frame(empty_csv) + + +def test_compute_derived_rejects_missing_inputs(): + with pytest.raises(ValueError, match="requires --daily-path or --minute-path"): + compute_derived("minute_daily_summary") + + def test_derived_ingest_cli_accepts_csv_and_parquet(tmp_path): runner = CliRunner() source = pd.DataFrame({ @@ -136,6 +163,26 @@ def test_derived_validate_cli_rejects_duplicate_csv_columns(tmp_path): assert "duplicate columns" in result.output +def test_derived_ingest_cli_wraps_validation_errors(tmp_path): + runner = CliRunner() + csv_path = tmp_path / "bad_ingest.csv" + csv_path.write_text("symbol_id,date\nsh600000,2024-01-02\n") + + result = runner.invoke(cli, [ + "derived", + "ingest", + "--input-path", + str(csv_path), + "--derived-name", + "bad", + "--output-dir", + str(tmp_path / "derived"), + ]) + + assert result.exit_code != 0 + assert "at least one value column" in result.output + + def test_external_derived_plugin_loads_filters_params_and_uses_inputs(tmp_path): module_path = tmp_path / "external_derived.py" module_path.write_text(textwrap.dedent(''' @@ -204,6 +251,57 @@ def test_external_derived_plugin_loads_filters_params_and_uses_inputs(tmp_path): assert {"daily_scaled_close", "minute_volume_sum"}.issubset(both_result.columns) +def test_derived_registry_rejects_bad_plugins_and_load_failures(tmp_path, monkeypatch): + with pytest.raises(TypeError): + register_derived(object) # type: ignore[arg-type] + + with pytest.raises(ValueError, match="non-empty"): + @register_derived + class NoNameDerived(BaseDerivedData): + def compute(self, daily=None, minute=None): + return pd.DataFrame() + + @register_derived + class _CoverageDerived(BaseDerivedData): + name = "_coverage_derived_registry" + + def compute(self, daily=None, minute=None): + return pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [pd.Timestamp("2024-01-02")], + "x": [1.0], + }) + + with pytest.raises(ValueError, match="already registered"): + @register_derived + class _CoverageDerivedDuplicate(BaseDerivedData): + name = "_coverage_derived_registry" + + def compute(self, daily=None, minute=None): + return pd.DataFrame() + + with pytest.raises(KeyError, match="Unknown derived data"): + get_derived("does_not_exist") + + missing_path = tmp_path / "missing_derived.py" + with pytest.raises(FileNotFoundError): + load_derived_module(str(missing_path)) + + bad_path = tmp_path / "bad_derived.py" + bad_path.write_text("x = 1\n") + monkeypatch.setattr( + "pipeline.derived.registry.importlib.util.spec_from_file_location", + lambda *args, **kwargs: None, + ) + with pytest.raises(ImportError, match="Cannot load derived data module"): + load_derived_module(str(bad_path)) + + load_derived_module("math") + instance = _CoverageDerived() + instance.scale = 2 + assert repr(instance) == "_CoverageDerived(scale=2)" + + def test_derived_compute_cli_writes_builtin_minute_summary(tmp_path): runner = CliRunner() minute_path = tmp_path / "minute.pq" @@ -223,6 +321,24 @@ def test_derived_compute_cli_writes_builtin_minute_summary(tmp_path): assert "minute_vwap" in written.columns +def test_minute_summary_uses_time_sort_and_daily_without_close(): + minute = _minute_bars().drop(columns=["datetime"]).sample(frac=1.0, random_state=0) + daily = _daily_bars()[["symbol_id", "date", "open"]] + + result = compute_derived( + "minute_daily_summary", + daily=daily, + minute=minute, + ) + + by_symbol = result.set_index(["symbol_id", "date"]) + assert np.isclose( + by_symbol.loc[("sh600000", pd.Timestamp("2024-01-02")), "minute_intraday_return"], + 0.10, + ) + assert "minute_vwap_deviation" in result.columns + + def test_alpha_feature_join_rejects_derived_column_collisions(): data = _daily_bars() derived_a = data[["symbol_id", "date"]].copy() @@ -258,4 +374,3 @@ def test_legacy_feature_cli_delegates_to_derived_registry(tmp_path): ]) assert compute_result.exit_code == 0, compute_result.output assert (out_dir / "minute_summary.pq").exists() - diff --git a/tests/test_downloader_contracts.py b/tests/test_downloader_contracts.py index 1989ee8..7d32991 100644 --- a/tests/test_downloader_contracts.py +++ b/tests/test_downloader_contracts.py @@ -132,6 +132,41 @@ def test_download_daily_raises_when_requested_source_has_no_data(monkeypatch): ) +def test_download_daily_akshare_source_skips_baostock(monkeypatch): + calls: list[str] = [] + fallback = pd.DataFrame({ + "symbol": ["sh600000"], + "date": ["2024-01-02"], + "open": [10.0], + "high": [11.0], + "low": [9.0], + "close": [10.5], + "volume": [1000.0], + "amount": [10500.0], + }) + + monkeypatch.setattr( + downloader, + "_download_baostock", + lambda *args: calls.append("baostock") or None, + ) + monkeypatch.setattr( + downloader, + "_download_akshare", + lambda *args: calls.append("akshare") or fallback, + ) + + result = download_daily( + "sh600000", + "2024-01-02", + "2024-01-02", + source="akshare", + ) + + assert calls == ["akshare"] + assert result["date"].tolist() == [pd.Timestamp("2024-01-02")] + + def test_akshare_daily_downloader_maps_columns_and_failures(monkeypatch): calls: list[dict] = [] raw = pd.DataFrame({ @@ -326,6 +361,34 @@ def test_download_daily_batch_periodic_relogin_and_none_result(monkeypatch): assert logout_count == 2 +def test_download_daily_batch_empty_rows_yields_none(monkeypatch): + monkeypatch.setattr(downloader.bs, "login", lambda: None) + monkeypatch.setattr(downloader.bs, "logout", lambda: None) + monkeypatch.setattr( + downloader.bs, + "query_history_k_data_plus", + lambda **kwargs: _FakeResult([]), + ) + + assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [ + ("sh600000", None) + ] + + +def test_download_daily_batch_generic_exception_yields_none(monkeypatch): + monkeypatch.setattr(downloader.bs, "login", lambda: None) + monkeypatch.setattr(downloader.bs, "logout", lambda: None) + monkeypatch.setattr( + downloader.bs, + "query_history_k_data_plus", + lambda **kwargs: (_ for _ in ()).throw(RuntimeError("query failed")), + ) + + assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [ + ("sh600000", None) + ] + + def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch): responses = [ _FakeResult([], error_code="10002007", error_msg="用户未登录"), @@ -359,6 +422,32 @@ def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch): assert logout_count == 2 +def test_download_daily_batch_second_session_loss_and_logout_failure(monkeypatch): + responses = [ + _FakeResult([], error_code="10002007", error_msg="用户未登录"), + _FakeResult([], error_code="10002007", error_msg="用户未登录"), + ] + logout_count = 0 + + def fake_logout(): + nonlocal logout_count + logout_count += 1 + raise RuntimeError("logout failed") + + monkeypatch.setattr(downloader.bs, "login", lambda: None) + monkeypatch.setattr(downloader.bs, "logout", fake_logout) + monkeypatch.setattr( + downloader.bs, + "query_history_k_data_plus", + lambda **kwargs: responses.pop(0), + ) + + assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [ + ("sh600000", None) + ] + assert logout_count == 2 + + def test_download_daily_batch_uses_akshare_fallback_when_enabled(monkeypatch): fallback = pd.DataFrame({ "symbol": ["sh600000"], @@ -438,6 +527,10 @@ def test_download_universe_writes_daily_partitions_from_mock_batch(tmp_path, mon monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch) + stale_file = tmp_path / "toy" / "month=2024-01" / "stale.pq" + stale_file.parent.mkdir(parents=True) + batch_frame.iloc[[0]][DATA_COLUMNS[2:]].to_parquet(stale_file, index=False) + stats = download_universe( universe="toy", start_date="2024-01-02", @@ -458,5 +551,81 @@ def test_download_universe_writes_daily_partitions_from_mock_batch(tmp_path, mon } assert (dataset_path / "month=2024-01").exists() assert (dataset_path / "month=2024-02").exists() + assert not stale_file.exists() assert written[DATA_COLUMNS].columns.tolist() == DATA_COLUMNS assert written["symbol_name"].tolist() == ["PF Bank", "PF Bank"] + + +def test_download_universe_raises_when_all_daily_symbols_empty(tmp_path, monkeypatch): + monkeypatch.setattr( + pipeline_downloader, + "_resolve_universe", + lambda universe, max_symbols=0: pd.DataFrame({ + "symbol_id": ["sh600000"], + "symbol_name": ["PF Bank"], + }), + ) + monkeypatch.setattr( + pipeline_downloader, + "download_daily_batch", + lambda symbols, start, end, adjust="qfq": iter([("sh600000", None)]), + ) + + with pytest.raises(RuntimeError, match="No data downloaded"): + download_universe( + universe="toy", + start_date="2024-01-02", + end_date="2024-01-02", + output_dir=str(tmp_path), + ) + + +def test_download_universe_progress_branch_at_100_symbols(tmp_path, monkeypatch): + symbols = [f"sh6{i:05d}" for i in range(100)] + batch_frame = pd.DataFrame({ + "symbol": ["sh600000"], + "date": [pd.Timestamp("2024-01-02")], + "open": [10.0], + "high": [11.0], + "low": [9.0], + "close": [10.5], + "preclose": [10.0], + "volume": [1000.0], + "amount": [10500.0], + "vwap": [10.5], + "turn": [1.0], + "pctChg": [5.0], + "tradestatus": [1], + "isST": [0], + "peTTM": [8.0], + "pbMRQ": [1.1], + "psTTM": [2.1], + "pcfNcfTTM": [3.1], + }) + + monkeypatch.setattr( + pipeline_downloader, + "_resolve_universe", + lambda universe, max_symbols=0: pd.DataFrame({ + "symbol_id": symbols, + "symbol_name": symbols, + }), + ) + + def fake_batch(requested_symbols, start, end, adjust="qfq"): + assert requested_symbols == symbols + for symbol in requested_symbols: + yield symbol, batch_frame.copy() + + monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch) + + stats = download_universe( + universe="toy100", + start_date="2024-01-02", + end_date="2024-01-02", + output_dir=str(tmp_path), + chunk_size=200, + ) + + assert stats["n_symbols"] == 100 + assert stats["n_rows"] == 100 diff --git a/tests/test_features.py b/tests/test_features.py index 248ee3d..b719259 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -7,6 +7,7 @@ import pandas as pd import pytest from pipeline.features.compute import compute_feature, validate_feature_frame +from pipeline.features.compute import read_feature_frames from pipeline.features.library.minute_daily_summary import MinuteDailySummaryFeature from pipeline.features.registry import ( available_features, @@ -99,6 +100,21 @@ def test_legacy_feature_compute_matches_canonical_derived_compute(): pd.testing.assert_frame_equal(legacy_feature, canonical_derived) +def test_read_feature_frames_delegates_to_derived_validation(tmp_path): + feature = pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": ["2024-01-02 15:00:00"], + "toy_feature": [1.5], + }) + feature_path = tmp_path / "feature.pq" + feature.to_parquet(feature_path, index=False) + + [result] = read_feature_frames([feature_path]) + + assert result["date"].tolist() == [pd.Timestamp("2024-01-02")] + assert result["toy_feature"].tolist() == [1.5] + + def test_load_external_feature_module_and_filter_params(tmp_path): module_path = tmp_path / "external_feature.py" module_path.write_text(textwrap.dedent(''' diff --git a/tests/test_minute_downloader.py b/tests/test_minute_downloader.py index 26b81c6..36d9fd7 100644 --- a/tests/test_minute_downloader.py +++ b/tests/test_minute_downloader.py @@ -178,6 +178,37 @@ def test_download_minute_batch_second_session_loss_yields_none(monkeypatch): ] +def test_download_minute_batch_ignores_relogin_and_final_logout_failures(monkeypatch): + responses = [ + _FakeResult([], error_code="1", error_msg="bad symbol"), + _FakeResult([]), + ] + logout_count = 0 + + def fake_logout(): + nonlocal logout_count + logout_count += 1 + raise RuntimeError("logout failed") + + monkeypatch.setattr(low_level_downloader.bs, "login", lambda: None) + monkeypatch.setattr(low_level_downloader.bs, "logout", fake_logout) + monkeypatch.setattr( + low_level_downloader.bs, + "query_history_k_data_plus", + lambda **kwargs: responses.pop(0), + ) + + assert list( + download_minute_batch( + ["sh600000", "sz000001"], + "2024-01-02", + "2024-01-02", + relogin_every=1, + ) + ) == [("sh600000", None), ("sz000001", None)] + assert logout_count == 2 + + def test_download_minute_batch_rejects_unparsed_timestamps(monkeypatch): bad_rows = [[ "2024-01-02", @@ -244,6 +275,9 @@ def test_download_minute_universe_writes_frequency_month_partitions(tmp_path, mo preserved_minute["symbol_id"] = "sh600000" preserved_minute["symbol_name"] = "PF Bank" preserved_minute[MINUTE_BAR_COLUMNS].to_parquet(preserved, index=False) + stale = tmp_path / "toy" / "frequency=5m" / "month=2024-01" / "stale.pq" + stale.parent.mkdir(parents=True) + preserved_minute.assign(frequency="5m")[MINUTE_BAR_COLUMNS].to_parquet(stale, index=False) stats = download_minute_universe( universe="toy", @@ -257,6 +291,7 @@ def test_download_minute_universe_writes_frequency_month_partitions(tmp_path, mo dataset_path = Path(stats["dataset_path"]) assert (dataset_path / "frequency=5m" / "month=2024-01").is_dir() assert preserved.exists() + assert not stale.exists() out = pd.read_parquet(dataset_path / "frequency=5m") assert (set(MINUTE_BAR_COLUMNS) - {"frequency"}) <= set(out.columns) assert set(out["symbol_id"]) == {"sh600000"} @@ -286,3 +321,49 @@ def test_download_minute_universe_raises_when_all_symbols_empty(tmp_path, monkey end_date="2024-01-02", output_dir=str(tmp_path), ) + + +def test_download_minute_universe_progress_branch_at_100_symbols(tmp_path, monkeypatch): + symbols = [f"sh6{i:05d}" for i in range(100)] + minute = pd.DataFrame({ + "symbol": ["sh600000"], + "datetime": [pd.Timestamp("2024-01-02 09:35:00")], + "date": [pd.Timestamp("2024-01-02")], + "time": ["09:35:00"], + "frequency": ["5m"], + "open": [10.0], + "high": [11.0], + "low": [9.0], + "close": [10.5], + "volume": [1000.0], + "amount": [10500.0], + "vwap": [10.5], + "adjustflag": ["3"], + }) + + monkeypatch.setattr( + pipeline_downloader, + "_resolve_universe", + lambda universe, max_symbols=0: pd.DataFrame({ + "symbol_id": symbols, + "symbol_name": symbols, + }), + ) + + def fake_batch(requested_symbols, start, end, frequency=5): + assert requested_symbols == symbols + for symbol in requested_symbols: + yield symbol, minute.copy() + + monkeypatch.setattr(pipeline_downloader, "download_minute_batch", fake_batch) + + stats = download_minute_universe( + universe="toy100", + start_date="2024-01-02", + end_date="2024-01-02", + output_dir=str(tmp_path), + chunk_size=200, + ) + + assert stats["n_symbols"] == 100 + assert stats["n_rows"] == 100 diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py index 7b3999b..dcb86f0 100644 --- a/tests/test_portfolio.py +++ b/tests/test_portfolio.py @@ -13,12 +13,17 @@ from pipeline.portfolio.market_rules import ( Board, LimitStatus, MarketRule, + _to_date, compute_limit_status, detect_board, ) from pipeline.portfolio.research import evaluate_portfolio from pipeline.portfolio.constraints import ( + TradeConstraint, + available_constraints, + get_constraint, PriceLimitConstraint, + register_constraint, SuspensionConstraint, VolumeCapConstraint, ) @@ -82,6 +87,7 @@ def test_detect_board(): assert detect_board("sh688981") == Board.STAR assert detect_board("sz300750") == Board.CHINEXT assert detect_board("bj830000") == Board.UNKNOWN + assert detect_board("sh") == Board.UNKNOWN # --- MarketRule date transitions --------------------------------------------- @@ -120,6 +126,26 @@ def test_get_rules_vectorized(): assert list(limit) == [0.10, 0.20, 0.20] +def test_market_rule_date_coercion_unknown_board_and_st_vector_override(): + rules = MarketRule() + + assert _to_date(dt.datetime(2024, 1, 2, 15, 0)) == dt.date(2024, 1, 2) + assert _to_date(pd.Timestamp("2024-01-03 09:30")) == dt.date(2024, 1, 3) + assert _to_date("2024-01-04 10:00:00") == dt.date(2024, 1, 4) + + unknown_rule = rules.get_rule("xx999999", "2024-01-02") + assert unknown_rule.minimum_open_size == 100 + assert unknown_rule.share_increment == 100 + assert unknown_rule.price_limit_pct == 0.10 + + _, _, _, limit = rules.get_rules_vectorized( + np.array(["sh600000", "sh600000"], dtype=object), + "2024-01-02", + np.array([0, 1]), + ) + assert limit.tolist() == [0.10, 0.05] + + def test_compute_limit_status(): price = np.array([110.0, 90.0, 100.0]) preclose = np.array([100.0, 100.0, 100.0]) @@ -130,6 +156,20 @@ def test_compute_limit_status(): assert status[2] == LimitStatus.NORMAL.value +def test_compute_limit_status_treats_bad_preclose_as_normal(): + status = compute_limit_status( + price=np.array([np.nan, 110.0, 90.0]), + preclose=np.array([100.0, np.nan, 0.0]), + limit_pct=np.array([0.10, 0.10, 0.10]), + ) + + assert status.tolist() == [ + LimitStatus.NORMAL.value, + LimitStatus.NORMAL.value, + LimitStatus.NORMAL.value, + ] + + # --- continuous targets ------------------------------------------------------ def test_continuous_targets_normalization(): @@ -295,6 +335,70 @@ def test_repair_scales_to_4000_names(): assert abs(gross - B) <= 0.03 * B +def test_repair_handles_empty_input(): + result = repair_exposure( + np.array([], dtype=np.int64), + np.array([], dtype=float), + np.array([], dtype=float), + np.array([], dtype=np.int64), + np.array([], dtype=np.int64), + np.array([], dtype=np.int64), + ) + + assert result.dtype == np.int64 + assert result.tolist() == [] + + +def test_repair_respects_max_iters_and_zero_increment_noop(): + q_round = np.array([100, -100], dtype=np.int64) + q_target = np.array([300.0, -300.0]) + price = np.array([10.0, 10.0]) + min_open = np.array([100, 100]) + prev = np.zeros(2, dtype=np.int64) + + capped = repair_exposure( + q_round, + q_target, + price, + increment=np.array([1, 1]), + min_open=min_open, + prev_shares=prev, + booksize=10_000.0, + gross_tol=0.0, + max_iters=0, + ) + zero_increment = repair_exposure( + q_round, + q_target, + price, + increment=np.array([0, 0]), + min_open=min_open, + prev_shares=prev, + booksize=0.0, + net_tol=0.0, + gross_tol=0.0, + ) + + assert capped.tolist() == [100, -100] + assert zero_increment.tolist() == [100, -100] + + +def test_repair_gross_growth_obeys_net_band(): + pos = repair_exposure( + q_round=np.array([100], dtype=np.int64), + q_target=np.array([300.0]), + price=np.array([10.0]), + increment=np.array([1]), + min_open=np.array([100]), + prev_shares=np.array([0], dtype=np.int64), + booksize=2_000.0, + net_tol=0.5, + gross_tol=0.0, + ) + + assert pos.tolist() == [100] + + # --- construct_positions ----------------------------------------------------- def test_construct_positions_schema(): @@ -497,8 +601,81 @@ def test_constraints_compose_repeatably_regardless_of_order(): assert np.array_equal(first_order.cost, reversed_order.cost) +def test_constraint_registry_and_default_adjust_targets(): + class _NoopConstraint(TradeConstraint): + name = "_coverage_noop_constraint" + + def delta_bounds(self, ctx): + return np.zeros(1), np.ones(1) + + registered = register_constraint(_NoopConstraint) + + assert registered is _NoopConstraint + assert "_coverage_noop_constraint" in available_constraints() + assert isinstance(get_constraint("_coverage_noop_constraint"), _NoopConstraint) + assert get_constraint("_coverage_noop_constraint").adjust_targets(object()) is None + with np.testing.assert_raises(KeyError): + get_constraint("_missing_constraint") + with np.testing.assert_raises(TypeError): + register_constraint(object) # type: ignore[arg-type] + with np.testing.assert_raises(ValueError): + class _NoNameConstraint(TradeConstraint): + def delta_bounds(self, ctx): + return np.zeros(1), np.ones(1) + + register_constraint(_NoNameConstraint) + with np.testing.assert_raises(ValueError): + class _DuplicateConstraint(TradeConstraint): + name = "_coverage_noop_constraint" + + def delta_bounds(self, ctx): + return np.zeros(1), np.ones(1) + + register_constraint(_DuplicateConstraint) + + # --- ReferenceSimulator ------------------------------------------------------ +def test_simulator_applies_constraint_target_adjustment(): + class _HalveTarget(TradeConstraint): + name = "_halve_target" + + def adjust_targets(self, ctx): + return ctx.target_shares // 2 + + def delta_bounds(self, ctx): + return np.full(len(ctx.target_shares), -np.inf), np.full(len(ctx.target_shares), np.inf) + + sl = _slice(1, price=np.array([10.0])) + ctx = TradeContext(np.array([0], np.int64), np.array([100], np.int64), sl, 1e6) + + result = ReferenceSimulator(constraints=[_HalveTarget()]).fill(ctx) + + assert result.traded_shares.tolist() == [50] + assert result.realized_shares.tolist() == [50] + + +def test_simulator_empty_positions_uses_default_booksize(): + data = pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [pd.Timestamp("2024-01-02")], + "open": [10.0], + "close": [10.0], + "preclose": [10.0], + "amount": [1e9], + "tradestatus": [1], + "isST": [0], + }) + positions = pd.DataFrame(columns=POSITION_COLUMNS) + + fills, pnl = ReferenceSimulator().run(positions, data) + + assert list(fills.columns) == FILL_COLUMNS + assert list(pnl.columns) == PNL_COLUMNS + assert fills.empty + assert pnl.empty + + def test_simulator_next_open_and_blocked_buy_holds_prev(): data = _make_data(n_days=15) weights = _make_weights(data) @@ -749,3 +926,34 @@ def test_evaluate_portfolio_excludes_signal_without_forward_return(): metrics = evaluate_portfolio(positions, data) assert metrics["n_dates"] == 1 + + +def test_evaluate_portfolio_empty_and_single_return_paths(): + empty_metrics = evaluate_portfolio( + pd.DataFrame(columns=POSITION_COLUMNS), + pd.DataFrame(columns=["symbol_id", "date", "open"]), + ) + assert empty_metrics["n_dates"] == 0 + assert empty_metrics["cumulative_return"] == 0.0 + + dates = pd.date_range("2024-01-01", periods=3) + data = pd.DataFrame([ + {"symbol_id": "sh600000", "date": d, "open": price} + for d, price in zip(dates, [100.0, 100.0, 110.0]) + ]) + positions = pd.DataFrame({ + "symbol_id": ["sh600000"], + "date": [dates[0]], + "portfolio_name": ["single"], + "target_weight": [1.0], + "target_value": [1000.0], + "target_shares": [10.0], + "position_shares": [10], + "position_value": [1000.0], + "price": [100.0], + }) + + single_metrics = evaluate_portfolio(positions, data) + + assert single_metrics["n_dates"] == 1 + assert single_metrics["cumulative_return"] == 0.0