royy/chinese-equity-quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add daily derived data pipeline

Browse Source
This commit is contained in:
Yuxuan Yan
2026-06-16 15:55:30 +08:00
parent 83a006bbe4
commit 8d908477e2
19 changed files with 897 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/alpha/cli.py
+1 -1
View File
@@ -58,7 +58,7 @@ def list_(alpha_modules):
@click.option("--vol-window", default=20, type=int, help="Volatility window (reversal_vol only)")
@click.option(
"--feature-path", "feature_paths", multiple=True,
help="Daily feature parquet file/dataset to left-join on symbol_id,date (repeatable)",
help="Daily derived/feature parquet file or dataset to left-join on symbol_id,date (repeatable)",
)
@click.option(
"--alpha-module", "alpha_modules", multiple=True,
pipeline/alpha/compute.py
+10 -6
View File
@@ -15,7 +15,11 @@ import pandas as pd
from pipeline.alpha.registry import get_alpha
from pipeline.common.schema import ALPHA_COLUMNS
from pipeline.features.compute import FEATURE_KEY_COLUMNS, read_feature_frames, validate_feature_frame
from pipeline.derived.compute import (
DERIVED_KEY_COLUMNS,
read_derived_frames,
validate_derived_frame,
)
logger = logging.getLogger(__name__)
@@ -40,15 +44,15 @@ def join_feature_frames(
data: pd.DataFrame,
feature_frames: Iterable[pd.DataFrame],
) -> pd.DataFrame:
"""Left-join validated daily feature frames onto long daily data."""
"""Left-join validated daily derived/feature frames onto long daily data."""
out = data.copy()
out["date"] = pd.to_datetime(out["date"])
existing = set(out.columns)
joined_cols: list[str] = []
for frame in feature_frames:
features = validate_feature_frame(frame)
feature_cols = [col for col in features.columns if col not in FEATURE_KEY_COLUMNS]
features = validate_derived_frame(frame)
feature_cols = [col for col in features.columns if col not in DERIVED_KEY_COLUMNS]
overlap = sorted(existing.intersection(feature_cols))
if overlap:
raise ValueError(
@@ -56,7 +60,7 @@ def join_feature_frames(
)
out = out.merge(
features,
on=FEATURE_KEY_COLUMNS,
on=DERIVED_KEY_COLUMNS,
how="left",
validate="many_to_one",
)
@@ -171,7 +175,7 @@ def compute_alpha(
"""
feature_inputs: list[pd.DataFrame] = []
if feature_paths:
feature_inputs.extend(read_feature_frames(feature_paths))
feature_inputs.extend(read_derived_frames(feature_paths))
if feature_frames:
feature_inputs.extend(feature_frames)
if feature_inputs:
pipeline/common/schema.py
+7
View File
@@ -44,6 +44,13 @@ MINUTE_BAR_COLUMNS: Final[list[str]] = [
"adjustflag", # str: baostock adjustment flag; '3' for raw/unadjusted
]
# Required key columns for daily derived-data parquet files. Value columns are
# user/plugin-defined and must be numeric.
DERIVED_KEY_COLUMNS: Final[list[str]] = [
"symbol_id", # str
"date", # date: normalized daily timestamp
]
# Required columns for alpha parquet files.
# Alphas are position WEIGHTS: positive=long, negative=short.
ALPHA_COLUMNS: Final[list[str]] = [
pipeline/derived/__init__.py
+2
View File
@@ -0,0 +1,2 @@
"""Daily derived-data plugin package."""
pipeline/derived/base.py
+38
View File
@@ -0,0 +1,38 @@
"""Base class for daily derived-data plugins."""
from abc import ABC, abstractmethod
import pandas as pd
class BaseDerivedData(ABC):
"""Compute daily, symbol-keyed numeric derived data.
Derived-data plugins may use daily bars, minute bars, or both as inputs, but
they must always return daily rows keyed by ``symbol_id,date``.
"""
#: Unique registry key. Every concrete derived-data plugin must set this.
name: str = ""
@abstractmethod
def compute(
self,
daily: pd.DataFrame | None = None,
minute: pd.DataFrame | None = None,
) -> pd.DataFrame:
"""Compute daily derived data.
Args:
daily: Optional daily market data.
minute: Optional raw minute bars.
Returns:
DataFrame with ``symbol_id``, ``date``, and one or more numeric
derived-data columns.
"""
def __repr__(self) -> str:
params = ", ".join(f"{k}={v!r}" for k, v in vars(self).items())
return f"{type(self).__name__}({params})"
pipeline/derived/cli.py
+145
View File
@@ -0,0 +1,145 @@
"""CLI for daily derived-data ingestion and computation."""
import click
import pandas as pd
from pipeline.derived.compute import (
DERIVED_KEY_COLUMNS,
compute_derived,
read_derived_frame,
write_derived_frame,
)
from pipeline.derived.registry import (
available_derived,
load_derived_module,
)
@click.group(name="derived")
def derived():
"""Ingest, compute, and validate daily derived data."""
def _coerce(value: str):
"""Best-effort coercion of a CLI string to int, then float, else str."""
for cast in (int, float):
try:
return cast(value)
except ValueError:
continue
return value
def _parse_params(pairs: tuple[str, ...]) -> dict:
"""Parse repeated ``name=value`` options into a params dict."""
params: dict = {}
for pair in pairs:
if "=" not in pair:
raise click.BadParameter(f"--param must be name=value, got '{pair}'")
key, value = pair.split("=", 1)
params[key.strip()] = _coerce(value.strip())
return params
def _read_optional_parquet(path: str | None) -> pd.DataFrame | None:
return None if path is None else pd.read_parquet(path)
def _summarize(result: pd.DataFrame) -> str:
value_cols = [col for col in result.columns if col not in DERIVED_KEY_COLUMNS]
return f"{len(result):,} rows, {len(value_cols)} columns"
@derived.command("list")
@click.option(
"--derived-module", "derived_modules", multiple=True,
help="External module(s) to import first (dotted path or .py file)",
)
def list_(derived_modules):
"""List the registered derived-data plugin types."""
for spec in derived_modules:
load_derived_module(spec)
for name in available_derived():
click.echo(name)
@derived.command("validate")
@click.option("--input-path", required=True, help="CSV/parquet file or parquet dataset to validate")
def validate(input_path):
"""Validate a daily derived-data file without writing output."""
try:
result = read_derived_frame(input_path)
except Exception as exc:
raise click.ClickException(str(exc)) from exc
click.echo(f"Valid derived data: {input_path} ({_summarize(result)})")
@derived.command("ingest")
@click.option("--input-path", required=True, help="CSV/parquet file to ingest")
@click.option("--derived-name", required=True, help="Name for this derived-data output file")
@click.option("--output-dir", default="derived", help="Directory to save derived parquet")
def ingest(input_path, derived_name, output_dir):
"""Ingest a user-provided daily derived-data CSV/parquet file."""
try:
result = read_derived_frame(input_path)
out_path = write_derived_frame(result, derived_name, output_dir=output_dir)
except Exception as exc:
raise click.ClickException(str(exc)) from exc
click.echo(f"Saved derived data: {out_path} ({_summarize(result)})")
@derived.command("compute")
@click.option("--daily-path", default=None, help="Optional daily data parquet/dataset")
@click.option("--minute-path", default=None, help="Optional minute parquet/dataset")
@click.option("--derived-type", required=True, help="Registry key of the derived-data plugin")
@click.option("--derived-name", required=True, help="Name for this derived-data output file")
@click.option("--output-dir", default="derived", help="Directory to save derived parquet")
@click.option(
"--derived-module", "derived_modules", multiple=True,
help="External module(s) to import so their derived-data plugins register",
)
@click.option(
"--param", "extra_params", multiple=True,
help="Extra derived-data constructor param as name=value (repeatable)",
)
def compute(
daily_path,
minute_path,
derived_type,
derived_name,
output_dir,
derived_modules,
extra_params,
):
"""Compute one daily derived-data file from daily and/or minute inputs."""
for spec in derived_modules:
load_derived_module(spec)
options = available_derived()
if derived_type not in options:
raise click.BadParameter(
f"Unknown derived-type '{derived_type}'. Available: {options}. "
f"Use --derived-module to register an external derived-data plugin.",
param_hint="--derived-type",
)
if daily_path is None and minute_path is None:
raise click.UsageError("At least one of --daily-path or --minute-path is required")
daily = _read_optional_parquet(daily_path)
if daily_path:
click.echo(f"Loaded daily data: {len(daily):,} rows from {daily_path}")
minute = _read_optional_parquet(minute_path)
if minute_path:
click.echo(f"Loaded minute bars: {len(minute):,} rows from {minute_path}")
try:
result = compute_derived(
derived_type=derived_type,
daily=daily,
minute=minute,
**_parse_params(extra_params),
)
out_path = write_derived_frame(result, derived_name, output_dir=output_dir)
except Exception as exc:
raise click.ClickException(str(exc)) from exc
click.echo(f"Saved derived data: {out_path} ({_summarize(result)})")
pipeline/derived/compute.py
+115
View File
@@ -0,0 +1,115 @@
"""Derived-data computation and validation."""
import csv
import logging
from pathlib import Path
from typing import Iterable
import pandas as pd
from pandas.api.types import is_bool_dtype, is_numeric_dtype
from pipeline.common.schema import DERIVED_KEY_COLUMNS
from pipeline.derived.registry import get_derived
logger = logging.getLogger(__name__)
def validate_derived_frame(derived: pd.DataFrame) -> pd.DataFrame:
"""Validate and normalize a daily derived-data frame.
A valid derived frame is keyed by unique ``symbol_id,date`` rows and has at
least one numeric value column beyond those keys. Dates are normalized to
daily timestamps before duplicate-key checks.
"""
duplicated = derived.columns[derived.columns.duplicated()].tolist()
if duplicated:
raise ValueError(f"Derived data has duplicate columns: {duplicated}")
missing = [col for col in DERIVED_KEY_COLUMNS if col not in derived.columns]
if missing:
raise ValueError(f"Derived data missing required columns: {missing}")
out = derived.copy()
out["date"] = pd.to_datetime(out["date"]).dt.normalize()
if out.duplicated(DERIVED_KEY_COLUMNS).any():
raise ValueError("Derived data has duplicate symbol_id,date rows")
value_cols = [col for col in out.columns if col not in DERIVED_KEY_COLUMNS]
if not value_cols:
raise ValueError("Derived data must include at least one value column")
non_numeric = [
col
for col in value_cols
if is_bool_dtype(out[col]) or not is_numeric_dtype(out[col])
]
if non_numeric:
raise ValueError(f"Derived data value columns must be numeric: {non_numeric}")
out = out[DERIVED_KEY_COLUMNS + value_cols].copy()
return out.sort_values(DERIVED_KEY_COLUMNS).reset_index(drop=True)
def compute_derived(
derived_type: str,
daily: pd.DataFrame | None = None,
minute: pd.DataFrame | None = None,
**params,
) -> pd.DataFrame:
"""Compute one registered derived-data plugin."""
if daily is None and minute is None:
raise ValueError("Derived data computation requires --daily-path or --minute-path")
derived = get_derived(derived_type, **params)
result = validate_derived_frame(derived.compute(daily=daily, minute=minute))
value_cols = [col for col in result.columns if col not in DERIVED_KEY_COLUMNS]
logger.info(
"Derived data '%s' (%r): %d symbols × %d dates, columns=%s",
derived_type,
derived,
result["symbol_id"].nunique(),
result["date"].nunique(),
value_cols,
)
return result
def read_derived_frame(path: str | Path) -> pd.DataFrame:
"""Read and validate one derived CSV/parquet file or parquet dataset."""
path = Path(path)
if path.suffix.lower() == ".csv":
return validate_derived_frame(_read_csv_with_duplicate_header_check(path))
return validate_derived_frame(pd.read_parquet(path))
def read_derived_frames(derived_paths: Iterable[str | Path]) -> list[pd.DataFrame]:
"""Read and validate derived-data files."""
return [read_derived_frame(path) for path in derived_paths]
def write_derived_frame(
derived: pd.DataFrame,
derived_name: str,
output_dir: str | Path = "derived",
) -> Path:
"""Validate and write derived data to ``{output_dir}/{derived_name}.pq``."""
result = validate_derived_frame(derived)
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
out_path = output_dir / f"{derived_name}.pq"
result.to_parquet(out_path, index=False)
return out_path
def _read_csv_with_duplicate_header_check(path: Path) -> pd.DataFrame:
with path.open(newline="") as fh:
reader = csv.reader(fh)
try:
header = next(reader)
except StopIteration as exc:
raise ValueError("CSV input is empty") from exc
duplicated = sorted({col for col in header if header.count(col) > 1})
if duplicated:
raise ValueError(f"Derived data has duplicate columns: {duplicated}")
return pd.read_csv(path)
pipeline/derived/library/__init__.py
+4
View File
@@ -0,0 +1,4 @@
"""Built-in derived-data library."""
from pipeline.derived.library import minute_daily_summary # noqa: F401
pipeline/derived/library/minute_daily_summary.py
+88
View File
@@ -0,0 +1,88 @@
"""Daily summary data derived from raw minute bars."""
import numpy as np
import pandas as pd
from pipeline.derived.base import BaseDerivedData
from pipeline.derived.registry import register_derived
@register_derived
class MinuteDailySummaryDerived(BaseDerivedData):
"""Aggregate intraday bars into daily summary columns."""
name = "minute_daily_summary"
def compute(
self,
daily: pd.DataFrame | None = None,
minute: pd.DataFrame | None = None,
) -> pd.DataFrame:
if minute is None:
raise ValueError("minute_daily_summary requires minute input")
minute = minute.copy()
minute["date"] = pd.to_datetime(minute["date"]).dt.normalize()
sort_cols = ["symbol_id", "date"]
if "datetime" in minute.columns:
minute["datetime"] = pd.to_datetime(minute["datetime"])
sort_cols.append("datetime")
elif "time" in minute.columns:
sort_cols.append("time")
minute = minute.sort_values(sort_cols)
grouped = minute.groupby(["symbol_id", "date"], sort=True)
summary = grouped.agg(
minute_bar_count=("close", "count"),
first_open=("open", "first"),
last_close=("close", "last"),
high=("high", "max"),
low=("low", "min"),
volume_sum=("volume", "sum"),
amount_sum=("amount", "sum"),
)
summary["minute_intraday_return"] = (
summary["last_close"] / summary["first_open"] - 1.0
)
summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
summary["minute_vwap"] = (
summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
)
summary = summary.reset_index()
if daily is not None:
daily_keys = daily[["symbol_id", "date"]].copy()
daily_keys["date"] = pd.to_datetime(daily_keys["date"]).dt.normalize()
daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
if "close" in daily.columns:
daily_close = daily[["symbol_id", "date", "close"]].copy()
daily_close["date"] = pd.to_datetime(daily_close["date"]).dt.normalize()
daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
result = result.merge(
daily_close.rename(columns={"close": "daily_close"}),
on=["symbol_id", "date"],
how="left",
)
reference_close = result["daily_close"].fillna(result["last_close"])
else:
reference_close = result["last_close"]
else:
result = summary
reference_close = result["last_close"]
result["minute_vwap_deviation"] = (
result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
)
return result[
[
"symbol_id",
"date",
"minute_bar_count",
"minute_intraday_return",
"minute_intraday_range",
"minute_vwap",
"minute_vwap_deviation",
]
]
pipeline/derived/registry.py
+80
View File
@@ -0,0 +1,80 @@
"""Registry and factory for daily derived-data plugins."""
import importlib
import importlib.util
import inspect
from pathlib import Path
from typing import Optional, Type
from pipeline.derived.base import BaseDerivedData
_REGISTRY: dict[str, Type[BaseDerivedData]] = {}
_builtins_loaded = False
def register_derived(cls: Type[BaseDerivedData]) -> Type[BaseDerivedData]:
"""Class decorator that registers derived data under ``BaseDerivedData.name``."""
if not (isinstance(cls, type) and issubclass(cls, BaseDerivedData)):
raise TypeError(f"{cls!r} is not a BaseDerivedData subclass")
key = getattr(cls, "name", "")
if not key:
raise ValueError(f"{cls.__name__} must set a non-empty class attribute `name`")
existing = _REGISTRY.get(key)
if existing is not None and existing is not cls:
raise ValueError(
f"Derived data name '{key}' already registered by {existing.__name__}"
)
_REGISTRY[key] = cls
return cls
def available_derived() -> list[str]:
"""Sorted names of all registered derived-data plugins."""
_ensure_builtins()
return sorted(_REGISTRY)
def get_derived(name: str, **params) -> BaseDerivedData:
"""Instantiate a registered derived-data plugin by name.
Only parameters accepted by the plugin class's ``__init__`` are forwarded.
"""
_ensure_builtins()
if name not in _REGISTRY:
raise KeyError(f"Unknown derived data '{name}'. Available: {sorted(_REGISTRY)}")
cls = _REGISTRY[name]
accepted = _accepted_params(cls)
kwargs = params if accepted is None else {k: v for k, v in params.items() if k in accepted}
return cls(**kwargs)
def load_derived_module(spec: str) -> None:
"""Import an external module so its ``@register_derived`` classes register."""
looks_like_file = spec.endswith(".py") or Path(spec).expanduser().exists()
if looks_like_file:
path = Path(spec).expanduser().resolve()
if not path.exists():
raise FileNotFoundError(f"Derived data module not found: {path}")
module_spec = importlib.util.spec_from_file_location(path.stem, path)
if module_spec is None or module_spec.loader is None:
raise ImportError(f"Cannot load derived data module from {path}")
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
else:
importlib.import_module(spec)
def _accepted_params(cls: Type[BaseDerivedData]) -> Optional[set[str]]:
"""Param names ``cls.__init__`` accepts, or None if it takes ``**kwargs``."""
sig = inspect.signature(cls.__init__)
if any(p.kind is p.VAR_KEYWORD for p in sig.parameters.values()):
return None
return {name for name in sig.parameters if name != "self"}
def _ensure_builtins() -> None:
global _builtins_loaded
if not _builtins_loaded:
import pipeline.derived.library # noqa: F401
_builtins_loaded = True
pipeline/features/base.py
+5 -31
View File
@@ -1,34 +1,8 @@
"""Base class for daily feature plugins."""
"""Compatibility alias for daily feature plugins.
from abc import ABC, abstractmethod
The canonical plugin API is ``pipeline.derived``. ``BaseFeature`` remains as an
alias so existing external feature modules continue to register unchanged.
"""
import pandas as pd
from pipeline.derived.base import BaseDerivedData as BaseFeature
class BaseFeature(ABC):
"""Aggregate raw minute bars into daily, symbol-keyed feature columns."""
#: Unique registry key. Every concrete feature must set this to a non-empty str.
name: str = ""
@abstractmethod
def compute(
self,
minute: pd.DataFrame,
daily: pd.DataFrame | None = None,
) -> pd.DataFrame:
"""Compute daily features.
Args:
minute: Raw minute bars with ``symbol_id`` and ``date`` keys.
daily: Optional daily data frame for calendar alignment or
reference daily columns.
Returns:
DataFrame with ``symbol_id``, ``date``, and one or more numeric
feature columns.
"""
def __repr__(self) -> str:
params = ", ".join(f"{k}={v!r}" for k, v in vars(self).items())
return f"{type(self).__name__}({params})"
pipeline/features/compute.py
+19 -53
View File
@@ -1,49 +1,23 @@
"""Feature computation and validation."""
"""Compatibility wrappers for daily feature computation and validation."""
import logging
from pathlib import Path
from typing import Iterable
import pandas as pd
from pandas.api.types import is_numeric_dtype
from pipeline.features.registry import get_feature
from pipeline.derived.compute import (
DERIVED_KEY_COLUMNS,
compute_derived,
read_derived_frames,
validate_derived_frame,
)
logger = logging.getLogger(__name__)
FEATURE_KEY_COLUMNS = ["symbol_id", "date"]
FEATURE_KEY_COLUMNS = DERIVED_KEY_COLUMNS
def validate_feature_frame(features: pd.DataFrame) -> pd.DataFrame:
"""Validate and normalize a daily feature frame.
A valid feature frame is keyed by unique ``symbol_id,date`` rows and has at
least one numeric feature column beyond those keys.
"""
duplicated = features.columns[features.columns.duplicated()].tolist()
if duplicated:
raise ValueError(f"Feature output has duplicate columns: {duplicated}")
missing = [col for col in FEATURE_KEY_COLUMNS if col not in features.columns]
if missing:
raise ValueError(f"Feature output missing required columns: {missing}")
out = features.copy()
out["date"] = pd.to_datetime(out["date"])
if out.duplicated(FEATURE_KEY_COLUMNS).any():
raise ValueError("Feature output has duplicate symbol_id,date rows")
feature_cols = [col for col in out.columns if col not in FEATURE_KEY_COLUMNS]
if not feature_cols:
raise ValueError("Feature output must include at least one feature column")
non_numeric = [col for col in feature_cols if not is_numeric_dtype(out[col])]
if non_numeric:
raise ValueError(f"Feature columns must be numeric: {non_numeric}")
out = out[FEATURE_KEY_COLUMNS + feature_cols].copy()
return out.sort_values(FEATURE_KEY_COLUMNS).reset_index(drop=True)
"""Validate and normalize a legacy daily feature frame."""
return validate_derived_frame(features)
def compute_feature(
@@ -52,24 +26,16 @@ def compute_feature(
daily: pd.DataFrame | None = None,
**params,
) -> pd.DataFrame:
"""Compute one registered feature from raw minute bars."""
feature = get_feature(feature_type, **params)
result = validate_feature_frame(feature.compute(minute=minute, daily=daily))
feature_cols = [col for col in result.columns if col not in FEATURE_KEY_COLUMNS]
logger.info(
"Feature '%s' (%r): %d symbols × %d dates, columns=%s",
feature_type,
feature,
result["symbol_id"].nunique(),
result["date"].nunique(),
feature_cols,
"""Compute one registered feature through the derived-data registry."""
return compute_derived(
derived_type=feature_type,
daily=daily,
minute=minute,
**params,
)
return result
def read_feature_frames(feature_paths: Iterable[str | Path]) -> list[pd.DataFrame]:
"""Read and validate feature parquet files."""
return [
validate_feature_frame(pd.read_parquet(path))
for path in feature_paths
]
"""Read and validate feature/derived-data parquet files."""
return read_derived_frames(feature_paths)
pipeline/features/library/minute_daily_summary.py
+5 -73
View File
@@ -1,84 +1,16 @@
"""Daily summary features derived from raw minute bars."""
"""Compatibility wrapper for the built-in minute daily summary plugin."""
import numpy as np
import pandas as pd
from pipeline.features.base import BaseFeature
from pipeline.features.registry import register_feature
from pipeline.derived.library.minute_daily_summary import MinuteDailySummaryDerived
@register_feature
class MinuteDailySummaryFeature(BaseFeature):
"""Aggregate intraday bars into daily summary columns."""
name = "minute_daily_summary"
class MinuteDailySummaryFeature(MinuteDailySummaryDerived):
"""Legacy minute-first wrapper around the derived-data implementation."""
def compute(
self,
minute: pd.DataFrame,
daily: pd.DataFrame | None = None,
) -> pd.DataFrame:
minute = minute.copy()
minute["date"] = pd.to_datetime(minute["date"])
sort_cols = ["symbol_id", "date"]
if "datetime" in minute.columns:
minute["datetime"] = pd.to_datetime(minute["datetime"])
sort_cols.append("datetime")
elif "time" in minute.columns:
sort_cols.append("time")
minute = minute.sort_values(sort_cols)
grouped = minute.groupby(["symbol_id", "date"], sort=True)
summary = grouped.agg(
minute_bar_count=("close", "count"),
first_open=("open", "first"),
last_close=("close", "last"),
high=("high", "max"),
low=("low", "min"),
volume_sum=("volume", "sum"),
amount_sum=("amount", "sum"),
)
summary["minute_intraday_return"] = (
summary["last_close"] / summary["first_open"] - 1.0
)
summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
summary["minute_vwap"] = (
summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
)
summary = summary.reset_index()
if daily is not None:
daily_keys = daily[["symbol_id", "date"]].copy()
daily_keys["date"] = pd.to_datetime(daily_keys["date"])
daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
if "close" in daily.columns:
daily_close = daily[["symbol_id", "date", "close"]].copy()
daily_close["date"] = pd.to_datetime(daily_close["date"])
daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
result = result.merge(
daily_close.rename(columns={"close": "daily_close"}),
on=["symbol_id", "date"],
how="left",
)
reference_close = result["daily_close"].fillna(result["last_close"])
else:
reference_close = result["last_close"]
else:
result = summary
reference_close = result["last_close"]
result["minute_vwap_deviation"] = (
result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
)
return result[
[
"symbol_id",
"date",
"minute_bar_count",
"minute_intraday_return",
"minute_intraday_range",
"minute_vwap",
"minute_vwap_deviation",
]
]
return super().compute(daily=daily, minute=minute)
pipeline/features/registry.py
+20 -63
View File
@@ -1,79 +1,36 @@
"""Registry and factory for daily feature plugins."""
"""Compatibility registry wrappers for daily feature plugins."""
import importlib
import importlib.util
import inspect
from pathlib import Path
from typing import Optional, Type
from typing import Type
from pipeline.derived.base import BaseDerivedData
from pipeline.derived.registry import (
available_derived,
get_derived,
load_derived_module,
register_derived,
)
from pipeline.features.base import BaseFeature
_REGISTRY: dict[str, Type[BaseFeature]] = {}
_builtins_loaded = False
def register_feature(cls: Type[BaseFeature]) -> Type[BaseFeature]:
"""Class decorator that registers a feature under ``BaseFeature.name``."""
if not (isinstance(cls, type) and issubclass(cls, BaseFeature)):
raise TypeError(f"{cls!r} is not a BaseFeature subclass")
key = getattr(cls, "name", "")
if not key:
raise ValueError(f"{cls.__name__} must set a non-empty class attribute `name`")
existing = _REGISTRY.get(key)
if existing is not None and existing is not cls:
raise ValueError(
f"Feature name '{key}' already registered by {existing.__name__}"
)
_REGISTRY[key] = cls
return cls
"""Register a legacy feature plugin in the derived-data registry."""
return register_derived(cls)
def available_features() -> list[str]:
"""Sorted names of all registered features (built-ins are loaded lazily)."""
_ensure_builtins()
return sorted(_REGISTRY)
"""Sorted names of all registered feature/derived-data plugins."""
return available_derived()
def get_feature(name: str, **params) -> BaseFeature:
"""Instantiate a registered feature by name.
def get_feature(name: str, **params) -> BaseDerivedData:
"""Instantiate a registered feature/derived-data plugin by name."""
if name == "minute_daily_summary":
from pipeline.features.library.minute_daily_summary import MinuteDailySummaryFeature
Only parameters accepted by the feature class's ``__init__`` are forwarded.
"""
_ensure_builtins()
if name not in _REGISTRY:
raise KeyError(f"Unknown feature '{name}'. Available: {sorted(_REGISTRY)}")
cls = _REGISTRY[name]
accepted = _accepted_params(cls)
kwargs = params if accepted is None else {k: v for k, v in params.items() if k in accepted}
return cls(**kwargs)
return MinuteDailySummaryFeature(**params)
return get_derived(name, **params)
def load_feature_module(spec: str) -> None:
"""Import an external module so its ``@register_feature`` classes register."""
looks_like_file = spec.endswith(".py") or Path(spec).expanduser().exists()
if looks_like_file:
path = Path(spec).expanduser().resolve()
if not path.exists():
raise FileNotFoundError(f"Feature module not found: {path}")
module_spec = importlib.util.spec_from_file_location(path.stem, path)
if module_spec is None or module_spec.loader is None:
raise ImportError(f"Cannot load feature module from {path}")
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
else:
importlib.import_module(spec)
def _accepted_params(cls: Type[BaseFeature]) -> Optional[set[str]]:
"""Param names ``cls.__init__`` accepts, or None if it takes ``**kwargs``."""
sig = inspect.signature(cls.__init__)
if any(p.kind is p.VAR_KEYWORD for p in sig.parameters.values()):
return None
return {name for name in sig.parameters if name != "self"}
def _ensure_builtins() -> None:
global _builtins_loaded
if not _builtins_loaded:
import pipeline.features.library # noqa: F401
_builtins_loaded = True
load_derived_module(spec)