Add daily derived data pipeline
This commit is contained in:
Yuxuan Yan
@@ -1,34 +1,8 @@
|
||||
"""Base class for daily feature plugins."""
|
||||
"""Compatibility alias for daily feature plugins.
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
The canonical plugin API is ``pipeline.derived``. ``BaseFeature`` remains as an
|
||||
alias so existing external feature modules continue to register unchanged.
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
from pipeline.derived.base import BaseDerivedData as BaseFeature
|
||||
|
||||
|
||||
class BaseFeature(ABC):
|
||||
"""Aggregate raw minute bars into daily, symbol-keyed feature columns."""
|
||||
|
||||
#: Unique registry key. Every concrete feature must set this to a non-empty str.
|
||||
name: str = ""
|
||||
|
||||
@abstractmethod
|
||||
def compute(
|
||||
self,
|
||||
minute: pd.DataFrame,
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Compute daily features.
|
||||
|
||||
Args:
|
||||
minute: Raw minute bars with ``symbol_id`` and ``date`` keys.
|
||||
daily: Optional daily data frame for calendar alignment or
|
||||
reference daily columns.
|
||||
|
||||
Returns:
|
||||
DataFrame with ``symbol_id``, ``date``, and one or more numeric
|
||||
feature columns.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str:
|
||||
params = ", ".join(f"{k}={v!r}" for k, v in vars(self).items())
|
||||
return f"{type(self).__name__}({params})"
|
||||
|
||||
@@ -1,49 +1,23 @@
|
||||
"""Feature computation and validation."""
|
||||
"""Compatibility wrappers for daily feature computation and validation."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
import pandas as pd
|
||||
from pandas.api.types import is_numeric_dtype
|
||||
|
||||
from pipeline.features.registry import get_feature
|
||||
from pipeline.derived.compute import (
|
||||
DERIVED_KEY_COLUMNS,
|
||||
compute_derived,
|
||||
read_derived_frames,
|
||||
validate_derived_frame,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FEATURE_KEY_COLUMNS = ["symbol_id", "date"]
|
||||
FEATURE_KEY_COLUMNS = DERIVED_KEY_COLUMNS
|
||||
|
||||
|
||||
def validate_feature_frame(features: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Validate and normalize a daily feature frame.
|
||||
|
||||
A valid feature frame is keyed by unique ``symbol_id,date`` rows and has at
|
||||
least one numeric feature column beyond those keys.
|
||||
"""
|
||||
duplicated = features.columns[features.columns.duplicated()].tolist()
|
||||
if duplicated:
|
||||
raise ValueError(f"Feature output has duplicate columns: {duplicated}")
|
||||
|
||||
missing = [col for col in FEATURE_KEY_COLUMNS if col not in features.columns]
|
||||
if missing:
|
||||
raise ValueError(f"Feature output missing required columns: {missing}")
|
||||
|
||||
out = features.copy()
|
||||
out["date"] = pd.to_datetime(out["date"])
|
||||
|
||||
if out.duplicated(FEATURE_KEY_COLUMNS).any():
|
||||
raise ValueError("Feature output has duplicate symbol_id,date rows")
|
||||
|
||||
feature_cols = [col for col in out.columns if col not in FEATURE_KEY_COLUMNS]
|
||||
if not feature_cols:
|
||||
raise ValueError("Feature output must include at least one feature column")
|
||||
|
||||
non_numeric = [col for col in feature_cols if not is_numeric_dtype(out[col])]
|
||||
if non_numeric:
|
||||
raise ValueError(f"Feature columns must be numeric: {non_numeric}")
|
||||
|
||||
out = out[FEATURE_KEY_COLUMNS + feature_cols].copy()
|
||||
return out.sort_values(FEATURE_KEY_COLUMNS).reset_index(drop=True)
|
||||
"""Validate and normalize a legacy daily feature frame."""
|
||||
return validate_derived_frame(features)
|
||||
|
||||
|
||||
def compute_feature(
|
||||
@@ -52,24 +26,16 @@ def compute_feature(
|
||||
daily: pd.DataFrame | None = None,
|
||||
**params,
|
||||
) -> pd.DataFrame:
|
||||
"""Compute one registered feature from raw minute bars."""
|
||||
feature = get_feature(feature_type, **params)
|
||||
result = validate_feature_frame(feature.compute(minute=minute, daily=daily))
|
||||
feature_cols = [col for col in result.columns if col not in FEATURE_KEY_COLUMNS]
|
||||
logger.info(
|
||||
"Feature '%s' (%r): %d symbols × %d dates, columns=%s",
|
||||
feature_type,
|
||||
feature,
|
||||
result["symbol_id"].nunique(),
|
||||
result["date"].nunique(),
|
||||
feature_cols,
|
||||
"""Compute one registered feature through the derived-data registry."""
|
||||
return compute_derived(
|
||||
derived_type=feature_type,
|
||||
daily=daily,
|
||||
minute=minute,
|
||||
**params,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def read_feature_frames(feature_paths: Iterable[str | Path]) -> list[pd.DataFrame]:
|
||||
"""Read and validate feature parquet files."""
|
||||
return [
|
||||
validate_feature_frame(pd.read_parquet(path))
|
||||
for path in feature_paths
|
||||
]
|
||||
"""Read and validate feature/derived-data parquet files."""
|
||||
return read_derived_frames(feature_paths)
|
||||
|
||||
|
||||
@@ -1,84 +1,16 @@
|
||||
"""Daily summary features derived from raw minute bars."""
|
||||
"""Compatibility wrapper for the built-in minute daily summary plugin."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from pipeline.features.base import BaseFeature
|
||||
from pipeline.features.registry import register_feature
|
||||
from pipeline.derived.library.minute_daily_summary import MinuteDailySummaryDerived
|
||||
|
||||
|
||||
@register_feature
|
||||
class MinuteDailySummaryFeature(BaseFeature):
|
||||
"""Aggregate intraday bars into daily summary columns."""
|
||||
|
||||
name = "minute_daily_summary"
|
||||
class MinuteDailySummaryFeature(MinuteDailySummaryDerived):
|
||||
"""Legacy minute-first wrapper around the derived-data implementation."""
|
||||
|
||||
def compute(
|
||||
self,
|
||||
minute: pd.DataFrame,
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
minute = minute.copy()
|
||||
minute["date"] = pd.to_datetime(minute["date"])
|
||||
sort_cols = ["symbol_id", "date"]
|
||||
if "datetime" in minute.columns:
|
||||
minute["datetime"] = pd.to_datetime(minute["datetime"])
|
||||
sort_cols.append("datetime")
|
||||
elif "time" in minute.columns:
|
||||
sort_cols.append("time")
|
||||
minute = minute.sort_values(sort_cols)
|
||||
|
||||
grouped = minute.groupby(["symbol_id", "date"], sort=True)
|
||||
summary = grouped.agg(
|
||||
minute_bar_count=("close", "count"),
|
||||
first_open=("open", "first"),
|
||||
last_close=("close", "last"),
|
||||
high=("high", "max"),
|
||||
low=("low", "min"),
|
||||
volume_sum=("volume", "sum"),
|
||||
amount_sum=("amount", "sum"),
|
||||
)
|
||||
summary["minute_intraday_return"] = (
|
||||
summary["last_close"] / summary["first_open"] - 1.0
|
||||
)
|
||||
summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
|
||||
summary["minute_vwap"] = (
|
||||
summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
|
||||
)
|
||||
summary = summary.reset_index()
|
||||
|
||||
if daily is not None:
|
||||
daily_keys = daily[["symbol_id", "date"]].copy()
|
||||
daily_keys["date"] = pd.to_datetime(daily_keys["date"])
|
||||
daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
|
||||
result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
|
||||
if "close" in daily.columns:
|
||||
daily_close = daily[["symbol_id", "date", "close"]].copy()
|
||||
daily_close["date"] = pd.to_datetime(daily_close["date"])
|
||||
daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
|
||||
result = result.merge(
|
||||
daily_close.rename(columns={"close": "daily_close"}),
|
||||
on=["symbol_id", "date"],
|
||||
how="left",
|
||||
)
|
||||
reference_close = result["daily_close"].fillna(result["last_close"])
|
||||
else:
|
||||
reference_close = result["last_close"]
|
||||
else:
|
||||
result = summary
|
||||
reference_close = result["last_close"]
|
||||
|
||||
result["minute_vwap_deviation"] = (
|
||||
result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
|
||||
)
|
||||
return result[
|
||||
[
|
||||
"symbol_id",
|
||||
"date",
|
||||
"minute_bar_count",
|
||||
"minute_intraday_return",
|
||||
"minute_intraday_range",
|
||||
"minute_vwap",
|
||||
"minute_vwap_deviation",
|
||||
]
|
||||
]
|
||||
return super().compute(daily=daily, minute=minute)
|
||||
|
||||
@@ -1,79 +1,36 @@
|
||||
"""Registry and factory for daily feature plugins."""
|
||||
"""Compatibility registry wrappers for daily feature plugins."""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
from typing import Type
|
||||
|
||||
from pipeline.derived.base import BaseDerivedData
|
||||
from pipeline.derived.registry import (
|
||||
available_derived,
|
||||
get_derived,
|
||||
load_derived_module,
|
||||
register_derived,
|
||||
)
|
||||
from pipeline.features.base import BaseFeature
|
||||
|
||||
_REGISTRY: dict[str, Type[BaseFeature]] = {}
|
||||
_builtins_loaded = False
|
||||
|
||||
|
||||
def register_feature(cls: Type[BaseFeature]) -> Type[BaseFeature]:
|
||||
"""Class decorator that registers a feature under ``BaseFeature.name``."""
|
||||
if not (isinstance(cls, type) and issubclass(cls, BaseFeature)):
|
||||
raise TypeError(f"{cls!r} is not a BaseFeature subclass")
|
||||
key = getattr(cls, "name", "")
|
||||
if not key:
|
||||
raise ValueError(f"{cls.__name__} must set a non-empty class attribute `name`")
|
||||
existing = _REGISTRY.get(key)
|
||||
if existing is not None and existing is not cls:
|
||||
raise ValueError(
|
||||
f"Feature name '{key}' already registered by {existing.__name__}"
|
||||
)
|
||||
_REGISTRY[key] = cls
|
||||
return cls
|
||||
"""Register a legacy feature plugin in the derived-data registry."""
|
||||
return register_derived(cls)
|
||||
|
||||
|
||||
def available_features() -> list[str]:
|
||||
"""Sorted names of all registered features (built-ins are loaded lazily)."""
|
||||
_ensure_builtins()
|
||||
return sorted(_REGISTRY)
|
||||
"""Sorted names of all registered feature/derived-data plugins."""
|
||||
return available_derived()
|
||||
|
||||
|
||||
def get_feature(name: str, **params) -> BaseFeature:
|
||||
"""Instantiate a registered feature by name.
|
||||
def get_feature(name: str, **params) -> BaseDerivedData:
|
||||
"""Instantiate a registered feature/derived-data plugin by name."""
|
||||
if name == "minute_daily_summary":
|
||||
from pipeline.features.library.minute_daily_summary import MinuteDailySummaryFeature
|
||||
|
||||
Only parameters accepted by the feature class's ``__init__`` are forwarded.
|
||||
"""
|
||||
_ensure_builtins()
|
||||
if name not in _REGISTRY:
|
||||
raise KeyError(f"Unknown feature '{name}'. Available: {sorted(_REGISTRY)}")
|
||||
cls = _REGISTRY[name]
|
||||
accepted = _accepted_params(cls)
|
||||
kwargs = params if accepted is None else {k: v for k, v in params.items() if k in accepted}
|
||||
return cls(**kwargs)
|
||||
return MinuteDailySummaryFeature(**params)
|
||||
return get_derived(name, **params)
|
||||
|
||||
|
||||
def load_feature_module(spec: str) -> None:
|
||||
"""Import an external module so its ``@register_feature`` classes register."""
|
||||
looks_like_file = spec.endswith(".py") or Path(spec).expanduser().exists()
|
||||
if looks_like_file:
|
||||
path = Path(spec).expanduser().resolve()
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Feature module not found: {path}")
|
||||
module_spec = importlib.util.spec_from_file_location(path.stem, path)
|
||||
if module_spec is None or module_spec.loader is None:
|
||||
raise ImportError(f"Cannot load feature module from {path}")
|
||||
module = importlib.util.module_from_spec(module_spec)
|
||||
module_spec.loader.exec_module(module)
|
||||
else:
|
||||
importlib.import_module(spec)
|
||||
|
||||
|
||||
def _accepted_params(cls: Type[BaseFeature]) -> Optional[set[str]]:
|
||||
"""Param names ``cls.__init__`` accepts, or None if it takes ``**kwargs``."""
|
||||
sig = inspect.signature(cls.__init__)
|
||||
if any(p.kind is p.VAR_KEYWORD for p in sig.parameters.values()):
|
||||
return None
|
||||
return {name for name in sig.parameters if name != "self"}
|
||||
|
||||
|
||||
def _ensure_builtins() -> None:
|
||||
global _builtins_loaded
|
||||
if not _builtins_loaded:
|
||||
import pipeline.features.library # noqa: F401
|
||||
_builtins_loaded = True
|
||||
load_derived_module(spec)
|
||||
|
||||
Reference in New Issue
Block a user