Add daily derived data pipeline
This commit is contained in:
@@ -15,7 +15,11 @@ import pandas as pd
|
||||
|
||||
from pipeline.alpha.registry import get_alpha
|
||||
from pipeline.common.schema import ALPHA_COLUMNS
|
||||
from pipeline.features.compute import FEATURE_KEY_COLUMNS, read_feature_frames, validate_feature_frame
|
||||
from pipeline.derived.compute import (
|
||||
DERIVED_KEY_COLUMNS,
|
||||
read_derived_frames,
|
||||
validate_derived_frame,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,15 +44,15 @@ def join_feature_frames(
|
||||
data: pd.DataFrame,
|
||||
feature_frames: Iterable[pd.DataFrame],
|
||||
) -> pd.DataFrame:
|
||||
"""Left-join validated daily feature frames onto long daily data."""
|
||||
"""Left-join validated daily derived/feature frames onto long daily data."""
|
||||
out = data.copy()
|
||||
out["date"] = pd.to_datetime(out["date"])
|
||||
existing = set(out.columns)
|
||||
joined_cols: list[str] = []
|
||||
|
||||
for frame in feature_frames:
|
||||
features = validate_feature_frame(frame)
|
||||
feature_cols = [col for col in features.columns if col not in FEATURE_KEY_COLUMNS]
|
||||
features = validate_derived_frame(frame)
|
||||
feature_cols = [col for col in features.columns if col not in DERIVED_KEY_COLUMNS]
|
||||
overlap = sorted(existing.intersection(feature_cols))
|
||||
if overlap:
|
||||
raise ValueError(
|
||||
@@ -56,7 +60,7 @@ def join_feature_frames(
|
||||
)
|
||||
out = out.merge(
|
||||
features,
|
||||
on=FEATURE_KEY_COLUMNS,
|
||||
on=DERIVED_KEY_COLUMNS,
|
||||
how="left",
|
||||
validate="many_to_one",
|
||||
)
|
||||
@@ -171,7 +175,7 @@ def compute_alpha(
|
||||
"""
|
||||
feature_inputs: list[pd.DataFrame] = []
|
||||
if feature_paths:
|
||||
feature_inputs.extend(read_feature_frames(feature_paths))
|
||||
feature_inputs.extend(read_derived_frames(feature_paths))
|
||||
if feature_frames:
|
||||
feature_inputs.extend(feature_frames)
|
||||
if feature_inputs:
|
||||
|
||||
Reference in New Issue
Block a user