Add minute bar feature pipeline

This commit is contained in:
Yuxuan Yan
2026-06-16 13:57:17 +08:00
parent 17fa75495d
commit 83a006bbe4
19 changed files with 1289 additions and 11 deletions
+35 -1
View File
@@ -3,7 +3,7 @@
import click
from datetime import date
from pipeline.data.downloader import download_universe
from pipeline.data.downloader import download_minute_universe, download_universe
@click.group(name="data")
@@ -42,3 +42,37 @@ def download(universe, start_date, end_date, output_dir, symbols, chunk_size, ad
f"{stats['n_rows']:,} bars, {stats['date_min']}{stats['date_max']}"
)
click.echo(f"Dataset: {stats['dataset_path']}")
@data.command("download-minute")
@click.option(
"--universe", default="csi500",
help="Which universe: hs300, csi500, all (~5000 A-shares), file path, or comma-separated symbols",
)
@click.option("--start-date", default="2017-01-01", help="Start date YYYY-MM-DD")
@click.option("--end-date", default=str(date.today()), help="End date YYYY-MM-DD")
@click.option("--output-dir", default="data/minute_bars", help="Root for the partitioned dataset")
@click.option("--symbols", default=0, type=int, help="Max symbols (0=all)")
@click.option("--chunk-size", default=100, type=int, help="Symbols per durability flush")
@click.option("--frequency", default="5", help="Minute frequency: 5, 15, 30, or 60")
def download_minute(universe, start_date, end_date, output_dir, symbols, chunk_size, frequency):
"""Download raw Baostock minute bars into a partitioned parquet dataset.
Writes ``{output_dir}/{universe}/frequency=5m/month=YYYY-MM/*.pq`` for the
default 5-minute frequency.
"""
stats = download_minute_universe(
universe=universe,
start_date=start_date,
end_date=end_date,
output_dir=output_dir,
max_symbols=symbols,
chunk_size=chunk_size,
frequency=frequency,
)
click.echo(
f"\nSummary: {stats['n_symbols']}/{stats['n_requested']} symbols, "
f"{stats['n_rows']:,} bars, {stats['date_min']}{stats['date_max']}, "
f"frequency={stats['frequency']}"
)
click.echo(f"Dataset: {stats['dataset_path']}")