feat: add pqcat helper to inspect parquet files
A standalone tools/pqcat.py (head/tail/columns/info) wired as `cli.py pqcat` and runnable directly via a PATH symlink, plus README docs. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -153,6 +153,32 @@ python3 cli.py combo combine \
|
|||||||
--combo-name eq --method equal_weight
|
--combo-name eq --method equal_weight
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `pqcat` — inspect a parquet file, like `cat`
|
||||||
|
|
||||||
|
Quickly dump any pipeline parquet (a single `.pq` file or a partitioned dataset
|
||||||
|
directory) to stdout, without writing a throwaway script.
|
||||||
|
|
||||||
|
| Option | Default | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `-n, --head N` | — | Show only the first `N` rows |
|
||||||
|
| `-t, --tail N` | — | Show only the last `N` rows |
|
||||||
|
| `-c, --columns` | — | Comma-separated subset of columns to show |
|
||||||
|
| `--info` | off | Show shape + dtypes instead of the rows |
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 cli.py pqcat alphas/reversal_5d.pq # dump all rows
|
||||||
|
python3 cli.py pqcat data/daily_bars/csi500 --info # shape + dtypes
|
||||||
|
python3 cli.py pqcat data/daily_bars/csi500 -n 10 -c symbol_id,date,close,vwap
|
||||||
|
```
|
||||||
|
|
||||||
|
**Standalone command.** `tools/pqcat.py` has no repo dependencies, so it can be
|
||||||
|
run directly. Symlink it onto your `PATH` once and call `pqcat` from anywhere:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ln -sf "$(pwd)/tools/pqcat.py" ~/.local/bin/pqcat # ~/.local/bin must be on PATH
|
||||||
|
pqcat alphas/reversal_5d.pq --info
|
||||||
|
```
|
||||||
|
|
||||||
## Alphas: the factory / plugin interface
|
## Alphas: the factory / plugin interface
|
||||||
|
|
||||||
An **alpha** is a class that maps a wide close matrix (date index × `symbol_id`
|
An **alpha** is a class that maps a wide close matrix (date index × `symbol_id`
|
||||||
@@ -245,6 +271,7 @@ directory yields an extra `month` (`YYYY-MM`) partition column on top of
|
|||||||
- `pipeline/combo/` — alpha combination → `combos/*.pq`
|
- `pipeline/combo/` — alpha combination → `combos/*.pq`
|
||||||
- `pipeline/common/schema.py` — parquet column contracts
|
- `pipeline/common/schema.py` — parquet column contracts
|
||||||
- `data/downloader.py`, `data/universe.py` — baostock/akshare download + constituents
|
- `data/downloader.py`, `data/universe.py` — baostock/akshare download + constituents
|
||||||
|
- `tools/pqcat.py` — standalone parquet inspector (`pqcat`), also wired as `cli.py pqcat`
|
||||||
- `examples/alphas/` — example external alpha(s)
|
- `examples/alphas/` — example external alpha(s)
|
||||||
|
|
||||||
## Roadmap (not yet implemented)
|
## Roadmap (not yet implemented)
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import click
|
|||||||
from pipeline.data.cli import data
|
from pipeline.data.cli import data
|
||||||
from pipeline.alpha.cli import alpha
|
from pipeline.alpha.cli import alpha
|
||||||
from pipeline.combo.cli import combo
|
from pipeline.combo.cli import combo
|
||||||
|
from tools.pqcat import pqcat
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
@@ -37,6 +38,7 @@ def cli(log_level):
|
|||||||
cli.add_command(data)
|
cli.add_command(data)
|
||||||
cli.add_command(alpha)
|
cli.add_command(alpha)
|
||||||
cli.add_command(combo)
|
cli.add_command(combo)
|
||||||
|
cli.add_command(pqcat)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Standalone helper tools (not part of the data→alpha→combo pipeline)."""
|
||||||
Executable
+50
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""CLI helper to inspect parquet files, like `cat` for `.pq`."""
|
||||||
|
|
||||||
|
import click
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
|
||||||
|
@click.command(name="pqcat")
|
||||||
|
@click.argument("path", type=click.Path(exists=True))
|
||||||
|
@click.option("-n", "--head", "head", type=int, default=None, help="Show only the first N rows")
|
||||||
|
@click.option("-t", "--tail", "tail", type=int, default=None, help="Show only the last N rows")
|
||||||
|
@click.option("-c", "--columns", default=None, help="Comma-separated subset of columns to show")
|
||||||
|
@click.option("--info", is_flag=True, help="Show shape + dtypes instead of the rows")
|
||||||
|
def pqcat(path, head, tail, columns, info):
|
||||||
|
"""Dump a parquet file (or partitioned dataset dir) to stdout.
|
||||||
|
|
||||||
|
PATH is a single .pq file or a Hive-partitioned dataset directory.
|
||||||
|
"""
|
||||||
|
df = pd.read_parquet(path)
|
||||||
|
|
||||||
|
if columns:
|
||||||
|
cols = [c.strip() for c in columns.split(",") if c.strip()]
|
||||||
|
missing = [c for c in cols if c not in df.columns]
|
||||||
|
if missing:
|
||||||
|
raise click.ClickException(f"Columns not found: {', '.join(missing)}")
|
||||||
|
df = df[cols]
|
||||||
|
|
||||||
|
if info:
|
||||||
|
click.echo(f"path: {path}")
|
||||||
|
click.echo(f"shape: {df.shape[0]:,} rows x {df.shape[1]} cols")
|
||||||
|
click.echo("dtypes:")
|
||||||
|
for name in df.columns:
|
||||||
|
click.echo(f" {name}: {df[name].dtype}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if head is not None:
|
||||||
|
df = df.head(head)
|
||||||
|
elif tail is not None:
|
||||||
|
df = df.tail(tail)
|
||||||
|
|
||||||
|
with pd.option_context(
|
||||||
|
"display.max_rows", None if (head is None and tail is None) else len(df),
|
||||||
|
"display.max_columns", None,
|
||||||
|
"display.width", None,
|
||||||
|
):
|
||||||
|
click.echo(df.to_string(index=False))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
pqcat()
|
||||||
Reference in New Issue
Block a user