feat: add pqcat helper to inspect parquet files

A standalone tools/pqcat.py (head/tail/columns/info) wired as `cli.py pqcat`
and runnable directly via a PATH symlink, plus README docs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Yuxuan Yan
2026-06-09 15:33:55 +08:00
parent de43444ad4
commit f4b7ef3ae6
4 changed files with 80 additions and 0 deletions
+27
View File
@@ -153,6 +153,32 @@ python3 cli.py combo combine \
--combo-name eq --method equal_weight --combo-name eq --method equal_weight
``` ```
### `pqcat` — inspect a parquet file, like `cat`
Quickly dump any pipeline parquet (a single `.pq` file or a partitioned dataset
directory) to stdout, without writing a throwaway script.
| Option | Default | Description |
| --- | --- | --- |
| `-n, --head N` | — | Show only the first `N` rows |
| `-t, --tail N` | — | Show only the last `N` rows |
| `-c, --columns` | — | Comma-separated subset of columns to show |
| `--info` | off | Show shape + dtypes instead of the rows |
```bash
python3 cli.py pqcat alphas/reversal_5d.pq # dump all rows
python3 cli.py pqcat data/daily_bars/csi500 --info # shape + dtypes
python3 cli.py pqcat data/daily_bars/csi500 -n 10 -c symbol_id,date,close,vwap
```
**Standalone command.** `tools/pqcat.py` has no repo dependencies, so it can be
run directly. Symlink it onto your `PATH` once and call `pqcat` from anywhere:
```bash
ln -sf "$(pwd)/tools/pqcat.py" ~/.local/bin/pqcat # ~/.local/bin must be on PATH
pqcat alphas/reversal_5d.pq --info
```
## Alphas: the factory / plugin interface ## Alphas: the factory / plugin interface
An **alpha** is a class that maps a wide close matrix (date index × `symbol_id` An **alpha** is a class that maps a wide close matrix (date index × `symbol_id`
@@ -245,6 +271,7 @@ directory yields an extra `month` (`YYYY-MM`) partition column on top of
- `pipeline/combo/` — alpha combination → `combos/*.pq` - `pipeline/combo/` — alpha combination → `combos/*.pq`
- `pipeline/common/schema.py` — parquet column contracts - `pipeline/common/schema.py` — parquet column contracts
- `data/downloader.py`, `data/universe.py` — baostock/akshare download + constituents - `data/downloader.py`, `data/universe.py` — baostock/akshare download + constituents
- `tools/pqcat.py` — standalone parquet inspector (`pqcat`), also wired as `cli.py pqcat`
- `examples/alphas/` — example external alpha(s) - `examples/alphas/` — example external alpha(s)
## Roadmap (not yet implemented) ## Roadmap (not yet implemented)
+2
View File
@@ -14,6 +14,7 @@ import click
from pipeline.data.cli import data from pipeline.data.cli import data
from pipeline.alpha.cli import alpha from pipeline.alpha.cli import alpha
from pipeline.combo.cli import combo from pipeline.combo.cli import combo
from tools.pqcat import pqcat
@click.group() @click.group()
@@ -37,6 +38,7 @@ def cli(log_level):
cli.add_command(data) cli.add_command(data)
cli.add_command(alpha) cli.add_command(alpha)
cli.add_command(combo) cli.add_command(combo)
cli.add_command(pqcat)
if __name__ == "__main__": if __name__ == "__main__":
+1
View File
@@ -0,0 +1 @@
"""Standalone helper tools (not part of the data→alpha→combo pipeline)."""
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""CLI helper to inspect parquet files, like `cat` for `.pq`."""
import click
import pandas as pd
@click.command(name="pqcat")
@click.argument("path", type=click.Path(exists=True))
@click.option("-n", "--head", "head", type=int, default=None, help="Show only the first N rows")
@click.option("-t", "--tail", "tail", type=int, default=None, help="Show only the last N rows")
@click.option("-c", "--columns", default=None, help="Comma-separated subset of columns to show")
@click.option("--info", is_flag=True, help="Show shape + dtypes instead of the rows")
def pqcat(path, head, tail, columns, info):
"""Dump a parquet file (or partitioned dataset dir) to stdout.
PATH is a single .pq file or a Hive-partitioned dataset directory.
"""
df = pd.read_parquet(path)
if columns:
cols = [c.strip() for c in columns.split(",") if c.strip()]
missing = [c for c in cols if c not in df.columns]
if missing:
raise click.ClickException(f"Columns not found: {', '.join(missing)}")
df = df[cols]
if info:
click.echo(f"path: {path}")
click.echo(f"shape: {df.shape[0]:,} rows x {df.shape[1]} cols")
click.echo("dtypes:")
for name in df.columns:
click.echo(f" {name}: {df[name].dtype}")
return
if head is not None:
df = df.head(head)
elif tail is not None:
df = df.tail(tail)
with pd.option_context(
"display.max_rows", None if (head is None and tail is None) else len(df),
"display.max_columns", None,
"display.width", None,
):
click.echo(df.to_string(index=False))
if __name__ == "__main__":
pqcat()