Python — Data ETL CLI (Sample)
# Python — Data ETL and simple CLI tool # Purpose: Demonstrates reading CSV, basic cleaning, rolling
aggregation, # batching and writing output files. CLI friendly.
import csv import sys import os from datetime import datetime, timedelta from collections import
defaultdict, deque
# --- Configuration --- INPUT_FILE = "input_data.csv" OUTPUT_DIR = "output_batches" BATCH_SIZE = 1000 #
rows per output file DATE_FIELD = "timestamp" VALUE_FIELD = "value"
[Link](OUTPUT_DIR, exist_ok=True)
def parse_row(row): # Example parser: expects timestamp and numeric value ts = [Link](DATE_FIELD) or
[Link]("time") or [Link]("date") val = [Link](VALUE_FIELD) or [Link]("val") or [Link]("value") try:
ts_parsed = [Link](ts) except Exception: # fallback simple formats ts_parsed =
[Link](ts, "%Y-%m-%d %H:%M:%S") try: v = float(val) except Exception: v = 0.0 return {"ts":
ts_parsed, "value": v, "raw": row}
def rolling_average(window): # simple generator computing rolling average over deque s = 0.0 q = deque()
for v in window: [Link](v) s += v if len(q) > 5: s -= [Link]() yield s / len(q)
def process_file(input_path): rows = [] with open(input_path, newline='') as csvfile: reader =
[Link](csvfile) for r in reader: [Link](parse_row(r))
# sort by timestamp [Link](key=lambda x: x["ts"])
# compute hourly buckets buckets = defaultdict(list) for r in rows: hour = r["ts"].replace(minute=0,
second=0, microsecond=0) buckets[hour].append(r["value"])
# compute aggregates and write to batch files batch = [] counter = 0 for hour, values in
sorted([Link]()): agg = { "hour": [Link](), "count": len(values), "sum": sum(values),
"mean": sum(values)/len(values) if values else 0.0, "min": min(values) if values else None, "max":
max(values) if values else None } [Link](agg) if len(batch) >= BATCH_SIZE: counter += 1 out_path =
[Link](OUTPUT_DIR, f"batch_{counter}.csv") write_aggregates(out_path, batch) batch = [] # final
batch if batch: counter += 1 out_path = [Link](OUTPUT_DIR, f"batch_{counter}.csv")
write_aggregates(out_path, batch)
def write_aggregates(path, data): keys = ["hour", "count", "sum", "mean", "min", "max"] with open(path,
"w", newline='') as f: writer = [Link](f, fieldnames=keys) [Link]() for d in data:
[Link](d) print(f"Wrote {len(data)} aggregates to {path}")
def main(argv): input_path = argv[1] if len(argv) > 1 else INPUT_FILE process_file(input_path)
if __name__ == "__main__": main([Link]) # End of Python sample
# filler comment line 1 # filler comment line 2 # filler comment line 3 # filler comment line 4 # filler
comment line 5 # filler comment line 6 # filler comment line 7 # filler comment line 8 # filler comment
line 9 # filler comment line 10 # filler comment line 11 # filler comment line 12 # filler comment line
13 # filler comment line 14 # filler comment line 15 # filler comment line 16 # filler comment line 17 #
filler comment line 18 # filler comment line 19 # filler comment line 20 # filler comment line 21 #
filler comment line 22 # filler comment line 23 # filler comment line 24 # filler comment line 25 #
filler comment line 26 # filler comment line 27 # filler comment line 28 # filler comment line 29 #
filler comment line 30 # filler comment line 31 # filler comment line 32 # filler comment line 33 #
filler comment line 34 # filler comment line 35 # filler comment line 36 # filler comment line 37 #
filler comment line 38 # filler comment line 39 # filler comment line 40 # filler comment line 41 #
filler comment line 42 # filler comment line 43 # filler comment line 44 # filler comment line 45 #
filler comment line 46 # filler comment line 47 # filler comment line 48 # filler comment line 49 #
filler comment line 50 # filler comment line 51 # filler comment line 52 # filler comment line 53 #
filler comment line 54 # filler comment line 55 # filler comment line 56 # filler comment line 57 #
filler comment line 58 # filler comment line 59 # filler comment line 60 # filler comment line 61 #
filler comment line 62 # filler comment line 63 # filler comment line 64 # filler comment line 65 #
filler comment line 66 # filler comment line 67 # filler comment line 68 # filler comment line 69 #
filler comment line 70 # filler comment line 71 # filler comment line 72 # filler comment line 73 #
filler comment line 74 # filler comment line 75 # filler comment line 76 # filler comment line 77 #
filler comment line 78 # filler comment line 79