// Data Pipelines & ETL — visual DAG designer, CDC, streaming, AI-assisted transforms. const PIPELINES = [ { id: "p1", name: "GeM Ingest → Warehouse", type: "Streaming ETL", status: "running", rows: "1.2M/hr", last: "live" }, { id: "p2", name: "Vendor Master Sync", type: "CDC · Oracle", status: "running", rows: "48K/min", last: "live" }, { id: "p3", name: "Daily Revenue Rollup", type: "Batch ELT", status: "success", rows: "38.4M", last: "5h ago" }, { id: "p4", name: "IoT Sensor Aggregation", type: "Streaming · Kafka", status: "running", rows: "12.4K/s", last: "live" }, { id: "p5", name: "Scheme DBT Reconcile", type: "Batch", status: "failed", rows: "—", last: "1h ago" }, { id: "p6", name: "Legacy Oracle Migrate", type: "Batch · one-off", status: "paused", rows: "1.8M", last: "2d ago" }, ]; const DAG_NODES = [ { id: "n1", x: 16, y: 40, kind: "source", icon: "cloud", title: "GeM Portal API", sub: "REST · 1.2M/hr", active: true }, { id: "n2", x: 16, y: 150, kind: "source", icon: "database", title: "Postgres CDC", sub: "logical repl.", active: true }, { id: "n3", x: 16, y: 260, kind: "source", icon: "activity", title: "Kafka · orders", sub: "12.4K ev/s", active: true }, { id: "n4", x: 220, y: 150, kind: "transform", icon: "filter", title: "Cleanse & Dedup", sub: "6 rules · 0.4% dropped", active: true }, { id: "n5", x: 424, y: 96, kind: "ai", icon: "sparkles", title: "AI Join", sub: "vendor_id · 98% conf", active: true }, { id: "n6", x: 424, y: 236, kind: "transform", icon: "map", title: "Enrich — Geo", sub: "state + district", active: true }, { id: "n7", x: 628, y: 96, kind: "sink", icon: "layers", title: "gem_transactions", sub: "428.6M rows", active: true }, { id: "n8", x: 628, y: 236, kind: "sink", icon: "database", title: "ClickHouse mart", sub: "12.4B rows", active: true }, ]; const DAG_EDGES = [ ["n1", "n4"], ["n2", "n4"], ["n3", "n4"], ["n4", "n5"], ["n5", "n7"], ["n5", "n6"], ["n6", "n8"], ]; function PipeNode({ n, selected, onClick }) { const styles = { source: { bg: "var(--panel-2)", bd: "var(--border-strong)", ic: "var(--ink-2)" }, transform: { bg: "var(--panel)", bd: "var(--primary)", ic: "var(--primary)" }, ai: { bg: "var(--primary-tint)", bd: "var(--primary)", ic: "var(--primary)" }, sink: { bg: "var(--accent-tint)", bd: "var(--accent-2)", ic: "var(--accent-2)" }, }[n.kind]; const w = 156, h = 52; return ( onClick(n.id)}> {n.title.length > 15 ? n.title.slice(0, 14) + '…' : n.title} {n.sub} {n.active && } ); } function PipelinesScreen() { const [sel, setSel] = useState("p1"); const [node, setNode] = useState("n5"); const nodeMap = Object.fromEntries(DAG_NODES.map(n => [n.id, n])); const selNode = nodeMap[node]; const stColor = (s) => s === "running" ? "ok" : s === "success" ? "primary" : s === "failed" ? "danger" : ""; return (
{/* Left: pipeline list */}
Pipelines
18 active · 3 streaming
{PIPELINES.map(p => (
setSel(p.id)} style={{ padding: 10, borderRadius: 8, marginBottom: 5, cursor: 'pointer', border: '1px solid ' + (sel === p.id ? 'color-mix(in oklch, var(--primary), transparent 65%)' : 'transparent'), background: sel === p.id ? 'var(--primary-tint)' : 'transparent' }}>
{p.name}
{p.type}{p.rows}
))}
{/* Center: DAG canvas */}
GeM Ingest → Warehouse
Running Streaming ETL
{DAG_EDGES.map(([a, b], i) => { const na = nodeMap[a], nb = nodeMap[b]; const x1 = na.x + 156, y1 = na.y + 26, x2 = nb.x, y2 = nb.y + 26; const mx = (x1 + x2) / 2; return ( ); })} {DAG_NODES.map(n => )}
{/* Run log strip */}
Run log tailing
14:44:12 INFO Cleanse & Dedup — 428,612 rows in, 426,901 out (0.4% dropped)
14:44:12 AI Join resolved vendor_id ↔ vendor_master.id at 98.2% confidence
14:44:13 INFO Enrich — geo-tagged 426,901 rows (state + district)
14:44:13 INFO Sink gem_transactions ← upsert 426,901 · 218ms
14:44:14 WARN ClickHouse mart backpressure — buffer 68% · throttling
{/* Right: inspector */}
Node inspector
{selNode.title}
{selNode.kind === "ai" && (
AI-assisted transform
Suggested join key vendor_id ↔ vendor_master.id with 98.2% confidence. Also detected a datatype cast for gst_no.
)}
Throughput
{selNode.sub}
Config
{[["Type", selNode.kind], ["Parallelism", "8 workers"], ["On error", "Dead-letter queue"], ["Checkpoint", "every 30s"]].map(([k, v]) => (
{k}{v}
))}
CDC status
Replication lag 0.8s
Log-based CDC on Oracle + Postgres · exactly-once · schema-drift auto-handled.
); } window.PipelinesScreen = PipelinesScreen;