From 89ff93121216fe24655f091458af8562d18506ab Mon Sep 17 00:00:00 2001
From: Deeman <hendriknote@gmail.com>
Date: Fri, 27 Feb 2026 13:23:54 +0100
Subject: [PATCH] feat(lineage): hover tooltip + click-to-inspect schema panel
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- New route GET /admin/pipeline/lineage/schema/<model> — returns JSON
  with columns+types (from information_schema for serving models),
  row count, upstream and downstream model lists. Validates model
  against _DAG to prevent arbitrary table access.
- Precomputes _DOWNSTREAM map at import time from _DAG.
- Lineage template: replaces minimal edge-highlight JS with full UX —
  hover triggers schema prefetch + floating tooltip (layer badge, top 4
  columns, "+N more" note); click opens 320px slide-in panel showing
  row count, full schema table, upstream/downstream dep lists.
  Dep items in panel are clickable to navigate between models.
  Schema responses are cached client-side to avoid repeat fetches.
  Staging/foundation models show "schema in lakehouse.duckdb only".

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 web/src/padelnomics/admin/pipeline_routes.py  |  67 ++++
 .../admin/partials/pipeline_lineage.html      | 285 ++++++++++++++++--
 2 files changed, 335 insertions(+), 17 deletions(-)

diff --git a/web/src/padelnomics/admin/pipeline_routes.py b/web/src/padelnomics/admin/pipeline_routes.py
index cea4070..3ce3ac8 100644
--- a/web/src/padelnomics/admin/pipeline_routes.py
+++ b/web/src/padelnomics/admin/pipeline_routes.py
@@ -767,6 +767,12 @@ async def pipeline_trigger_extract():
 
 # ── Lineage tab ───────────────────────────────────────────────────────────────
 
+# Compute downstream map once at import time (DAG is static).
+_DOWNSTREAM: dict[str, list[str]] = {n: [] for n in _DAG}
+for _name, _deps in _DAG.items():
+    for _dep in _deps:
+        _DOWNSTREAM.setdefault(_dep, []).append(_name)
+
 
 @bp.route("/lineage")
 @role_required("admin")
@@ -780,6 +786,67 @@ async def pipeline_lineage():
     )
 
 
+@bp.route("/lineage/schema/<model>")
+@role_required("admin")
+async def pipeline_lineage_schema(model: str):
+    """JSON: schema details for a lineage node.
+
+    Returns columns + types from information_schema (serving models only —
+    staging/foundation live in lakehouse.duckdb which the web app cannot open).
+    Row count is included for serving models when the table exists.
+    """
+    from quart import jsonify
+
+    from ..analytics import fetch_analytics
+
+    if model not in _DAG:
+        return jsonify({"error": "unknown model"}), 404
+
+    layer = _classify_layer(model)
+    upstream = _DAG[model]
+    downstream = _DOWNSTREAM.get(model, [])
+
+    row_count = None
+    columns: list[dict] = []
+
+    if layer == "serving":
+        col_rows = await fetch_analytics(
+            """
+            SELECT column_name, data_type, is_nullable
+            FROM information_schema.columns
+            WHERE table_schema = 'serving' AND table_name = ?
+            ORDER BY ordinal_position
+            """,
+            [model],
+        )
+        columns = [
+            {
+                "name": r["column_name"],
+                "type": r["data_type"],
+                "nullable": r["is_nullable"] == "YES",
+            }
+            for r in col_rows
+        ]
+        if columns:
+            # model is validated against _DAG keys — safe to interpolate
+            count_rows = await fetch_analytics(
+                f"SELECT count(*) AS n FROM serving.{model}"
+            )
+            if count_rows:
+                row_count = count_rows[0]["n"]
+
+    return jsonify(
+        {
+            "model": model,
+            "layer": layer,
+            "upstream": upstream,
+            "downstream": downstream,
+            "row_count": row_count,
+            "columns": columns,
+        }
+    )
+
+
 # ── Catalog tab ───────────────────────────────────────────────────────────────
 
 
diff --git a/web/src/padelnomics/admin/templates/admin/partials/pipeline_lineage.html b/web/src/padelnomics/admin/templates/admin/partials/pipeline_lineage.html
index 825ed08..d112924 100644
--- a/web/src/padelnomics/admin/templates/admin/partials/pipeline_lineage.html
+++ b/web/src/padelnomics/admin/templates/admin/partials/pipeline_lineage.html
@@ -5,45 +5,296 @@
     <span class="text-xs font-normal text-slate ml-2">
       {{ node_count }} models &mdash; staging &rarr; foundation &rarr; serving
     </span>
+    <span class="text-xs font-normal text-slate" style="margin-left:auto">
+      hover to preview &middot; click to inspect
+    </span>
   </p>
   <div style="overflow-x:auto;padding:1rem 0.5rem 0.5rem">
     {{ lineage_svg | safe }}
   </div>
 </div>
 
+<!-- Detail panel: fixed right, slides in on node click -->
+<div id="ln-panel" style="
+  position:fixed;top:0;right:0;bottom:0;width:320px;
+  background:#fff;border-left:1px solid #E2E8F0;
+  display:flex;flex-direction:column;
+  transform:translateX(100%);
+  transition:transform 0.2s cubic-bezier(0.4,0,0.2,1);
+  z-index:200;overflow:hidden;
+  box-shadow:-4px 0 24px rgba(0,0,0,0.06);
+">
+  <div style="padding:0.75rem 1rem;border-bottom:1px solid #F1F5F9;flex-shrink:0">
+    <div style="display:flex;align-items:center;gap:0.5rem;margin-bottom:0.375rem">
+      <span id="ln-model-name" style="
+        font-family:'Commit Mono',ui-monospace,monospace;
+        font-size:0.8125rem;font-weight:500;color:#1E293B;
+        flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;
+      ">—</span>
+      <button id="ln-close" style="
+        background:none;border:none;cursor:pointer;color:#94A3B8;
+        font-size:1rem;padding:2px 4px;border-radius:4px;line-height:1;
+        transition:color 0.1s,background 0.1s;
+      " title="Close (Esc)">✕</button>
+    </div>
+    <div style="display:flex;align-items:center;gap:0.5rem">
+      <span id="ln-layer-chip" style="
+        font-size:0.625rem;font-weight:700;letter-spacing:0.06em;
+        text-transform:uppercase;padding:2px 7px;border-radius:99px;
+      ">—</span>
+      <span id="ln-mat" style="font-size:0.6875rem;color:#64748B"></span>
+    </div>
+  </div>
+  <div id="ln-body" style="flex:1;overflow-y:auto"></div>
+</div>
+
+<!-- Hover tooltip -->
+<div id="ln-tooltip" style="
+  position:fixed;z-index:300;pointer-events:none;
+  opacity:0;transition:opacity 0.08s;
+  filter:drop-shadow(0 4px 14px rgba(0,0,0,0.14));
+">
+  <div id="ln-tt-inner" style="
+    background:#0F172A;color:#fff;border-radius:8px;
+    padding:10px 12px;min-width:190px;max-width:250px;
+  "></div>
+</div>
+
 <style>
-  .lineage-node { cursor: default; }
-  .lineage-node rect:first-of-type { transition: filter 0.12s; }
-  .lineage-node:hover rect:first-of-type { filter: brightness(0.94); }
+  .lineage-node { cursor: pointer; }
+  .lineage-node rect:first-of-type { transition: filter 0.1s; }
+  .lineage-node:hover rect:first-of-type { filter: brightness(0.92); }
+  .lineage-node.ln-selected rect:first-of-type { filter: brightness(0.86) !important; }
   .lineage-edge { transition: stroke 0.12s, stroke-width 0.12s, opacity 0.12s; }
   .lineage-edge.hi { stroke: #1D4ED8 !important; stroke-width: 2 !important; marker-end: url(#arr-hi) !important; }
-  .lineage-edge.dim { opacity: 0.12; }
+  .lineage-edge.dim { opacity: 0.1; }
+
+  .ln-section { border-bottom: 1px solid #F1F5F9; padding: 0.75rem 1rem; }
+  .ln-section:last-child { border-bottom: none; }
+  .ln-label {
+    font-size: 0.5875rem; font-weight: 700; letter-spacing: 0.08em;
+    text-transform: uppercase; color: #94A3B8; margin-bottom: 0.5rem;
+  }
+  .ln-schema-table { width: 100%; border-collapse: collapse; font-size: 0.6875rem; }
+  .ln-schema-table th {
+    text-align: left; font-weight: 600; color: #64748B;
+    padding: 0 0 0.375rem; font-size: 0.5875rem; letter-spacing: 0.04em; text-transform: uppercase;
+  }
+  .ln-schema-table td { padding: 3px 4px 3px 0; vertical-align: middle; }
+  .ln-schema-table tr + tr td { border-top: 1px solid #F1F5F9; }
+  .ln-col-name { font-family: 'Commit Mono', ui-monospace, monospace; color: #1E293B; font-weight: 500; }
+  .ln-col-type { font-family: 'Commit Mono', ui-monospace, monospace; color: #94A3B8; font-size: 0.625rem; }
+  .ln-col-null { font-size: 0.5625rem; color: #CBD5E1; text-align: right; }
+  .ln-col-null.yes { color: #D97706; }
+
+  .ln-dep-item {
+    display: flex; align-items: center; gap: 6px; padding: 4px 6px;
+    border-radius: 5px; cursor: pointer;
+    font-family: 'Commit Mono', ui-monospace, monospace;
+    font-size: 0.6875rem; color: #334155; transition: background 0.1s;
+  }
+  .ln-dep-item:hover { background: #F8FAFC; }
+  .ln-dep-dot { width: 6px; height: 6px; border-radius: 50%; flex-shrink: 0; }
+  .ln-dep-dot.staging    { background: #16A34A; }
+  .ln-dep-dot.foundation { background: #1D4ED8; }
+  .ln-dep-dot.serving    { background: #D97706; }
+
+  .ln-chip-staging    { background: #DCFCE7; color: #14532D; border: 1px solid #BBF7D0; }
+  .ln-chip-foundation { background: #DBEAFE; color: #1E3A8A; border: 1px solid #BFDBFE; }
+  .ln-chip-serving    { background: #FEF3C7; color: #78350F; border: 1px solid #FDE68A; }
 </style>
 
 <script>
 (function () {
+  var SCHEMA_BASE = "{{ url_for('pipeline.pipeline_lineage_schema', model='MODEL') }}".replace('/MODEL', '/');
   var svg = document.querySelector('.lineage-svg');
   if (!svg) return;
   var nodes = svg.querySelectorAll('.lineage-node');
   var edges = svg.querySelectorAll('.lineage-edge');
+  var panel = document.getElementById('ln-panel');
+  var panelBody = document.getElementById('ln-body');
+  var tooltip = document.getElementById('ln-tooltip');
+  var ttInner = document.getElementById('ln-tt-inner');
+  var activeModel = null;
+  var cache = {};
 
+  // ── Helpers ────────────────────────────────────────────────────────────
+  function layer(model) {
+    if (model.startsWith('stg_')) return 'staging';
+    if (model.startsWith('dim_') || model.startsWith('fct_')) return 'foundation';
+    return 'serving';
+  }
+
+  function fmt(n) {
+    return n == null ? '—' : Number(n).toLocaleString();
+  }
+
+  // ── Edge highlight ─────────────────────────────────────────────────────
+  function highlightEdges(model) {
+    edges.forEach(function (e) {
+      if (e.dataset.from === model || e.dataset.to === model) {
+        e.classList.add('hi'); e.classList.remove('dim');
+      } else {
+        e.classList.add('dim'); e.classList.remove('hi');
+      }
+    });
+  }
+
+  function clearEdges() {
+    edges.forEach(function (e) { e.classList.remove('hi', 'dim'); });
+  }
+
+  // ── Schema fetch (cached) ──────────────────────────────────────────────
+  function fetchSchema(model, cb) {
+    if (cache[model]) { cb(cache[model]); return; }
+    fetch(SCHEMA_BASE + encodeURIComponent(model))
+      .then(function (r) { return r.json(); })
+      .then(function (d) { cache[model] = d; cb(d); })
+      .catch(function () {
+        var fallback = { model: model, layer: layer(model), columns: [], upstream: [], downstream: [], row_count: null };
+        cache[model] = fallback;
+        cb(fallback);
+      });
+  }
+
+  // ── Tooltip ────────────────────────────────────────────────────────────
+  function showTooltip(data, x, y) {
+    var cols = data.columns || [];
+    var preview = cols.slice(0, 4);
+    var extra = cols.length - 4;
+    var lc = layer(data.model);
+    var badge = lc === 'staging'
+      ? 'background:rgba(22,163,74,0.25);color:#86EFAC'
+      : lc === 'foundation'
+        ? 'background:rgba(29,78,216,0.3);color:#93C5FD'
+        : 'background:rgba(217,119,6,0.25);color:#FCD34D';
+
+    ttInner.innerHTML =
+      '<div style="display:flex;align-items:center;gap:6px;margin-bottom:8px;padding-bottom:7px;border-bottom:1px solid rgba(255,255,255,0.08)">' +
+        '<span style="font-family:\'Commit Mono\',monospace;font-size:0.6875rem;font-weight:500;flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">' + data.model + '</span>' +
+        '<span style="font-size:0.5625rem;font-weight:700;letter-spacing:0.07em;text-transform:uppercase;padding:2px 5px;border-radius:3px;flex-shrink:0;' + badge + '">' + lc + '</span>' +
+      '</div>' +
+      (cols.length === 0
+        ? '<div style="font-size:0.6875rem;color:#475569;font-style:italic">schema in lakehouse only</div>'
+        : preview.map(function (c) {
+            return '<div style="display:flex;align-items:baseline;gap:6px;padding:2px 0">' +
+              '<span style="font-family:\'Commit Mono\',monospace;font-size:0.6875rem;color:#CBD5E1;flex:1;overflow:hidden;text-overflow:ellipsis">' + c.name + '</span>' +
+              '<span style="font-family:\'Commit Mono\',monospace;font-size:0.625rem;color:#475569;flex-shrink:0">' + c.type + '</span>' +
+            '</div>';
+          }).join('') +
+          (extra > 0
+            ? '<div style="font-size:0.625rem;color:#475569;margin-top:5px;padding-top:5px;border-top:1px solid rgba(255,255,255,0.06)">+' + extra + ' more &mdash; click to view all</div>'
+            : '<div style="font-size:0.625rem;color:#334155;margin-top:5px;padding-top:5px;border-top:1px solid rgba(255,255,255,0.06);font-style:italic">click to inspect</div>')
+      );
+
+    var vw = window.innerWidth, vh = window.innerHeight;
+    var left = x + 14, top = y - 10;
+    if (left + 252 > vw - 12) left = x - 252 - 14;
+    if (top + 160 > vh - 12) top = vh - 160 - 12;
+    tooltip.style.left = left + 'px';
+    tooltip.style.top = top + 'px';
+    tooltip.style.opacity = '1';
+  }
+
+  function hideTooltip() { tooltip.style.opacity = '0'; }
+
+  // ── Panel ──────────────────────────────────────────────────────────────
+  function depItems(list) {
+    if (!list.length) return '<div style="font-size:0.6875rem;color:#94A3B8;font-style:italic">none</div>';
+    return list.map(function (d) {
+      return '<div class="ln-dep-item" data-model="' + d + '"><span class="ln-dep-dot ' + layer(d) + '"></span>' + d + '</div>';
+    }).join('');
+  }
+
+  function renderPanel(data) {
+    var cols = data.columns || [];
+    var lc = data.layer || layer(data.model);
+
+    document.getElementById('ln-mat').textContent =
+      cols.length > 0 ? 'table' : (lc === 'serving' ? '' : '');
+
+    panelBody.innerHTML =
+      '<div class="ln-section">' +
+        '<div class="ln-label">Row count</div>' +
+        (data.row_count != null
+          ? '<div style="font-family:\'Commit Mono\',monospace;font-size:1rem;font-weight:500;color:#1E293B">' + fmt(data.row_count) +
+            ' <span style="font-size:0.6875rem;font-weight:400;color:#64748B">rows</span></div>'
+          : '<div style="font-size:0.6875rem;color:#94A3B8;font-style:italic">' +
+            (lc !== 'serving' ? 'staging/foundation — in lakehouse.duckdb' : 'not yet built') + '</div>') +
+      '</div>' +
+
+      '<div class="ln-section">' +
+        '<div class="ln-label">Schema &middot; ' + cols.length + ' columns</div>' +
+        (cols.length > 0
+          ? '<table class="ln-schema-table"><thead><tr><th>column</th><th>type</th><th style="text-align:right">null?</th></tr></thead><tbody>' +
+            cols.map(function (c) {
+              return '<tr><td class="ln-col-name">' + c.name + '</td><td class="ln-col-type">' + c.type + '</td>' +
+                '<td class="ln-col-null' + (c.nullable ? ' yes' : '') + '">' + (c.nullable ? 'null' : '—') + '</td></tr>';
+            }).join('') + '</tbody></table>'
+          : '<div style="font-size:0.6875rem;color:#94A3B8;font-style:italic">schema available in lakehouse.duckdb only</div>') +
+      '</div>' +
+
+      '<div class="ln-section"><div class="ln-label">Upstream &middot; ' + (data.upstream || []).length + '</div>' + depItems(data.upstream || []) + '</div>' +
+      '<div class="ln-section"><div class="ln-label">Downstream &middot; ' + (data.downstream || []).length + '</div>' + depItems(data.downstream || []) + '</div>';
+
+    panelBody.querySelectorAll('.ln-dep-item').forEach(function (el) {
+      el.addEventListener('click', function () { openPanel(el.dataset.model); });
+    });
+  }
+
+  function openPanel(model) {
+    activeModel = model;
+    document.getElementById('ln-model-name').textContent = model;
+    var chip = document.getElementById('ln-layer-chip');
+    var lc = layer(model);
+    chip.textContent = lc;
+    chip.className = 'ln-chip-' + lc;
+    chip.setAttribute('style',
+      'font-size:0.625rem;font-weight:700;letter-spacing:0.06em;text-transform:uppercase;padding:2px 7px;border-radius:99px;');
+    document.getElementById('ln-mat').textContent = '';
+    panelBody.innerHTML = '<div style="padding:2rem 1rem;font-size:0.75rem;color:#94A3B8">Loading\u2026</div>';
+    panel.style.transform = 'translateX(0)';
+
+    nodes.forEach(function (n) { n.classList.toggle('ln-selected', n.dataset.model === model); });
+    highlightEdges(model);
+
+    fetchSchema(model, renderPanel);
+  }
+
+  function closePanel() {
+    panel.style.transform = 'translateX(100%)';
+    clearEdges();
+    nodes.forEach(function (n) { n.classList.remove('ln-selected'); });
+    activeModel = null;
+  }
+
+  document.getElementById('ln-close').addEventListener('click', closePanel);
+  document.addEventListener('keydown', function (e) { if (e.key === 'Escape') closePanel(); });
+
+  // ── Node wiring ────────────────────────────────────────────────────────
   nodes.forEach(function (g) {
     var model = g.dataset.model;
-    g.addEventListener('mouseenter', function () {
-      edges.forEach(function (e) {
-        if (e.dataset.from === model || e.dataset.to === model) {
-          e.classList.add('hi');
-          e.classList.remove('dim');
-        } else {
-          e.classList.add('dim');
-          e.classList.remove('hi');
-        }
-      });
+
+    g.addEventListener('mouseenter', function (e) {
+      if (activeModel === null) highlightEdges(model);
+      // prefetch so tooltip has data on arrive
+      fetchSchema(model, function (data) { showTooltip(data, e.clientX, e.clientY); });
     });
+
+    g.addEventListener('mousemove', function (e) {
+      if (cache[model]) showTooltip(cache[model], e.clientX, e.clientY);
+    });
+
     g.addEventListener('mouseleave', function () {
-      edges.forEach(function (e) {
-        e.classList.remove('hi', 'dim');
-      });
+      hideTooltip();
+      if (activeModel === null) clearEdges();
+      else highlightEdges(activeModel);
+    });
+
+    g.addEventListener('click', function () {
+      hideTooltip();
+      if (activeModel === model) closePanel();
+      else openPanel(model);
     });
   });
 })();