"""ICE certified Coffee C warehouse stock extraction.

Downloads daily certified stock reports from the ICE Report Center and stores
as gzip CSV in the landing directory. Uses SHA256 of content as the
idempotency key — skips if a file with the same hash already exists.

Landing paths:
  LANDING_DIR/ice_stocks/{year}/{date}_{hash8}.csv.gzip       (daily rolling stocks)
  LANDING_DIR/ice_aging/{year}/{date}_{hash8}.csv.gzip        (monthly aging report)
  LANDING_DIR/ice_stocks_by_port/{year}/{date}_{hash8}.csv.gzip (historical by port)

CSV schemas:
  ice_stocks:      report_date,total_certified_bags,pending_grading_bags
  ice_aging:       report_date,age_bucket,antwerp_bags,hamburg_bremen_bags,
                   houston_bags,miami_bags,new_orleans_bags,new_york_bags,total_bags
  ice_stocks_by_port: report_date,new_york_bags,new_orleans_bags,houston_bags,
                      miami_bags,antwerp_bags,hamburg_bremen_bags,barcelona_bags,
                      virginia_bags,total_bags
"""

import csv
import gzip
import io
import logging
import os
import sys
from datetime import datetime
from pathlib import Path

import niquests
import xlrd
from extract_core import (
    content_hash,
    end_run,
    landing_path,
    open_state_db,
    start_run,
    write_bytes_atomic,
)

from ice_stocks.ice_api import find_all_reports, find_latest_report
from ice_stocks.xls_parse import OLE2_MAGIC, detect_file_format, xls_to_rows

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
    handlers=[logging.StreamHandler(sys.stdout)],
)
logger = logging.getLogger("ICE Stocks Extractor")

LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing"))

# ── ice_stocks (daily rolling) ──────────────────────────────────────────────
DEST_SUBDIR = "ice_stocks"

# Static rolling CSV URL — try this first, fall back to API on 404.
ICE_ROLLING_CSV_URL = (
    "https://www.theice.com/publicdocs/futures_us/exchange_notices/coffee_certifiedstocks.csv"
)
ICE_STOCKS_LABEL = "Daily Warehouse Stocks"

HTTP_TIMEOUT_SECONDS = 60

COLUMN_MAPPINGS = {
    "date": "report_date",
    "report date": "report_date",
    "Date": "report_date",
    "certified stocks": "total_certified_bags",
    "Certified Stocks": "total_certified_bags",
    "certified stocks (bags)": "total_certified_bags",
    "total certified": "total_certified_bags",
    "pending grading": "pending_grading_bags",
    "Pending Grading": "pending_grading_bags",
    "pending grading (bags)": "pending_grading_bags",
}

# ── ice_aging (monthly aging report) ────────────────────────────────────────
ICE_AGING_LABEL = "Certified Stock Aging Report"
AGING_DEST_SUBDIR = "ice_aging"

AGING_PORT_HEADERS = [
    "antwerp_bags",
    "hamburg_bremen_bags",
    "houston_bags",
    "miami_bags",
    "new_orleans_bags",
    "new_york_bags",
    "total_bags",
]

# ── ice_stocks_by_port (historical end-of-month) ─────────────────────────────
ICE_HISTORICAL_URL = (
    "https://www.ice.com/publicdocs/futures_us_reports/coffee/"
    "EOM_KC_cert_stox_by_port_nov96-present.xls"
)
HISTORICAL_DEST_SUBDIR = "ice_stocks_by_port"
HISTORICAL_HTTP_TIMEOUT_SECONDS = 120

HISTORICAL_PORT_COLS = [
    "new_york_bags",
    "new_orleans_bags",
    "houston_bags",
    "miami_bags",
    "antwerp_bags",
    "hamburg_bremen_bags",
    "barcelona_bags",
    "virginia_bags",
    "total_bags",
]


# ── shared helpers ───────────────────────────────────────────────────────────

def _write_landing_file(canonical_csv: bytes, dest_subdir: str, date_label: str) -> int:
    """SHA256-hash canonical_csv, skip if exists, else gzip and write atomically.

    Returns bytes_written (0 if skipped).
    """
    assert canonical_csv, "canonical_csv must not be empty"
    assert dest_subdir, "dest_subdir must not be empty"
    assert date_label, "date_label must not be empty"

    etag = content_hash(canonical_csv)
    year = date_label[:4]

    dest_dir = landing_path(LANDING_DIR, dest_subdir, year)
    local_file = dest_dir / f"{date_label}_{etag}.csv.gzip"

    if local_file.exists():
        logger.info(f"File {local_file.name} already exists — content unchanged, skipping")
        return 0

    compressed = gzip.compress(canonical_csv)
    bytes_written = write_bytes_atomic(local_file, compressed)

    logger.info(f"Stored {local_file} ({bytes_written:,} bytes)")
    return bytes_written


def _build_csv_bytes(fieldnames: list[str], rows: list[dict]) -> bytes:
    """Serialize list of dicts to CSV bytes."""
    out = io.StringIO()
    writer = csv.DictWriter(out, fieldnames=fieldnames)
    writer.writeheader()
    writer.writerows(rows)
    return out.getvalue().encode("utf-8")


# ── ice_stocks (daily rolling) ───────────────────────────────────────────────

def _normalize_row(row: dict) -> dict | None:
    """Map raw ICE CSV columns to canonical schema. Returns None if date missing."""
    normalized = {}
    for raw_key, value in row.items():
        canonical = COLUMN_MAPPINGS.get(raw_key.strip()) or COLUMN_MAPPINGS.get(raw_key.strip().lower())
        if canonical:
            normalized[canonical] = value.strip().replace(",", "") if value else ""

    if "report_date" not in normalized or not normalized["report_date"]:
        return None

    normalized.setdefault("total_certified_bags", "")
    normalized.setdefault("pending_grading_bags", "")
    return normalized


def _build_canonical_csv_from_csv(raw_content: bytes) -> bytes:
    """Parse ICE CSV bytes and emit canonical CSV."""
    text = raw_content.decode("utf-8", errors="replace")
    reader = csv.DictReader(io.StringIO(text))

    rows = []
    for row in reader:
        normalized = _normalize_row(row)
        if normalized:
            rows.append(normalized)

    if not rows:
        return b""

    return _build_csv_bytes(["report_date", "total_certified_bags", "pending_grading_bags"], rows)


def _build_canonical_csv_from_xls(xls_bytes: bytes) -> bytes:
    """Extract total certified bags from ICE daily stocks XLS.

    Sheet structure:
      Row 2: header with report date in cell [0]
      Row 23: ['Total in Bags', ANT, BAR, HA/BR, HOU, MIAMI, NOLA, NY, VA, total]
    """
    rows = xls_to_rows(xls_bytes)

    # Extract report date from row 2, cell 0
    # Formats seen: "As of: 1/30/2026", "As of: Feb 20, 2026  1:35:39PM"
    header_cell = str(rows[2][0]) if len(rows) > 2 else ""
    report_date = ""
    if "as of" in header_cell.lower():
        # Strip prefix, preserve original case for month name parsing
        after = header_cell.strip()
        for prefix in ("As of:", "As Of:", "as of:", "As of", "As Of", "as of"):
            if after.lower().startswith(prefix.lower()):
                after = after[len(prefix):].strip()
                break
        tokens = after.split()
        # Try "M/D/YYYY" (single token) then "Mon DD, YYYY" (three tokens)
        candidates = [tokens[0]] if tokens else []
        if len(tokens) >= 3:
            candidates.append(" ".join(tokens[:3]))
        for candidate in candidates:
            candidate = candidate.rstrip(",")
            for fmt in ("%m/%d/%Y", "%b %d, %Y", "%b %d %Y", "%B %d, %Y", "%B %d %Y"):
                try:
                    dt = datetime.strptime(candidate, fmt)
                    report_date = dt.strftime("%Y-%m-%d")
                    break
                except ValueError:
                    pass
            if report_date:
                break

    if not report_date:
        logger.warning(f"Could not parse report date from XLS header: {header_cell!r}")
        return b""

    # Find "Total in Bags" row
    total_bags = ""
    for row in rows:
        if row and str(row[0]).strip().lower() == "total in bags":
            val = row[-1]
            if isinstance(val, float):
                total_bags = str(int(val))
            else:
                total_bags = str(val).replace(",", "").strip()
            break

    canonical_row = {
        "report_date": report_date,
        "total_certified_bags": total_bags,
        "pending_grading_bags": "",
    }
    return _build_csv_bytes(["report_date", "total_certified_bags", "pending_grading_bags"], [canonical_row])


def extract_ice_stocks() -> None:
    """Download ICE certified Coffee C warehouse stocks and store as gzip CSV.

    Tries static rolling CSV URL first. On 404 or error, falls back to API
    discovery to find the latest 'Daily Warehouse Stocks' report.
    Idempotent: skips if content hash already on disk.
    """
    conn = open_state_db(LANDING_DIR)
    run_id = start_run(conn, "ice_stocks")
    try:
        with niquests.Session() as session:
            logger.info(f"Trying ICE rolling CSV: {ICE_ROLLING_CSV_URL}")
            try:
                response = session.get(ICE_ROLLING_CSV_URL, timeout=HTTP_TIMEOUT_SECONDS)
            except Exception as e:
                logger.warning(f"Rolling CSV fetch failed: {e} — trying API discovery")
                response = None

            use_api = response is None or response.status_code == 404

            if use_api:
                logger.info("Falling back to ICE API discovery for Daily Warehouse Stocks")
                report = find_latest_report(session, ICE_STOCKS_LABEL)
                if not report:
                    logger.error("ICE API: no 'Daily Warehouse Stocks' report found")
                    end_run(conn, run_id, status="failed", error_message="No report found via API")
                    return
                logger.info(f"Found report via API: {report['download_label']} ({report['publish_date']})")
                try:
                    response = session.get(report["download_url"], timeout=HTTP_TIMEOUT_SECONDS)
                except Exception as e:
                    logger.error(f"Failed to download report from API URL: {e}")
                    end_run(conn, run_id, status="failed", error_message=str(e))
                    return

            if response.status_code != 200:
                logger.error(f"Unexpected status {response.status_code}")
                end_run(conn, run_id, status="failed", error_message=f"HTTP {response.status_code}")
                return

            assert len(response.content) > 0, "Downloaded empty file from ICE"

            fmt = detect_file_format(response.content)
            if fmt == "xls":
                canonical_csv = _build_canonical_csv_from_xls(response.content)
            else:
                canonical_csv = _build_canonical_csv_from_csv(response.content)

        if not canonical_csv:
            logger.warning("ICE stocks parsed to 0 rows — check column mapping or XLS structure")
            end_run(conn, run_id, status="failed", error_message="Parsed 0 rows")
            return

        today = datetime.now().strftime("%Y-%m-%d")
        bytes_written = _write_landing_file(canonical_csv, DEST_SUBDIR, today)
        end_run(
            conn, run_id, status="success",
            files_written=1 if bytes_written > 0 else 0,
            files_skipped=1 if bytes_written == 0 else 0,
            bytes_written=bytes_written,
            cursor_value=today,
        )
    except Exception as e:
        end_run(conn, run_id, status="failed", error_message=str(e))
        raise
    finally:
        conn.close()


# ── ice_aging (monthly aging report) ────────────────────────────────────────

def _parse_aging_date(cell_value: str) -> str:
    """Parse 'As of Delivery 3/2/2026' or 'As of: 1/30/2026' → '2026-03-02'."""
    text = str(cell_value).strip()
    for prefix in ("as of delivery ", "as of:"):
        if text.lower().startswith(prefix):
            text = text[len(prefix):].strip()
            break
    date_part = text.split()[0]
    try:
        dt = datetime.strptime(date_part, "%m/%d/%Y")
        return dt.strftime("%Y-%m-%d")
    except ValueError:
        return ""


def extract_ice_aging() -> None:
    """Download ICE Certified Stock Aging Report and store as gzip CSV.

    Monthly report: stock quantities by age bucket × port.
    Idempotent: skips if content hash already on disk.
    """
    conn = open_state_db(LANDING_DIR)
    run_id = start_run(conn, "ice_aging")
    try:
        with niquests.Session() as session:
            logger.info("Fetching latest ICE Aging Report via API")
            report = find_latest_report(session, ICE_AGING_LABEL)
            if not report:
                logger.error(f"ICE API: no report matching {ICE_AGING_LABEL!r}")
                end_run(conn, run_id, status="failed", error_message="No aging report found via API")
                return

            logger.info(f"Downloading: {report['download_label']} ({report['publish_date']})")
            try:
                response = session.get(report["download_url"], timeout=HTTP_TIMEOUT_SECONDS)
            except Exception as e:
                logger.error(f"Failed to download aging report: {e}")
                end_run(conn, run_id, status="failed", error_message=str(e))
                return

            assert response.status_code == 200, f"HTTP {response.status_code}"
            assert response.content[:4] == OLE2_MAGIC, "Aging report is not an XLS file"

        rows = xls_to_rows(response.content)

        report_date = _parse_aging_date(str(rows[0][0]) if rows else "")
        if not report_date:
            msg = f"Could not parse aging report date from row 0: {rows[0] if rows else '(empty)'!r}"
            logger.error(msg)
            end_run(conn, run_id, status="failed", error_message=msg)
            return

        # Row 3+ are data rows; stop at row labelled "Total"
        fieldnames = ["report_date", "age_bucket"] + AGING_PORT_HEADERS
        data_rows = []

        for row in rows[3:]:
            if not row or not str(row[0]).strip():
                continue
            label = str(row[0]).strip()
            if label.lower() == "total":
                break

            port_values = []
            for cell in row[1:]:
                if isinstance(cell, float):
                    port_values.append(str(int(cell)))
                elif str(cell).strip() in ("-", ""):
                    port_values.append("0")
                else:
                    port_values.append(str(cell).replace(",", "").strip())

            while len(port_values) < len(AGING_PORT_HEADERS):
                port_values.append("0")
            port_values = port_values[:len(AGING_PORT_HEADERS)]

            record = {"report_date": report_date, "age_bucket": label}
            for col, val in zip(AGING_PORT_HEADERS, port_values):
                record[col] = val
            data_rows.append(record)

        if not data_rows:
            logger.warning("Aging report parsed to 0 data rows")
            end_run(conn, run_id, status="failed", error_message="Parsed 0 data rows")
            return

        canonical_csv = _build_csv_bytes(fieldnames, data_rows)
        bytes_written = _write_landing_file(canonical_csv, AGING_DEST_SUBDIR, report_date)
        end_run(
            conn, run_id, status="success",
            files_written=1 if bytes_written > 0 else 0,
            files_skipped=1 if bytes_written == 0 else 0,
            bytes_written=bytes_written,
            cursor_value=report_date,
        )
    except Exception as e:
        end_run(conn, run_id, status="failed", error_message=str(e))
        raise
    finally:
        conn.close()


# ── ice_stocks_by_port (historical) ─────────────────────────────────────────

def _excel_serial_to_date(serial: float, datemode: int) -> str:
    """Convert Excel date serial to ISO date string, or '' on failure."""
    try:
        dt = xlrd.xldate_as_datetime(serial, datemode)
        return dt.strftime("%Y-%m-%d")
    except Exception:
        return ""


def extract_ice_historical() -> None:
    """Download ICE historical end-of-month warehouse stocks by port.

    Static URL updated monthly. Covers Nov 1996 to present.
    Idempotent: skips if content hash already on disk.
    """
    conn = open_state_db(LANDING_DIR)
    run_id = start_run(conn, "ice_historical")
    try:
        logger.info(f"Downloading ICE historical by-port XLS: {ICE_HISTORICAL_URL}")

        with niquests.Session() as session:
            try:
                response = session.get(ICE_HISTORICAL_URL, timeout=HISTORICAL_HTTP_TIMEOUT_SECONDS)
            except Exception as e:
                logger.error(f"Failed to download historical XLS: {e}")
                end_run(conn, run_id, status="failed", error_message=str(e))
                return

            assert response.status_code == 200, f"HTTP {response.status_code}"
            assert response.content[:4] == OLE2_MAGIC, "Historical file is not an XLS"

        book = xlrd.open_workbook(file_contents=response.content)
        datemode = book.datemode
        rows = xls_to_rows(response.content)

        # Data starts at row 8 (0-indexed); rows 0-7 are headers
        fieldnames = ["report_date"] + HISTORICAL_PORT_COLS
        data_rows = []

        for row in rows[8:]:
            if not row or len(row) < 2:
                continue

            serial_cell = row[1]
            if not isinstance(serial_cell, float) or serial_cell <= 0:
                continue

            report_date = _excel_serial_to_date(serial_cell, datemode)
            if not report_date:
                continue

            port_cells = row[2:2 + len(HISTORICAL_PORT_COLS)]
            port_values = []
            for cell in port_cells:
                if cell == "" or str(cell).strip() in ("-", ""):
                    port_values.append("0")
                elif isinstance(cell, float):
                    port_values.append(str(int(cell)))
                else:
                    port_values.append(str(cell).replace(",", "").strip())

            while len(port_values) < len(HISTORICAL_PORT_COLS):
                port_values.append("0")

            record = {"report_date": report_date}
            for col, val in zip(HISTORICAL_PORT_COLS, port_values):
                record[col] = val
            data_rows.append(record)

        if not data_rows:
            logger.warning("Historical XLS parsed to 0 data rows")
            end_run(conn, run_id, status="failed", error_message="Parsed 0 data rows")
            return

        canonical_csv = _build_csv_bytes(fieldnames, data_rows)
        today = datetime.now().strftime("%Y-%m-%d")
        bytes_written = _write_landing_file(canonical_csv, HISTORICAL_DEST_SUBDIR, today)
        end_run(
            conn, run_id, status="success",
            files_written=1 if bytes_written > 0 else 0,
            files_skipped=1 if bytes_written == 0 else 0,
            bytes_written=bytes_written,
            cursor_value=today,
        )
    except Exception as e:
        end_run(conn, run_id, status="failed", error_message=str(e))
        raise
    finally:
        conn.close()


def extract_ice_stocks_backfill(max_pages: int = 3) -> None:
    """Download all available historical ICE daily stocks reports.

    Pages through the ICE API (date-descending) and downloads every
    'Daily Warehouse Stocks' XLS found. Each report is one trading day.
    Idempotent: already-downloaded files are skipped via content hash.

    max_pages=3 gives roughly 4 months of history (~120 trading days).
    Use max_pages=20 to fetch all available (~3 years).
    """
    assert max_pages > 0, f"max_pages must be positive, got {max_pages}"

    conn = open_state_db(LANDING_DIR)
    run_id = start_run(conn, "ice_stocks_backfill")
    files_written = 0
    files_skipped = 0
    bytes_written_total = 0
    try:
        with niquests.Session() as session:
            logger.info(f"Fetching all available Daily Warehouse Stocks reports (max {max_pages} pages)...")
            reports = find_all_reports(session, ICE_STOCKS_LABEL, max_pages=max_pages)

            if not reports:
                logger.error("ICE API: no 'Daily Warehouse Stocks' reports found")
                end_run(conn, run_id, status="failed", error_message="No reports found via API")
                return

            logger.info(f"Found {len(reports)} reports: {reports[-1]['publish_date']} → {reports[0]['publish_date']}")

            for report in reports:
                publish_date = report["publish_date"]
                try:
                    response = session.get(report["download_url"], timeout=HTTP_TIMEOUT_SECONDS)
                except Exception as e:
                    logger.warning(f"Failed to download {publish_date}: {e}")
                    continue

                if response.status_code != 200:
                    logger.warning(f"HTTP {response.status_code} for {publish_date} — skipping")
                    continue

                fmt = detect_file_format(response.content)
                if fmt == "xls":
                    canonical_csv = _build_canonical_csv_from_xls(response.content)
                else:
                    canonical_csv = _build_canonical_csv_from_csv(response.content)

                if not canonical_csv:
                    logger.warning(f"Parsed 0 rows for {publish_date} — skipping")
                    continue

                result = _write_landing_file(canonical_csv, DEST_SUBDIR, publish_date)
                if result > 0:
                    files_written += 1
                    bytes_written_total += result
                else:
                    files_skipped += 1

        logger.info(f"Backfill complete: {files_written} new files downloaded, {files_skipped} already existed")
        end_run(
            conn, run_id, status="success",
            files_written=files_written, files_skipped=files_skipped,
            bytes_written=bytes_written_total,
        )
    except Exception as e:
        end_run(conn, run_id, status="failed", error_message=str(e))
        raise
    finally:
        conn.close()


def extract_ice_all() -> None:
    """Run all three ICE extractors: daily stocks, aging report, historical by port."""
    extract_ice_stocks()
    extract_ice_aging()
    extract_ice_historical()


if __name__ == "__main__":
    extract_ice_stocks()
