"""CFTC COT Disaggregated data extraction.

Downloads yearly ZIP files from CFTC and stores as gzip CSV in the landing
directory. CFTC publishes one file per year that updates every Friday at
3:30 PM ET. On first run this backfills all years from 2006. On subsequent
runs it skips files whose etag matches what is already on disk.

Two report variants are supported:
  - Futures-only:          Landing path: LANDING_DIR/cot/{year}/{etag}.csv.gzip
  - Combined (fut+options): Landing path: LANDING_DIR/cot_combined/{year}/{etag}.csv.gzip
"""

import logging
import os
import sys
from datetime import datetime
from io import BytesIO
from pathlib import Path

import niquests
from extract_core import (
    end_run,
    landing_path,
    normalize_etag,
    open_state_db,
    start_run,
    write_bytes_atomic,
)

from .normalize import find_csv_inner_filename, normalize_zipped_csv

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
    handlers=[logging.StreamHandler(sys.stdout)],
)
logger = logging.getLogger("CFTC COT Extractor")

LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing"))

# CFTC publishes yearly ZIPs for both variants of the disaggregated report.
# The file for the current year is updated each Friday at 3:30 PM ET.
COT_URL_FUTURES_ONLY = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip"
COT_URL_COMBINED = "https://www.cftc.gov/files/dea/history/com_disagg_txt_{year}.zip"

FIRST_YEAR = 2006  # Disaggregated report starts June 2006
HTTP_TIMEOUT_SECONDS = 120  # COT ZIPs are up to ~30 MB
MAX_YEARS = 25  # Safety bound on backfill range


def _synthetic_etag(year: int, headers: dict) -> str:
    """Build a dedup key when CFTC omits the etag header.

    Uses content-length + last-modified. This is not as strong as a real etag
    (a server clock change would trigger a re-download), but it is safe because
    the staging layer deduplicates via hash key.
    """
    last_modified = headers.get("last-modified", "")
    content_length = headers.get("content-length", "0")
    etag = f"{year}_{content_length}_{hash(last_modified) & 0xFFFFFFFF:08x}"
    logger.info(f"No etag header for {year}, using synthetic etag: {etag}")
    return etag


def extract_cot_year(year: int, http_session: niquests.Session, url_template: str, landing_subdir: str) -> int:
    """Download and store COT data for a single year.

    Returns bytes_written (0 if skipped or unavailable).
    """
    url = url_template.format(year=year)
    logger.info(f"Checking COT data for {year}: {url}")

    head = http_session.head(url, timeout=HTTP_TIMEOUT_SECONDS)
    if head.status_code == 404:
        logger.info(f"Year {year} not available (404) — skipping")
        return 0
    assert head.status_code == 200, (
        f"Unexpected HEAD status {head.status_code} for {url}"
    )

    raw_etag = head.headers.get("etag", "")
    etag = normalize_etag(raw_etag) if raw_etag else _synthetic_etag(year, head.headers)

    dest_dir = landing_path(LANDING_DIR, landing_subdir, str(year))
    local_file = dest_dir / f"{etag}.csv.gzip"

    if local_file.exists():
        logger.info(f"Year {year}: {etag}.csv.gzip already exists, skipping")
        return 0

    logger.info(f"Downloading COT data for {year}...")
    response = http_session.get(url, timeout=HTTP_TIMEOUT_SECONDS)
    assert response.status_code == 200, (
        f"GET failed with status {response.status_code} for {url}"
    )
    assert len(response.content) > 0, f"Downloaded empty file from {url}"

    zip_buffer = BytesIO(response.content)
    inner_filename = find_csv_inner_filename(BytesIO(response.content))
    normalized = normalize_zipped_csv(zip_buffer, inner_filename)

    bytes_written = write_bytes_atomic(local_file, normalized.read())
    assert local_file.stat().st_size > 0, f"Written file is empty: {local_file}"

    logger.info(f"Stored {local_file} ({bytes_written:,} bytes)")
    return bytes_written


def _extract_cot(url_template: str, landing_subdir: str, extractor_name: str) -> None:
    """Shared extraction loop for any COT report variant.

    Downloads current year first (always re-checks for weekly Friday updates),
    then backfills historical years. Bounded to MAX_YEARS. Continues on
    individual year failures so a single bad year does not abort the run.
    """
    LANDING_DIR.mkdir(parents=True, exist_ok=True)
    current_year = datetime.now().year
    years = list(range(current_year, FIRST_YEAR - 1, -1))
    assert len(years) <= MAX_YEARS, (
        f"Year range {len(years)} exceeds MAX_YEARS={MAX_YEARS}"
    )

    conn = open_state_db(LANDING_DIR)
    run_id = start_run(conn, extractor_name)
    files_written = 0
    files_skipped = 0
    bytes_written_total = 0
    try:
        with niquests.Session() as session:
            for year in years:
                try:
                    result = extract_cot_year(year, session, url_template, landing_subdir)
                    if result > 0:
                        files_written += 1
                        bytes_written_total += result
                    else:
                        files_skipped += 1
                except Exception:
                    logger.exception(f"Failed to extract COT data for {year}, continuing")

        logger.info(f"COT extraction complete ({extractor_name}): {files_written} new file(s) downloaded")
        end_run(
            conn, run_id, status="success",
            files_written=files_written, files_skipped=files_skipped,
            bytes_written=bytes_written_total,
            cursor_value=str(current_year),
        )
    except Exception as e:
        end_run(conn, run_id, status="failed", error_message=str(e))
        raise
    finally:
        conn.close()


def extract_cot_dataset():
    """Extract CFTC COT disaggregated futures-only report."""
    _extract_cot(COT_URL_FUTURES_ONLY, "cot", "cftc_cot")


def extract_cot_combined():
    """Extract CFTC COT disaggregated combined (futures+options) report."""
    _extract_cot(COT_URL_COMBINED, "cot_combined", "cftc_cot_combined")


if __name__ == "__main__":
    extract_cot_dataset()
