"""
scrape_lesson_content.py  v3
Navega cada lección del curso con Playwright y extrae el contenido
textual que aparece debajo del video (texto bilingüe ES/EN, secciones).

Estructura de la página ClickFunnels:
  [Navegación lateral con menú del curso — siempre igual]
  [Título de la lección]
  [Video Vidalytics]
  1. Escucha con traducción
     Hello, my name is David.
     Hola, mi nombre es David.
     ...
  2. Escucha sin traducción
  3. Escucha y practica y responde
  << SECCIÓN PREVIA | SIGUIENTE SECCIÓN >>
  © Kale Anders...

Estrategia:
  1. document.body.innerText completo
  2. Eliminar navegación lateral de la parte superior (patrones conocidos)
  3. Eliminar footer (SECCIÓN PREVIA / copyright)
  4. Lo que queda = contenido de la lección
  5. Formatear como markdown
"""
import asyncio
import json
import re
import subprocess
import sys
import time
from pathlib import Path

from playwright.async_api import async_playwright

sys.stdout.reconfigure(encoding="utf-8")

SCRIPTS_DIR    = Path(__file__).parent
BASE_DIR       = SCRIPTS_DIR.parent
COOKIES_FILE   = SCRIPTS_DIR / ".course_cookies.json"
STRUCTURE_FILE = SCRIPTS_DIR / "course_structure.json"
MAP_FILE       = SCRIPTS_DIR / "course_map.json"
MEMBERS_URL    = "https://kaleanders.clickfunnels.com/members-raio"
DEBUG_DIR      = SCRIPTS_DIR / "debug_content"

# Lessons to SKIP (intro video only, no text content worth scraping)
SKIP_NUMS = {1}

# Seconds to wait after navigating to a lesson for content to render
PAGE_WAIT = 7


def log(msg: str, level: str = "INFO") -> None:
    icons = {"INFO": "   ", "OK": "[OK]", "SKIP": "[--]", "ERROR": "[!!]", "HEAD": "\n==="}
    print(f"{icons.get(level, '   ')} {msg}", flush=True)


# ── Text parsing ──────────────────────────────────────────────────────────────

# Lines that are navigation/sidebar items (appear before lesson content).
# Anchored to start-of-stripped-line so they don't match mid-sentence refs.
NAV_LINE_RE = re.compile(
    r'^(?:'
    r'\[.*\]'                               # [¡COMIENZA AQUÍ!] / [YES/NO]
    r'|\s*CAPÍTULO\s+\d+'                   # CAPÍTULO 1: ...
    r'|\s*BONO\s+\d+'                       # BONO 1: ...
    r'|\s*RAIO\s+UNLIMITED:'                # RAIO UNLIMITED: ...
    r'|\s*EMBAJADORES\s+RAIO:'              # EMBAJADORES RAIO: ...
    r'|\s*ENTREVISTAS\s+DE\s+ESTUDIANTES:'  # ENTREVISTAS DE ESTUDIANTES: ...
    r'|\s*¿Quieres\s+clases\s+privadas'     # header nav
    r'|\s*"?SWIMMING\s+IN\s+THE\s+OCEAN'   # special section name
    r'|\s*"?SWIMMING\s+WITHOUT\s+FLOATIES'
    r'|\s*¡MÁS\s+HISTORIAS'
    r'|\s*MÁS\s+HISTORIAS'
    r')',
    re.I,
)

# Lines that are video/audio player UI noise (appear mid-content, must be stripped)
PLAYER_NOISE_RE = re.compile(
    r'^(?:'
    r'Play$'
    r'|\d+:\d+(?::\d+)?$'               # timestamps: 00:00, 1:23:45
    r'|Closed\s+Captions$'
    r'|Settings$'
    r'|Fullscreen$'
    r'|Velocidad:\s*$'
    r'|\d+\.\d+x$'                      # 0.50x, 0.75x, 1.25x etc.
    r'|1x$'                             # 1x speed
    r'|2x$'                             # 2x speed
    r'|Descargar\s+texto.*$'            # download button
    r'|Section\s+\d+,\s+Chapter\s+\d+$'# "Section 1, Chapter 1"
    r'|Normal\s+version$'
    r'|Interactive\s+version$'
    r')',
    re.I,
)

# Lines that mark the footer (end of lesson content)
FOOTER_RE = re.compile(
    r'(?:'
    r'<<\s*SECCIÓN\s+PREVIA'               # << SECCIÓN PREVIA
    r'|SIGUIENTE\s+SECCIÓN\s*>>'           # SIGUIENTE SECCIÓN >>
    r'|©\s*Kale\s+Anders'                  # copyright
    r'|All\s+Rights\s+Reserved'
    r')',
    re.I,
)

# Section header pattern: "1. Escucha con traducción" or "1) Listen with..."
SECTION_RE = re.compile(
    r'^(\d+)\s*[.\)]\s*(.+)$',
    re.I,
)

# Question-style headers (bold questions in FAQ format)
QUESTION_RE = re.compile(
    r'^¿[A-ZÁÉÍÓÚÜÑ]',
    re.U,
)


def strip_nav_and_footer(body_text: str) -> str:
    """
    Remove the navigation sidebar from the top and the footer from the bottom.
    Returns only the lesson-specific content.
    """
    lines = body_text.splitlines()
    start_idx = 0
    end_idx = len(lines)

    # Find start of lesson content: skip navigation lines at the top
    # Navigation lines appear BEFORE the first meaningful content line.
    # We keep track of the last nav line seen and skip up to that point.
    last_nav_idx = -1
    for i, line in enumerate(lines):
        stripped = line.strip()
        if not stripped:
            continue
        if NAV_LINE_RE.search(stripped):
            last_nav_idx = i

    # Lesson content starts right after the last navigation line
    if last_nav_idx >= 0:
        start_idx = last_nav_idx + 1

    # Find footer: first line matching the footer pattern
    for i in range(start_idx, len(lines)):
        if FOOTER_RE.search(lines[i]):
            end_idx = i
            break

    content_lines = [
        l for l in lines[start_idx:end_idx]
        if not PLAYER_NOISE_RE.match(l.strip())
    ]
    return "\n".join(content_lines).strip()


def text_to_markdown(content: str, lesson_title: str = "") -> str:
    """
    Convert extracted lesson text content into structured markdown.
    Handles both story format (numbered sections + bilingual text)
    and FAQ format (question/answer).
    """
    if not content:
        return ""

    lines = [l.rstrip() for l in content.splitlines()]
    md = []
    in_blank_run = False

    if lesson_title:
        md.append(f"## {lesson_title}")
        md.append("")

    for line in lines:
        stripped = line.strip()

        if not stripped:
            if not in_blank_run:
                md.append("")
                in_blank_run = True
            continue
        in_blank_run = False

        # Numbered section headers: "1. Escucha con traducción"
        m = SECTION_RE.match(stripped)
        if m:
            md.append("")
            md.append(f"### {stripped}")
            md.append("")
            continue

        # Question-style FAQ headers in bold
        if QUESTION_RE.match(stripped) and len(stripped) < 120:
            md.append("")
            md.append(f"**{stripped}**")
            md.append("")
            continue

        md.append(stripped)

    text = "\n".join(md)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()


# ── Navigation ────────────────────────────────────────────────────────────────

async def click_next(page) -> bool:
    for sel in [
        "a[href='#next-lesson']", "#next-lesson",
        "a:has-text('SIGUIENTE SECCIÓN')", "a:has-text('Siguiente')", "a:has-text('Next')",
        "button:has-text('Siguiente')", "button:has-text('Next')",
    ]:
        try:
            btn = await page.query_selector(sel)
            if btn and await btn.is_visible():
                await btn.click()
                return True
        except Exception:
            pass
    return False


# ── Content extraction ────────────────────────────────────────────────────────

async def get_body_text(page) -> str:
    """Get the full body innerText from the page."""
    try:
        return await page.evaluate("() => document.body.innerText || ''")
    except Exception:
        return ""


async def scrape_lesson(page, lesson_num: int, lesson_title: str = "", debug: bool = False) -> str | None:
    """
    Extract lesson content from the current ClickFunnels page.
    Returns markdown string or None if nothing useful found.
    """
    raw = await get_body_text(page)

    if debug:
        DEBUG_DIR.mkdir(exist_ok=True)
        (DEBUG_DIR / f"{lesson_num:02d}_body_v3.txt").write_text(raw, encoding="utf-8")
        log(f"  [debug] body saved ({len(raw)} chars)")

    content = strip_nav_and_footer(raw)

    if not content or len(content) < 60:
        log(f"  [!] Sin contenido después de quitar nav/footer (quedan {len(content)} chars)", "ERROR")
        return None

    return text_to_markdown(content, lesson_title)


# ── Main ──────────────────────────────────────────────────────────────────────

async def main():
    log("RAIO Lesson Content Scraper v3", "HEAD")
    DEBUG_DIR.mkdir(exist_ok=True)

    lessons        = json.loads(STRUCTURE_FILE.read_text(encoding="utf-8"))
    cookies        = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
    course_map     = json.loads(MAP_FILE.read_text(encoding="utf-8"))
    map_by_num     = {l["num"]: l for l in course_map}

    to_scrape = [l for l in lessons if l["num"] not in SKIP_NUMS]
    log(f"Lecciones a procesar: {len(to_scrape)} (de {len(lessons)} totales)")

    scraped = 0
    failed  = 0
    skipped = 0

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=True)
        ctx = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
            viewport={"width": 1280, "height": 900},
        )
        await ctx.add_cookies(cookies)
        page = await ctx.new_page()

        log("Cargando página del curso...")
        await page.goto(MEMBERS_URL, wait_until="domcontentloaded", timeout=45000)

        for i, lesson in enumerate(lessons):
            num   = lesson["num"]
            title = map_by_num.get(num, {}).get("title", f"Lección {num}")

            # Wait for page to fully render (covers both initial load and post-navigation)
            await page.wait_for_timeout(PAGE_WAIT * 1000)

            if num in SKIP_NUMS:
                log(f"[{num:02d}] Intro — saltando", "SKIP")
                if i < len(lessons) - 1:
                    await click_next(page)
                skipped += 1
                continue

            # Debug first 4 lessons to verify extraction
            do_debug = num <= 4

            log(f"[{num:02d}] {title[:50]}...")
            content_md = await scrape_lesson(page, num, title, debug=do_debug)

            if content_md and len(content_md) > 60:
                map_entry = map_by_num.get(num)
                if map_entry is not None:
                    map_entry["notes"] = content_md
                    log(f"[{num:02d}] {len(content_md)} chars extraídos", "OK")
                    scraped += 1
                else:
                    log(f"[{num:02d}] No encontrado en course_map.json", "ERROR")
                    failed += 1
            else:
                log(f"[{num:02d}] Sin contenido útil", "SKIP")
                failed += 1

            # Navigate to next lesson — no wait here, wait is at top of next iteration
            if i < len(lessons) - 1:
                clicked = await click_next(page)
                if not clicked:
                    log("Sin botón siguiente. Fin.", "OK")
                    break

        await browser.close()

    # Save updated course_map.json
    MAP_FILE.write_text(
        json.dumps(course_map, ensure_ascii=False, indent=2),
        encoding="utf-8",
    )
    log(f"course_map.json guardado — {scraped} actualizadas, {failed} sin contenido", "OK")

    if scraped > 0:
        log("Actualizando DB...", "INFO")
        subprocess.run(
            ["php", "artisan", "course:import", "--force"],
            cwd=str(BASE_DIR),
            check=True,
        )
        # Restore lesson 2 availability (no video so import marks it unavailable)
        subprocess.run(
            ["php", "artisan", "tinker", "--execute",
             "App\\Models\\Lesson::where('position', 2)->update(['is_available' => true]); echo 'lesson2 OK';"],
            cwd=str(BASE_DIR),
        )
        log("Listo.", "OK")
    else:
        log("Sin cambios — no se actualizó la DB.", "SKIP")

    log(f"Totales → OK: {scraped} | Sin contenido: {failed} | Saltadas: {skipped}")


if __name__ == "__main__":
    asyncio.run(main())