"""Crawl autenticado do accia (Playwright) para comparar PHP 7.4 x 8.3.

Uso:
  python crawl.py --rotulo base74                                 # inventario + BFS
  python crawl.py --rotulo alvo83 --repetir .../crawl-base74.json  # mesma lista e ordem da base
"""
import argparse
import hashlib
import json
import re
import subprocess
import time
from collections import deque
from pathlib import Path
from urllib.parse import urljoin, urlsplit

from config import AMBIENTES, BANCO, EXCLUIR, LOGIN_SENHA, LOGIN_USUARIO, MYSQL_CONTAINER, SAIDA
from normaliza import normalizar_html

_ESTATICOS = re.compile(
    r"(?i)^/(assets|uploads|anexo|foto|img|css|js|fonts?)/"
    r"|\.(pdf|xlsx?|docx?|zip|png|jpe?g|gif|svg|css|js|ico|woff2?)$")


def filtrar(urls: list) -> list:
    return [u for u in urls if not EXCLUIR.search(u)]


def normalizar_link(href: str, base_url: str):
    if not href or href.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
        return None
    absoluto = urljoin(base_url + "/", href)
    partes, base = urlsplit(absoluto), urlsplit(base_url)
    if partes.netloc != base.netloc:
        return None
    caminho = partes.path or "/"
    if _ESTATICOS.search(caminho):
        return None
    return caminho + ("?" + partes.query if partes.query else "")


def detecta_erro_yii(html: str) -> bool:
    return 'class="site-error"' in html or "class='site-error'" in html


# ---------- logs ----------

def _mysql(sql: str) -> list:
    r = subprocess.run(
        ["docker", "exec", "-i", MYSQL_CONTAINER, "sh", "-c",
         f'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -N -B {BANCO} 2>/dev/null'],
        input=sql, capture_output=True, text=True, encoding="utf8", errors="replace")
    return [l.split("\t") for l in r.stdout.splitlines() if l]


_SQL_CM_LOG = (
    "select id, level, category, "
    "substring_index(replace(replace(message, char(10), ' '), char(13), ' '), ' Stack trace', 1) "
    "from cm_log where id > {ultimo} and level in (1, 2){filtro} order by id;"
)


class Logs:
    """Linhas novas de php_errors.log e de cm_log (DbTarget do Yii) desde a ultima leitura."""

    def __init__(self, php_log: Path):
        self.php_log = php_log
        # id da sessao PHP desta execucao: o prefixo do cm_log e [ip][userId][sessionId], o que permite
        # varias execucoes (agentes) em paralelo sem misturar os erros
        self.sessao = None
        self.pos = php_log.stat().st_size if php_log.exists() else 0
        self.ultimo_id = int(_mysql("select coalesce(max(id), 0) from cm_log;")[0][0])

    def novos(self) -> list:
        linhas = []
        if self.php_log.exists():
            with open(self.php_log, "rb") as f:
                f.seek(self.pos)
                dados = f.read()
                self.pos = f.tell()
            linhas += [re.sub(r"^\[[^\]]+\]\s*", "", l)
                       for l in dados.decode("utf8", "replace").splitlines() if l.strip()]
        filtro = f" and prefix like '%[{self.sessao}]%'" if self.sessao else ""
        for id_, nivel, cat, msg in _mysql(_SQL_CM_LOG.format(ultimo=self.ultimo_id, filtro=filtro)):
            self.ultimo_id = max(self.ultimo_id, int(id_))
            linhas.append(f"cm_log[{nivel}] {cat}: {msg[:400]}")
        return linhas


# ---------- crawl ----------

def _login(page, base_url):
    # a primeira requisicao publica os assets do vendor em web/assets (lento no bind mount do Windows)
    for tentativa in range(3):
        try:
            page.goto(base_url + "/site/login", wait_until="load", timeout=180000)
            break
        except Exception:
            if tentativa == 2:
                raise
    # campo com MaskedInput (999.999.999-99, removeMaskOnSubmit): digitar como o usuario, para a mascara agir
    # a tela tem dois formularios (desktop e mobile): usa o visivel
    form = page.locator("form").filter(has=page.locator("input[name='LoginForm[username]']:visible")).first
    campo = form.locator("input[name='LoginForm[username]']")
    campo.click()
    campo.press_sequentially(re.sub(r"\D", "", LOGIN_USUARIO), delay=30)
    form.locator("input[name='LoginForm[password]']").fill(LOGIN_SENHA)
    with page.expect_navigation(timeout=60000):
        form.locator("button[type=submit], input[type=submit]").first.click()
    if "/site/login" in page.url:
        raise SystemExit("login falhou em " + base_url)


def _visitar(context, page, base_url, url, logs, capturar, dir_capturas):
    reg = {"url": url, "status": None, "ms": 0, "erro_yii": False, "html_norm_sha": "", "html_norm_len": 0,
           "console_erros": [], "log_novos": [], "captura": None, "final": None, "links": []}
    erros_js = []

    def ouvinte(m):
        if m.type == "error":
            erros_js.append(m.text[:300])

    def ouvinte_exc(e):
        erros_js.append(("pageerror: " + str(e))[:300])

    page.on("console", ouvinte)
    page.on("pageerror", ouvinte_exc)
    inicio = time.time()
    html = ""
    try:
        resp = page.goto(base_url + url, wait_until="load", timeout=90000)
        reg["status"] = resp.status if resp else None
        html = page.content()
        reg["final"] = urlsplit(page.url).path
        reg["links"] = page.eval_on_selector_all("a[href]", "els => els.map(e => e.getAttribute('href'))")
        if capturar:
            nome = re.sub(r"[^\w.-]+", "_", url.strip("/"))[:120] or "raiz"
            destino = dir_capturas / f"{nome}.jpg"
            page.screenshot(path=str(destino), full_page=True, type="jpeg", quality=70, timeout=60000)
            reg["captura"] = destino.name
    except Exception as e:  # downloads, timeouts
        texto = str(e)
        if "Download is starting" in texto or "net::ERR_ABORTED" in texto:
            r = context.request.get(base_url + url, timeout=90000)
            reg["status"] = r.status
            html = hashlib.sha256(r.body()).hexdigest()
            reg["final"] = "download:" + r.headers.get("content-type", "")
        else:
            reg["status"] = -1
            reg["console_erros"].append("excecao do navegador: " + texto.splitlines()[0][:300])
    reg["ms"] = int((time.time() - inicio) * 1000)
    page.remove_listener("console", ouvinte)
    page.remove_listener("pageerror", ouvinte_exc)
    norm = normalizar_html(html)
    reg["erro_yii"] = detecta_erro_yii(html)
    reg["html_norm_sha"] = hashlib.sha256(norm.encode("utf8")).hexdigest()
    reg["html_norm_len"] = len(norm)
    reg["console_erros"] += erros_js
    reg["log_novos"] = logs.novos()
    return reg


def _salvar(saida: Path, rotulo: str, registros: list):
    saida.write_text(json.dumps({"rotulo": rotulo, "registros": registros}, ensure_ascii=False), encoding="utf8")


def rodar(rotulo: str, inventario: Path, repetir: Path = None, profundidade: int = 3, limite: int = 4000,
          so_urls: list = None, sufixo: str = "", sufixo_json: str = None):
    from playwright.sync_api import sync_playwright

    porta, _container, _raiz, php_log = AMBIENTES[rotulo]
    base_url = f"http://localhost:{porta}"
    dir_capturas = SAIDA / "capturas" / (rotulo + sufixo)
    dir_capturas.mkdir(parents=True, exist_ok=True)
    saida = SAIDA / f"crawl-{rotulo}{sufixo if sufixo_json is None else sufixo_json}.json"

    bfs = False
    if so_urls:
        fila = deque((u, 0) for u in so_urls)
    elif repetir:
        fila = deque((r["url"], 0) for r in json.loads(Path(repetir).read_text(encoding="utf8"))["registros"])
    else:
        bfs = True
        urls = json.loads(Path(inventario).read_text(encoding="utf8"))["urls"]
        fila = deque((u, 0) for u in filtrar(["/", "/site/index"] + urls))
    vistos = {u for u, _ in fila}
    controllers_capturados, registros = set(), []
    logs = Logs(php_log)

    with sync_playwright() as p:
        nav = p.chromium.launch()
        context = nav.new_context(viewport={"width": 1366, "height": 900})
        page = context.new_page()
        page.on("dialog", lambda d: d.dismiss())
        _login(page, base_url)
        logs.sessao = next((c["value"] for c in context.cookies() if c["name"] == "PHPSESSID"), None)
        logs.novos()  # descarta o que o login gerou
        while fila and len(registros) < limite:
            url, prof = fila.popleft()
            ctrl = "/".join(urlsplit(url).path.strip("/").split("/")[:-1]) or url
            capturar = ctrl not in controllers_capturados
            reg = _visitar(context, page, base_url, url, logs, capturar, dir_capturas)
            if reg["captura"]:
                controllers_capturados.add(ctrl)
            if reg["final"] and reg["final"].startswith("/site/login") and not url.startswith("/site/login"):
                reg["relogin"] = True
                _login(page, base_url)
                logs.novos()
            if bfs and prof < profundidade:
                for href in reg["links"]:
                    link = normalizar_link(href, base_url)
                    if link and link not in vistos and filtrar([link]):
                        vistos.add(link)
                        fila.append((link, prof + 1))
            del reg["links"]
            registros.append(reg)
            if len(registros) % 25 == 0:
                _salvar(saida, rotulo, registros)
                print(f"{len(registros)} visitadas, fila={len(fila)}", flush=True)
        nav.close()
    _salvar(saida, rotulo, registros)
    print(f"fim: {len(registros)} URLs -> {saida}")
    return registros


def rodar_em_fatias(rotulo: str, repetir: Path, fatias: int):
    """Divide a lista de --repetir em N processos (cada um com sua sessao PHP) e junta na ordem original."""
    import subprocess
    import sys
    urls = [r["url"] for r in json.loads(Path(repetir).read_text(encoding="utf8"))["registros"]]
    procs = [subprocess.Popen([sys.executable, __file__, "--rotulo", rotulo, "--repetir", str(repetir),
                               "--fatias", str(fatias), "--fatia", str(i)]) for i in range(fatias)]
    for pr in procs:
        pr.wait()
    por_url = {}
    for i in range(fatias):
        arq = SAIDA / f"crawl-{rotulo}-f{i}.json"
        for r in json.loads(arq.read_text(encoding="utf8"))["registros"]:
            por_url[r["url"]] = r
        arq.unlink()
    registros = [por_url[u] for u in urls if u in por_url]
    _salvar(SAIDA / f"crawl-{rotulo}.json", rotulo, registros)
    print(f"fim: {len(registros)}/{len(urls)} URLs em {fatias} fatias -> crawl-{rotulo}.json")


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--rotulo", required=True, choices=list(AMBIENTES))
    ap.add_argument("--inventario", default=str(SAIDA / "inventario.json"))
    ap.add_argument("--repetir")
    ap.add_argument("--profundidade", type=int, default=3)
    ap.add_argument("--limite", type=int, default=4000)
    ap.add_argument("--fatias", type=int, default=1, help="processos paralelos (so com --repetir)")
    ap.add_argument("--fatia", type=int, help=argparse.SUPPRESS)
    a = ap.parse_args()
    if a.fatias > 1 and a.fatia is None:
        rodar_em_fatias(a.rotulo, Path(a.repetir), a.fatias)
    elif a.fatia is not None:
        todas = [r["url"] for r in json.loads(Path(a.repetir).read_text(encoding="utf8"))["registros"]]
        minhas = [u for j, u in enumerate(todas) if j % a.fatias == a.fatia]
        rodar(a.rotulo, None, so_urls=minhas, sufixo_json=f"-f{a.fatia}")
    else:
        rodar(a.rotulo, Path(a.inventario), Path(a.repetir) if a.repetir else None, a.profundidade, a.limite)