#!/usr/bin/env python3
"""The parser and the dead-rule detector, to the definitions sealed in rule.py.

Nothing in this file decides anything the pre-registration did not already fix.
It is imported by `controls.py` (which must pass first) and by `analyse.py`.

Parsing is RFC 9309. A robots.txt is a sequence of GROUPS: one or more
`User-agent:` lines followed by rule lines. Inside the group that applies to a
crawler, §2.2.2 decides a URL by the MOST SPECIFIC match — longest path pattern
wins — and a tie is won by `Allow`.

The three dead classes, and the one thing to keep in mind about all of them:
every comparison is SKIPPED when either pattern carries `*` or `$`, because the
containment question stops being decidable by prefix and I would be guessing.
Skipping can only lose dead rules, never invent them, so every count here is a
LOWER BOUND.
"""


def _norm(path):
    """A rule path as the matcher sees it. Empty stays empty (it is a no-op)."""
    path = path.strip()
    if path and not path.startswith("/") and not path.startswith("*"):
        path = "/" + path
    return path


def has_wildcard(path):
    return "*" in path or "$" in path


class Rule:
    __slots__ = ("allow", "path", "line", "raw")

    def __init__(self, allow, path, line, raw):
        self.allow, self.path, self.line, self.raw = allow, path, line, raw

    def __repr__(self):
        return f"{'Allow' if self.allow else 'Disallow'}:{self.path}@{self.line}"


class Group:
    def __init__(self):
        self.agents = []
        self.rules = []


def parse(text):
    """-> (groups, other_lines). Comments and blank lines are dropped.

    A `User-agent:` line after a rule line starts a NEW group; consecutive
    `User-agent:` lines accumulate onto the same one (RFC 9309 §2.2.1).
    """
    groups, other = [], 0
    group, expecting_agent = None, True
    for number, raw in enumerate(text.splitlines(), 1):
        line = raw.split("#", 1)[0].strip()
        if not line:
            continue
        field, sep, value = line.partition(":")
        if not sep:
            continue
        field, value = field.strip().lower(), value.strip()
        if field == "user-agent":
            if group is None or not expecting_agent:
                group = Group()
                groups.append(group)
                expecting_agent = True
            group.agents.append(value.lower())
        elif field in ("allow", "disallow"):
            if group is None:          # rules before any User-agent: line
                continue               # no group applies to them; RFC ignores
            expecting_agent = False
            group.rules.append(Rule(field == "allow", _norm(value), number, line))
        else:
            other += 1
    return groups, other


def covers(outer, inner):
    """Does every path matched by `inner` also get matched by `outer`?

    Prefix semantics: `Disallow: /a` matches every path beginning `/a`. So
    `outer` covers `inner` exactly when outer's pattern is a prefix of inner's.
    Callers must have excluded wildcards before asking.
    """
    return inner.startswith(outer)


def overlaps(a, b):
    """Could any single path be matched by both patterns?

    With prefix semantics, two prefixes overlap iff one is a prefix of the
    other. Wildcards excluded by the caller.
    """
    return a.startswith(b) or b.startswith(a)


def dead_rules(group):
    """The dead rules of one group, by the three sealed classes.

    Returns a list of (rule, class_name). A rule is reported at most once, in
    the order D3, D1, D2 — the class names are a description of why, and
    double-counting a line would inflate the numerator of the kill rule.
    """
    found = {}
    rules = group.rules
    allows = [r for r in rules if r.allow and r.path]
    disallows = [r for r in rules if not r.allow and r.path]

    # -- D3, losing duplicate ------------------------------------------------
    # (a) the same path as both Allow and Disallow: the tie goes to Allow, so
    #     the Disallow can never win.
    allow_paths = {r.path for r in allows if not has_wildcard(r.path)}
    for rule in disallows:
        if not has_wildcard(rule.path) and rule.path in allow_paths:
            found.setdefault(rule.line, (rule, "D3-contradiction"))
    # (b) a rule line repeated verbatim in one group: every one after the
    #     first decides nothing. An empty path counts here — a repeated bare
    #     `Disallow:` is still a repeat — so this walks `rules`, not the
    #     non-empty lists.
    seen = set()
    for rule in rules:
        key = (rule.allow, rule.path)
        if key in seen:
            found.setdefault(rule.line, (rule, "D3-duplicate"))
        seen.add(key)

    # -- D1, redundant Allow -------------------------------------------------
    # An Allow permits nothing new unless some Disallow in the same group
    # could have matched a path it matches. Everything is permitted by
    # default, so with no such Disallow the line does nothing at all.
    for rule in allows:
        if rule.line in found or has_wildcard(rule.path):
            continue
        contested = any(
            not has_wildcard(other.path) and overlaps(rule.path, other.path)
            for other in disallows)
        if not contested:
            # A group whose Disallows are all wildcards is unjudgeable rather
            # than clean: skip instead of calling the Allow dead.
            if any(has_wildcard(other.path) for other in disallows):
                continue
            found[rule.line] = (rule, "D1-redundant-allow")

    # -- D2, shadowed Disallow -----------------------------------------------
    # A longer Disallow under a shorter one has the same verdict over a subset
    # of paths, so it only does work if it out-specifies an Allow. With no
    # Allow overlapping it at all, it is dead.
    for rule in disallows:
        if rule.line in found or has_wildcard(rule.path):
            continue
        shadowed = any(
            other is not rule and not has_wildcard(other.path) and other.path
            and other.path != rule.path and covers(other.path, rule.path)
            for other in disallows)
        if not shadowed:
            continue
        if any(has_wildcard(other.path) for other in allows):
            continue                    # unjudgeable, not clean
        if any(overlaps(rule.path, other.path) for other in allows):
            continue                    # it may be out-specifying that Allow
        found[rule.line] = (rule, "D2-shadowed-disallow")

    return [found[line] for line in sorted(found)]


def score(text):
    """-> dict for one robots.txt: rule count, dead list, per-class counts.

    `opaque` is the number of rules sitting in a group that contains at least
    one wildcard pattern. Those groups are where the detector mostly cannot
    see — every comparison touching a `*` or `$` is skipped — so `opaque`
    bounds how much of the corpus the lower bound is a lower bound *over*.
    Reporting a lower bound without saying how much it could not look at
    would be its own kind of unfalsifiable number.
    """
    groups, _other = parse(text)
    rules = sum(len(g.rules) for g in groups)
    opaque = sum(len(g.rules) for g in groups
                 if any(has_wildcard(r.path) for r in g.rules))
    dead = []
    for group in groups:
        dead.extend(dead_rules(group))
    classes = {}
    for _rule, name in dead:
        classes[name] = classes.get(name, 0) + 1
    return {"groups": len(groups), "rules": rules, "opaque": opaque,
            "dead": len(dead), "classes": classes,
            "lines": [(r.line, r.raw, name) for r, name in dead]}