#!/usr/bin/env python3
"""
score.py - score gate.py against the author-audited reference error set.

Reproduces Tables 2 and 3 of the paper.

  python3 score.py                    # both gate versions, both conditions
"""

import json
import os

import gate as G

HERE = os.path.dirname(os.path.abspath(__file__))
FB = os.path.join(HERE, "factbase.json")
GT = os.path.join(HERE, "ground_truth.json")
UNGATED = ["S1_general_ungated.md", "S2_tech_ungated.md",
           "S3_security_ungated.md", "S4_policy_ungated.md"]
GATED = ["S1_general_gated.md", "S2_tech_gated.md",
         "S3_security_gated.md", "S4_policy_gated.md"]


def run(files, ledger, **kw):
    out = {}
    for f in files:
        p = os.path.join(HERE, "briefs", f)
        with open(p) as fh:
            text = fh.read()
        out[f] = (G.check(text, ledger, p, **kw), G.word_count(text))
    return out


def matched(err, findings):
    m = err["matcher"].lower()
    return any(m in f["detail"].lower() or m in f["sentence"].lower()
               for f in findings)


def score(results, truth):
    tp_ids, caught = set(), []
    for e in truth["errors"]:
        fs = results.get(e["file"], ([], 0))[0]
        if matched(e, fs):
            tp_ids.add(e["id"])
            caught.append(e)
    # a finding counts as a true positive if it matches some reference error
    tp = fp = 0
    for f, (fs, _) in results.items():
        for finding in fs:
            hit = any(e["file"] == f and matched(e, [finding])
                      for e in truth["errors"])
            if hit:
                tp += 1
            else:
                fp += 1
    return tp, fp, tp_ids, caught


def table(title, results):
    print("\n%s" % title)
    print("  %-30s %7s %9s %14s" % ("brief", "words", "findings", "per 1k words"))
    tw = tf = 0
    for f, (fs, wc) in results.items():
        tw += wc
        tf += len(fs)
        print("  %-30s %7d %9d %14.1f"
              % (f, wc, len(fs), len(fs) * 1000.0 / wc))
    print("  %-30s %7d %9d %14.2f" % ("TOTAL", tw, tf, tf * 1000.0 / tw))
    return tw, tf


def main():
    ledger = G.Ledger(FB)
    with open(GT) as fh:
        truth = json.load(fh)
    n_err = len(truth["errors"])

    v01 = run(UNGATED, ledger, wordnum=False, closedset=False)
    v02 = run(UNGATED, ledger)
    gtd = run(GATED, ledger)

    uw, uf = table("Ungated briefs, gate v0.2 (Table 2, ungated rows)", v02)
    gw, gf = table("Gated briefs, gate v0.2 (Table 2, gated rows)", gtd)

    print("\nReference error set: %d errors across the four ungated briefs" % n_err)
    print("  %-34s %5s %5s %11s %8s" % ("gate version", "TP", "FP", "precision", "recall"))
    for name, res in (("v0.1 atoms only", v01),
                      ("v0.2 + word numbers + closed sets", v02)):
        tp, fp, ids, _ = score(res, truth)
        prec = tp / float(tp + fp) if (tp + fp) else 0.0
        rec = len(ids) / float(n_err)
        print("  %-34s %5d %5d %10.0f%% %7.0f%%"
              % (name, tp, fp, prec * 100, rec * 100))

    _, _, ids02, _ = score(v02, truth)
    missed = [e for e in truth["errors"] if e["id"] not in ids02]
    print("\n  v0.2 misses (%d):" % len(missed))
    for e in missed:
        print("    %-6s %-22s %s" % (e["id"], e["class"], e["note"]))

    print("\n  errors per 1000 words, by segment (reference set):")
    for f in UNGATED:
        wc = v02[f][1]
        n = sum(1 for e in truth["errors"] if e["file"] == f)
        print("    %-30s %2d errors  %5.1f per 1k" % (f, n, n * 1000.0 / wc))

    print("\n  drift rate ungated -> gated: %.2f -> %.2f findings per 1000 words"
          % (uf * 1000.0 / uw, gf * 1000.0 / gw))


if __name__ == "__main__":
    main()