{
  "report": "VR-2026-06",
  "title": "Measuring a sanctions screening engine through its live API, failures included",
  "publisher": "Verifex Research",
  "selfAdministered": true,
  "intervals": "Wilson score, 95%, in percent; exactLowerPct is the exact one-sided 95% lower bound the gates use",
  "outcomeRule": "UNRESOLVED (screening unavailable, not clearance-eligible, uncertain verdict, truncated or incomplete candidate set) counts against recall and is left out of the false-alert rate. A political-exposure-only answer (risk_level match) is not an alert, as the benchmark's scorer and codebook specify; MATCH-POLICY §1 read literally counts it (cleanNames.literalPolicy)",
  "run": {
    "id": "gated-003-post-pep",
    "date": "2026-10-13",
    "engine": "the engine as deployed on October 13, 2026",
    "caseFileSha256": "110825465fe0d8380e3d9e33702a0d9679ef24c9ea5719158c4a06ff2db44b05",
    "casesInFile": 37543,
    "rawLogSha256": "0ebc5782a42d25ea17f9fa7bb7aac7af6f830e4988caa675a29176b116a9c770",
    "recordsServed": 29002,
    "answered": 28420,
    "httpStatus": {
      "ok": 27363,
      "unavailable503": 1057,
      "quota402": 582
    },
    "cachedAnswers": 546,
    "planLists": 41,
    "listedSources": "OFAC (3,016) and UN (384); the listed-name suite has no EU or UK name",
    "stoppedEarly": "The run stopped after 29,002 of 37,543 cases when the test account's quota ran out; the last 582 were refused (HTTP 402), so 28,420 were answered, 1,057 of them screening unavailable. Suite F (PEP separation), Suite G (123 cases) and the 2,000 hard clean names never ran.",
    "verification": "Re-scored from the raw served log by a separate AI agent with no build context, under the pre-registered rules. It ran inside a Verifex working session and disclosed that the product's operating rules were in its context; it read neither the product code nor earlier reports."
  },
  "gates": [
    {
      "id": "G1",
      "name": "Every listed name, exactly as published, alerts",
      "rule": "0 misses in at least 2,995 cases (recall at least 99.9%, one-sided 95%)",
      "result": "FAIL",
      "detail": "3,050 of 3,053 alerted and none was served clear. 3 came back unresolved, which the rule counts against: exact lower bound 99.746%, below 99.9%."
    },
    {
      "id": "G2",
      "name": "Every must-alert variation alerts",
      "rule": "0 misses in 300 for each must cell, plus cross-list spellings and Cyrillic and Arabic script",
      "result": "FAIL",
      "detail": "All 8 variation cells, cross-list spellings and both scripts fail. Worst: legal-form variants 54.3%, one-letter typos 77.7%."
    },
    {
      "id": "G3",
      "name": "Clean names rarely alert",
      "rule": "false-alert rate upper 95% bound at most 10 per 10,000 easy clean names",
      "result": "FAIL",
      "detail": "213 false alerts in 9,202 resolved clean names: 2.3%, upper bound 264 per 10,000. Every company name ran but only part of the person names, which alert more often, so this understates the full set; inconclusive under G7."
    },
    {
      "id": "G4",
      "name": "Political exposure is kept apart from sanctions",
      "rule": "0 PEP-only critical; 0 PEP-and-sanctioned below critical",
      "result": "NOT_EVALUABLE",
      "detail": "Suite F did not run in this test (quota). In the previous run (October 8) it failed: no response in either run was served as critical, so a person who is both PEP and sanctioned returned high."
    },
    {
      "id": "G5",
      "name": "Same answer under concurrent load",
      "rule": "0 must-band decisions differ between concurrency 1 and 3",
      "result": "NOT_COMPUTABLE",
      "detail": "No concurrency records in this run."
    },
    {
      "id": "G6",
      "name": "The regression suite behaves as before",
      "rule": "Suite H (239 earlier cases) answers as it did",
      "result": "NOT_COMPUTABLE",
      "detail": "Suite H is not in this run's case file, so this gate could not be evaluated."
    },
    {
      "id": "G7",
      "name": "Few enough answers are unresolved to read a result",
      "rule": "at most 10% unresolved per suite, else the suite is inconclusive",
      "result": "INCONCLUSIVE",
      "detail": "Variations 13.5%, scripts 42.4% (mostly Chinese and Korean, which are not screened yet), clean names 32.8%."
    }
  ],
  "cells": [
    {
      "key": "A must-band",
      "suite": "A",
      "band": "must",
      "group": "listed",
      "label": "Listed names, exactly as published",
      "scored": 3053,
      "alert": 3050,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 3,
      "recallPct": 99.9,
      "ci95": [
        99.71,
        99.97
      ],
      "exactLowerPct": 99.746
    },
    {
      "key": "A should-band (S7 weak aliases)",
      "suite": "A",
      "band": "should",
      "group": "listed",
      "label": "Weak aliases the list itself flags as low quality",
      "scored": 347,
      "alert": 324,
      "miss": 0,
      "pepOnly": 1,
      "unresolved": 22,
      "recallPct": 93.37,
      "ci95": [
        90.25,
        95.54
      ],
      "exactLowerPct": 90.74
    },
    {
      "key": "B case_punct_space [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Case, punctuation or spacing changed",
      "scored": 300,
      "alert": 297,
      "miss": 1,
      "pepOnly": 0,
      "unresolved": 2,
      "recallPct": 99,
      "ci95": [
        97.1,
        99.66
      ],
      "exactLowerPct": 97.436
    },
    {
      "key": "B diacritics [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Accents added or removed",
      "scored": 300,
      "alert": 293,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 7,
      "recallPct": 97.67,
      "ci95": [
        95.26,
        98.87
      ],
      "exactLowerPct": 95.662
    },
    {
      "key": "B legal_form [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Company legal form added, dropped or swapped",
      "scored": 300,
      "alert": 163,
      "miss": 46,
      "pepOnly": 0,
      "unresolved": 91,
      "recallPct": 54.33,
      "ci95": [
        48.68,
        59.88
      ],
      "exactLowerPct": 49.42
    },
    {
      "key": "B particle [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Name particle joined or split (al-, bin, von)",
      "scored": 300,
      "alert": 269,
      "miss": 13,
      "pepOnly": 0,
      "unresolved": 18,
      "recallPct": 89.67,
      "ci95": [
        85.71,
        92.62
      ],
      "exactLowerPct": 86.309
    },
    {
      "key": "B token_reorder [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Name parts in a different order",
      "scored": 300,
      "alert": 297,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 3,
      "recallPct": 99,
      "ci95": [
        97.1,
        99.66
      ],
      "exactLowerPct": 97.436
    },
    {
      "key": "B translit_bgn_pcgn [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Official transliteration (BGN/PCGN)",
      "scored": 300,
      "alert": 276,
      "miss": 3,
      "pepOnly": 0,
      "unresolved": 21,
      "recallPct": 92,
      "ci95": [
        88.37,
        94.57
      ],
      "exactLowerPct": 88.934
    },
    {
      "key": "B translit_icao_9303 [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "Passport transliteration (ICAO 9303)",
      "scored": 300,
      "alert": 280,
      "miss": 4,
      "pepOnly": 0,
      "unresolved": 16,
      "recallPct": 93.33,
      "ci95": [
        89.93,
        95.64
      ],
      "exactLowerPct": 90.46
    },
    {
      "key": "B typo_keyboard_1 [must]",
      "suite": "B",
      "band": "must",
      "group": "variant",
      "label": "One-letter typo (names of 8+ letters)",
      "scored": 300,
      "alert": 233,
      "miss": 24,
      "pepOnly": 1,
      "unresolved": 42,
      "recallPct": 77.67,
      "ci95": [
        72.62,
        82.01
      ],
      "exactLowerPct": 73.349
    },
    {
      "key": "B honorific [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "Title or honorific added",
      "scored": 300,
      "alert": 270,
      "miss": 4,
      "pepOnly": 0,
      "unresolved": 26,
      "recallPct": 90,
      "ci95": [
        86.08,
        92.91
      ],
      "exactLowerPct": 86.681
    },
    {
      "key": "B initials [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "Given names reduced to initials",
      "scored": 300,
      "alert": 77,
      "miss": 11,
      "pepOnly": 2,
      "unresolved": 210,
      "recallPct": 25.67,
      "ci95": [
        21.05,
        30.9
      ],
      "exactLowerPct": 21.545
    },
    {
      "key": "B token_drop [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "A middle name dropped",
      "scored": 300,
      "alert": 263,
      "miss": 0,
      "pepOnly": 2,
      "unresolved": 35,
      "recallPct": 87.67,
      "ci95": [
        83.46,
        90.92
      ],
      "exactLowerPct": 84.096
    },
    {
      "key": "B typo_keyboard_2 [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "Two keyboard typos",
      "scored": 300,
      "alert": 204,
      "miss": 54,
      "pepOnly": 0,
      "unresolved": 42,
      "recallPct": 68,
      "ci95": [
        62.52,
        73.02
      ],
      "exactLowerPct": 63.283
    },
    {
      "key": "B typo_phonetic_1 [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "One phonetic misspelling",
      "scored": 300,
      "alert": 251,
      "miss": 28,
      "pepOnly": 0,
      "unresolved": 21,
      "recallPct": 83.67,
      "ci95": [
        79.06,
        87.42
      ],
      "exactLowerPct": 79.743
    },
    {
      "key": "B typo_phonetic_2 [should]",
      "suite": "B",
      "band": "should",
      "group": "variant",
      "label": "Two phonetic misspellings",
      "scored": 300,
      "alert": 228,
      "miss": 34,
      "pepOnly": 0,
      "unresolved": 38,
      "recallPct": 76,
      "ci95": [
        70.86,
        80.48
      ],
      "exactLowerPct": 71.595
    },
    {
      "key": "B acronym [not_expected]",
      "suite": "B",
      "band": "not_expected",
      "group": "variant",
      "label": "An acronym in place of the full company name",
      "scored": 300,
      "alert": 72,
      "miss": 193,
      "pepOnly": 0,
      "unresolved": 35,
      "recallPct": 24,
      "ci95": [
        19.52,
        29.14
      ],
      "exactLowerPct": 19.983
    },
    {
      "key": "B_control (un-perturbed siblings) [must]",
      "suite": "B",
      "band": "must",
      "group": "control",
      "label": "Controls: the same names before any change",
      "scored": 4316,
      "alert": 4238,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 78,
      "recallPct": 98.19,
      "ci95": [
        97.75,
        98.55
      ],
      "exactLowerPct": 97.822
    },
    {
      "key": "C must-band (gated)",
      "suite": "C",
      "band": "must",
      "group": "crosslist",
      "label": "Spelling another official list uses for the same subject",
      "scored": 561,
      "alert": 557,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 4,
      "recallPct": 99.29,
      "ci95": [
        98.18,
        99.72
      ],
      "exactLowerPct": 98.376
    },
    {
      "key": "C should-band",
      "suite": "C",
      "band": "should",
      "group": "crosslist",
      "label": "Other cross-list spellings",
      "scored": 725,
      "alert": 687,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 38,
      "recallPct": 94.76,
      "ci95": [
        92.89,
        96.16
      ],
      "exactLowerPct": 93.186
    },
    {
      "key": "D Cyrillic must+gated",
      "suite": "D",
      "band": "must",
      "group": "script",
      "label": "Original script: Cyrillic",
      "scored": 296,
      "alert": 272,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 24,
      "recallPct": 91.89,
      "ci95": [
        88.22,
        94.49
      ],
      "exactLowerPct": 88.787
    },
    {
      "key": "D Arabic must+gated",
      "suite": "D",
      "band": "must",
      "group": "script",
      "label": "Original script: Arabic",
      "scored": 297,
      "alert": 289,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 8,
      "recallPct": 97.31,
      "ci95": [
        94.78,
        98.63
      ],
      "exactLowerPct": 95.192
    }
  ],
  "scripts": [
    {
      "script": "Cyrillic",
      "gated": true,
      "scored": 296,
      "alert": 272,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 24,
      "recallPct": 91.89,
      "ci95": [
        88.22,
        94.49
      ],
      "exactLowerPct": 88.787
    },
    {
      "script": "Arabic",
      "gated": true,
      "scored": 297,
      "alert": 289,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 8,
      "recallPct": 97.31,
      "ci95": [
        94.78,
        98.63
      ],
      "exactLowerPct": 95.192
    },
    {
      "script": "Armenian",
      "gated": false,
      "scored": 2,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 2,
      "recallPct": 0,
      "ci95": [
        0,
        65.76
      ],
      "exactLowerPct": 0
    },
    {
      "script": "Georgian",
      "gated": false,
      "scored": 1,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 1,
      "recallPct": 0,
      "ci95": [
        0,
        79.35
      ],
      "exactLowerPct": 0
    },
    {
      "script": "Greek",
      "gated": false,
      "scored": 169,
      "alert": 99,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 70,
      "recallPct": 58.58,
      "ci95": [
        51.04,
        65.74
      ],
      "exactLowerPct": 51.975
    },
    {
      "script": "Chinese (Han)",
      "gated": false,
      "scored": 300,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 300,
      "recallPct": 0,
      "ci95": [
        0,
        1.26
      ],
      "exactLowerPct": 0
    },
    {
      "script": "Korean (Hangul)",
      "gated": false,
      "scored": 92,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 92,
      "recallPct": 0,
      "ci95": [
        0,
        4.01
      ],
      "exactLowerPct": 0
    },
    {
      "script": "Hebrew",
      "gated": false,
      "scored": 35,
      "alert": 35,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 0,
      "recallPct": 100,
      "ci95": [
        90.11,
        100
      ],
      "exactLowerPct": 91.797
    },
    {
      "script": "Japanese",
      "gated": false,
      "scored": 8,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 8,
      "recallPct": 0,
      "ci95": [
        0,
        32.44
      ],
      "exactLowerPct": 0
    },
    {
      "script": "Thai",
      "gated": false,
      "scored": 11,
      "alert": 0,
      "miss": 0,
      "pepOnly": 0,
      "unresolved": 11,
      "recallPct": 0,
      "ci95": [
        0,
        25.88
      ],
      "exactLowerPct": 0
    }
  ],
  "cleanNames": {
    "source": "GLEIF company names and ParaNames person names; checked against OFAC, UN, EU and UK only",
    "inCaseFile": 20000,
    "absent": 5718,
    "notScreened": 582,
    "scored": 13700,
    "falseAlerts": 213,
    "clean": 8989,
    "unresolved": 4498,
    "ratePct": 2.315,
    "ci95": [
      2.027,
      2.642
    ],
    "upperPer10k": 264.2,
    "unresolvedSharePct": 32.8,
    "mix": "all 10,000 company names (GLEIF) but only 3,700 scored person names (ParaNames)",
    "companyRatePct": 1.57,
    "personRatePct": 11.99,
    "likeForLike": {
      "before": 197,
      "after": 213,
      "previousRunLabel": "October 8, 2026"
    },
    "literalPolicy": {
      "falseAlerts": 347,
      "resolved": 9267,
      "ratePct": 3.74,
      "pepOnlyMatches": 134
    }
  },
  "regulatorContext": [
    {
      "name": "Finansinspektionen (Sweden)",
      "date": "2024",
      "measure": "share of test names caught by customer screening, averaged across banks",
      "scope": "19 banks, 21 customer-screening systems, tested by an external firm on UN and EU list names",
      "comparable": true,
      "exactPct": 86.3,
      "exactMedianPct": 99.3,
      "alteredPct": 63.9,
      "alteredMedianPct": 73.4,
      "alteredRange": "5.0%–96.1%",
      "cleanFalseAlertPct": 4.9,
      "cleanNames": 200,
      "url": "https://www.fi.se/contentassets/6090223c4c3e4de98aadedc6721afe7c/in-depht-analysis-sanction-screening-2024.pdf"
    },
    {
      "name": "Financial Conduct Authority (UK)",
      "date": "May 2026",
      "measure": "share of alerts raised in testing that identified the right sanctioned party — a different measure from names caught, which this run did not score",
      "scope": "firms in its sanctions screening testing workstream, on UK list names",
      "comparable": false,
      "exactPct": 90,
      "exactMedianPct": null,
      "alteredPct": 75,
      "alteredMedianPct": null,
      "alteredRange": null,
      "cleanFalseAlertPct": null,
      "cleanNames": null,
      "url": "https://www.fca.org.uk/publications/good-and-poor-practice/sanctions-systems-and-controls-our-firms-our-findings"
    }
  ]
}