{
  "block_threshold": 0.85,
  "datasets": {
    "sms": {
      "categories": {
        "sms": {
          "brier_score": 0.0200386421045614,
          "confusion_matrix": {
            "fn": 58,
            "fp": 37,
            "tn": 4480,
            "tp": 584
          },
          "f1": 0.9247822644497229,
          "false_positive_rate": 0.008191277396502104,
          "flagged_rate": 0.12037216514828455,
          "precision": 0.9404186795491143,
          "recall": 0.9096573208722741,
          "samples": 5159
        }
      },
      "lexical_floor_additional_flags": {
        "ham": 21,
        "spam": 11
      },
      "metrics": {
        "brier_score": 0.0200386421045614,
        "confusion_matrix": {
          "fn": 58,
          "fp": 37,
          "tn": 4480,
          "tp": 584
        },
        "f1": 0.9247822644497229,
        "false_positive_rate": 0.008191277396502104,
        "flagged_rate": 0.12037216514828455,
        "precision": 0.9404186795491143,
        "recall": 0.9096573208722741,
        "samples": 5159
      },
      "modes": {
        "enforce": {
          "artifact_sha256": "2aa2163dd9c43a47afd59c1e7dafd45fbc33c5c6412d0e79f654b7a1ba9182e7",
          "decisions_by_label": {
            "ham": {
              "allow": 4480,
              "review": 37
            },
            "spam": {
              "allow": 58,
              "review": 584
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 8.814,
            "p95": 12.45,
            "p99": 14.827
          },
          "requests": 5159,
          "score_max_delta": 0.0,
          "successful": 5159
        },
        "monitor": {
          "artifact_sha256": "b354bd8cb487a1bcabb0e869b094d337a8a9da399281740206f79ebe561ed3d8",
          "decisions_by_label": {
            "ham": {
              "allow": 4480,
              "review": 37
            },
            "spam": {
              "allow": 58,
              "review": 584
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 7.732,
            "p95": 10.07,
            "p99": 11.866
          },
          "requests": 5159,
          "score_max_delta": 0.0,
          "successful": 5159
        }
      },
      "no_lexical_floor": {
        "brier_score": 0.019548237982726802,
        "confusion_matrix": {
          "fn": 69,
          "fp": 16,
          "tn": 4501,
          "tp": 573
        },
        "f1": 0.9309504467912266,
        "false_positive_rate": 0.0035421740092982067,
        "flagged_rate": 0.11416941267687536,
        "precision": 0.9728353140916808,
        "recall": 0.8925233644859814,
        "samples": 5159
      },
      "rows": 5159,
      "scores_sha256": "1f2d33d1d8ff281ebeea4be5bab22dd7f0a47ac6398277844e00f2bf2859fae3",
      "truncated": 0,
      "wilson_95_sampling_intervals": {
        "false_positive_rate": [
          0.0059487389753379565,
          0.011269617281950446
        ],
        "precision": [
          0.9189545305944106,
          0.9564675341200376
        ],
        "recall": [
          0.8849777450878742,
          0.9294636210029239
        ]
      }
    },
    "synthetic": {
      "categories": {
        "benign_prefix_spam": {
          "brier_score": 0.12249999999999998,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 0,
            "tp": 1
          },
          "f1": 1.0,
          "false_positive_rate": null,
          "flagged_rate": 1.0,
          "precision": 1.0,
          "recall": 1.0,
          "samples": 1
        },
        "legitimate_link": {
          "brier_score": 0.42250000000000004,
          "confusion_matrix": {
            "fn": 0,
            "fp": 4,
            "tn": 0,
            "tp": 0
          },
          "f1": 0.0,
          "false_positive_rate": 1.0,
          "flagged_rate": 1.0,
          "precision": 0.0,
          "recall": null,
          "samples": 4
        },
        "multilingual_spam": {
          "brier_score": 0.3251378003752903,
          "confusion_matrix": {
            "fn": 2,
            "fp": 0,
            "tn": 0,
            "tp": 2
          },
          "f1": 0.6666666666666666,
          "false_positive_rate": null,
          "flagged_rate": 0.5,
          "precision": 1.0,
          "recall": 0.5,
          "samples": 4
        },
        "multilingual_support": {
          "brier_score": 0.011355788178105702,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 4,
            "tp": 0
          },
          "f1": null,
          "false_positive_rate": 0.0,
          "flagged_rate": 0.0,
          "precision": null,
          "recall": null,
          "samples": 4
        },
        "obfuscated_spam": {
          "brier_score": 0.3311400931505197,
          "confusion_matrix": {
            "fn": 1,
            "fp": 0,
            "tn": 0,
            "tp": 2
          },
          "f1": 0.8,
          "false_positive_rate": null,
          "flagged_rate": 0.6666666666666666,
          "precision": 1.0,
          "recall": 0.6666666666666666,
          "samples": 3
        },
        "quoted_spam": {
          "brier_score": 0.00454906093763599,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 2,
            "tp": 0
          },
          "f1": null,
          "false_positive_rate": 0.0,
          "flagged_rate": 0.0,
          "precision": null,
          "recall": null,
          "samples": 2
        },
        "scam": {
          "brier_score": 0.680516160147117,
          "confusion_matrix": {
            "fn": 3,
            "fp": 0,
            "tn": 0,
            "tp": 1
          },
          "f1": 0.4,
          "false_positive_rate": null,
          "flagged_rate": 0.25,
          "precision": 1.0,
          "recall": 0.25,
          "samples": 4
        },
        "short_content": {
          "brier_score": 0.0038896917565243716,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 2,
            "tp": 0
          },
          "f1": null,
          "false_positive_rate": 0.0,
          "flagged_rate": 0.0,
          "precision": null,
          "recall": null,
          "samples": 2
        },
        "transactional_support": {
          "brier_score": 0.00500364069318987,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 4,
            "tp": 0
          },
          "f1": null,
          "false_positive_rate": 0.0,
          "flagged_rate": 0.0,
          "precision": null,
          "recall": null,
          "samples": 4
        },
        "unsolicited_promotion": {
          "brier_score": 0.12249999999999998,
          "confusion_matrix": {
            "fn": 0,
            "fp": 0,
            "tn": 0,
            "tp": 4
          },
          "f1": 1.0,
          "false_positive_rate": null,
          "flagged_rate": 1.0,
          "precision": 1.0,
          "recall": 1.0,
          "samples": 4
        }
      },
      "lexical_floor_additional_flags": {
        "ham": 4,
        "spam": 8
      },
      "metrics": {
        "brier_score": 0.23127660445045903,
        "confusion_matrix": {
          "fn": 6,
          "fp": 4,
          "tn": 12,
          "tp": 10
        },
        "f1": 0.6666666666666666,
        "false_positive_rate": 0.25,
        "flagged_rate": 0.4375,
        "precision": 0.7142857142857143,
        "recall": 0.625,
        "samples": 32
      },
      "modes": {
        "enforce": {
          "artifact_sha256": "46eb6b3447a96db3fcdffd2f4ce01b42f57fd2076d19bae2dca9a9aa791d96b0",
          "decisions_by_label": {
            "ham": {
              "allow": 12,
              "review": 4
            },
            "spam": {
              "allow": 6,
              "review": 10
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 9.201,
            "p95": 10.452,
            "p99": 11.367
          },
          "requests": 32,
          "score_max_delta": 0.0,
          "successful": 32
        },
        "monitor": {
          "artifact_sha256": "97b74a045681bb849b1c369b87171720e0a4be7247ee475a465d2bc4390c3456",
          "decisions_by_label": {
            "ham": {
              "allow": 12,
              "review": 4
            },
            "spam": {
              "allow": 6,
              "review": 10
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 8.121,
            "p95": 9.724,
            "p99": 9.887
          },
          "requests": 32,
          "score_max_delta": 0.0,
          "successful": 32
        }
      },
      "no_lexical_floor": {
        "brier_score": 0.29063336097551007,
        "confusion_matrix": {
          "fn": 14,
          "fp": 0,
          "tn": 16,
          "tp": 2
        },
        "f1": 0.2222222222222222,
        "false_positive_rate": 0.0,
        "flagged_rate": 0.0625,
        "precision": 1.0,
        "recall": 0.125,
        "samples": 32
      },
      "rows": 32,
      "scores_sha256": "1fa8c7c5dbc9d56e8c00516c3b268d28f83d70ce9368e7690f94246b80ee5d3d",
      "truncated": 0,
      "wilson_95_sampling_intervals": {
        "false_positive_rate": [
          0.10182067491213048,
          0.49498316535508774
        ],
        "precision": [
          0.4535091567010681,
          0.8827862135542308
        ],
        "recall": [
          0.38641040545502714,
          0.8151876744113636
        ]
      }
    },
    "youtube": {
      "categories": {
        "comment": {
          "brier_score": 0.1287313173868984,
          "confusion_matrix": {
            "fn": 167,
            "fp": 26,
            "tn": 878,
            "tp": 670
          },
          "f1": 0.8741030658838878,
          "false_positive_rate": 0.028761061946902654,
          "flagged_rate": 0.39977024698449165,
          "precision": 0.9626436781609196,
          "recall": 0.8004778972520908,
          "samples": 1741
        }
      },
      "lexical_floor_additional_flags": {
        "ham": 1,
        "spam": 461
      },
      "metrics": {
        "brier_score": 0.1287313173868984,
        "confusion_matrix": {
          "fn": 167,
          "fp": 26,
          "tn": 878,
          "tp": 670
        },
        "f1": 0.8741030658838878,
        "false_positive_rate": 0.028761061946902654,
        "flagged_rate": 0.39977024698449165,
        "precision": 0.9626436781609196,
        "recall": 0.8004778972520908,
        "samples": 1741
      },
      "modes": {
        "enforce": {
          "artifact_sha256": "fabaade58c5d751139ea212446ddb9e4174db86dd22294973183575557ff602e",
          "decisions_by_label": {
            "ham": {
              "allow": 878,
              "review": 26
            },
            "spam": {
              "allow": 167,
              "review": 670
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 7.913,
            "p95": 9.609,
            "p99": 10.428
          },
          "requests": 1741,
          "score_max_delta": 0.0,
          "successful": 1741
        },
        "monitor": {
          "artifact_sha256": "e337612f41051dc1cfea5c3c9d31163281d7fca1032325a0397950caf83299c0",
          "decisions_by_label": {
            "ham": {
              "allow": 878,
              "review": 26
            },
            "spam": {
              "allow": 167,
              "review": 670
            }
          },
          "failed": 0,
          "latency_ms": {
            "p50": 8.808,
            "p95": 12.924,
            "p99": 18.021
          },
          "requests": 1741,
          "score_max_delta": 0.0,
          "successful": 1741
        }
      },
      "no_lexical_floor": {
        "brier_score": 0.30517222713748715,
        "confusion_matrix": {
          "fn": 628,
          "fp": 25,
          "tn": 879,
          "tp": 209
        },
        "f1": 0.3902894491129785,
        "false_positive_rate": 0.02765486725663717,
        "flagged_rate": 0.1344055140723722,
        "precision": 0.8931623931623932,
        "recall": 0.24970131421744324,
        "samples": 1741
      },
      "rows": 1741,
      "scores_sha256": "135fbc2b66e8a9d8ba378cf91ae5b9a49151fa8ece8e3e37325201fe15d7ffc0",
      "truncated": 3,
      "wilson_95_sampling_intervals": {
        "false_positive_rate": [
          0.01970174698552138,
          0.041808396969826724
        ],
        "precision": [
          0.9458269748000828,
          0.9743814408105146
        ],
        "recall": [
          0.7720580000311515,
          0.8261522745971629
        ]
      }
    }
  },
  "execution": "serial local HTTP; not a load test",
  "model": {
    "feedback": "empty; not read or written",
    "inputs": {
      "sms": "aa28d7a7d5014606a28d99e6fd9444ed6b7f0ccc84590e0ee58421aac18a0928",
      "synthetic": "c1117095789c82cad1f6fae7736a613d493a96edbf1371c6e741a0abf0ac2734",
      "youtube": "a04efe2dd887537c36af3f9784199169ad5129506bd12d701fa278259f5d1b0b"
    },
    "max_input_tokens": 512,
    "model": "mrm8488/bert-tiny-finetuned-sms-spam-detection",
    "model_version": "mrm8488/bert-tiny-finetuned-sms-spam-detection+cal3",
    "platform": "macOS-27.0-arm64-arm-64bit",
    "python": "3.12.12",
    "quantize_requested": true,
    "quantized_linear_modules": 0,
    "revision": "012e2681d92fa3f9be2ea08bd31cfa1db14599d2",
    "source_sha256": {
      "app/calibration.py": "c727bbda84fa172af020525853992c9068892b553078ca78c831e0852e5cd648",
      "app/langdetect.py": "c2619641d0b704e839a9f127fbc209918dbaa22e73bae1aafe2067d2c299f003",
      "app/main.py": "9d8f26b2707fc474738b34595cf46c7a619fe0f1e0792eeae33901b35904879a"
    },
    "torch": "2.13.0"
  },
  "proof_difficulty": 8,
  "protocol_tests": {
    "pass": 46
  },
  "review_threshold": 0.5
}