Human-in-the-loop-grading-research/reports/synthetic-evaluation-report.json at main · chering14/Human-in-the-loop-grading-research · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  "sample": {
    "records": 20,
    "objective_records": 10,
    "subjective_records": 10,
    "all_synthetic": true
  },
  "teacher_reliability": {
    "subjective_qwk": 0.9846
  },
  "model_against_adjudicated": {
    "objective_accuracy": 0.8,
    "objective_accuracy_wilson_95": [
      0.4902,
      0.9433
    ],
    "subjective_mae": 2.3,
    "subjective_rmse": 3.6469,
    "subjective_bias": -2.3,
    "subjective_tolerance_hit_rate": 0.8
  },
  "human_review_routing": {
    "automatic_pass_coverage": 0.3,
    "automatic_pass_error_rate": 0.0,
    "error_capture_rate": 1.0,
    "severe_error_escape_rate": 0.0,
    "review_burden": 0.7,
    "post_review_final_accuracy": 1.0
  },
  "descriptive_time": {
    "baseline_minutes": 40.45,
    "framework_minutes": 16.87,
    "relative_time_difference": 0.583
  },
  "deployment_authorized": false,
  "authorization_note": "Synthetic data are not valid evidence for deployment."
}