{
  "completed_at": "2026-10-10T07:21:42.580151+00:00",
  "papers": 20,
  "questions_per_condition": 200,
  "primary_qa_slots": 5600,
  "scores": {
    "no_context": {
      "correct": 86,
      "total": 200,
      "accuracy": 0.43,
      "invalid": 3,
      "confidence_interval_95": [
        0.38,
        0.48
      ]
    },
    "original": {
      "correct": 182,
      "total": 200,
      "accuracy": 0.91,
      "invalid": 0,
      "confidence_interval_95": [
        0.875,
        0.945
      ]
    },
    "qwen_ku1000": {
      "correct": 176,
      "total": 200,
      "accuracy": 0.88,
      "invalid": 0,
      "confidence_interval_95": [
        0.845,
        0.915
      ]
    },
    "qwen_ku500": {
      "correct": 177,
      "total": 200,
      "accuracy": 0.885,
      "invalid": 0,
      "confidence_interval_95": [
        0.8550000000000001,
        0.915
      ]
    },
    "qwen_summary": {
      "correct": 162,
      "total": 200,
      "accuracy": 0.81,
      "invalid": 0,
      "confidence_interval_95": [
        0.755,
        0.865
      ]
    },
    "gemma4_base_ku1000": {
      "correct": 133,
      "total": 200,
      "accuracy": 0.665,
      "invalid": 0,
      "confidence_interval_95": [
        0.6,
        0.735
      ]
    },
    "gemma4_r128_ku1000": {
      "correct": 159,
      "total": 200,
      "accuracy": 0.795,
      "invalid": 0,
      "confidence_interval_95": [
        0.73,
        0.8550000000000001
      ]
    },
    "gemma4_base_summary": {
      "correct": 128,
      "total": 200,
      "accuracy": 0.64,
      "invalid": 20,
      "confidence_interval_95": [
        0.515,
        0.75
      ]
    },
    "gemma4_r128_summary": {
      "correct": 136,
      "total": 200,
      "accuracy": 0.68,
      "invalid": 20,
      "confidence_interval_95": [
        0.545,
        0.8
      ]
    },
    "gemma12_base_ku1000": {
      "correct": 129,
      "total": 200,
      "accuracy": 0.645,
      "invalid": 10,
      "confidence_interval_95": [
        0.55,
        0.73
      ]
    },
    "gemma12_r128_ku1000": {
      "correct": 169,
      "total": 200,
      "accuracy": 0.845,
      "invalid": 0,
      "confidence_interval_95": [
        0.8,
        0.885
      ]
    },
    "gemma12_base_summary": {
      "correct": 147,
      "total": 200,
      "accuracy": 0.735,
      "invalid": 0,
      "confidence_interval_95": [
        0.67,
        0.795
      ]
    },
    "gemma12_r128_summary": {
      "correct": 161,
      "total": 200,
      "accuracy": 0.805,
      "invalid": 0,
      "confidence_interval_95": [
        0.74,
        0.865
      ]
    },
    "gemma12_previous_summary_lora": {
      "correct": 178,
      "total": 200,
      "accuracy": 0.89,
      "invalid": 0,
      "confidence_interval_95": [
        0.835,
        0.9349999999999999
      ]
    },
    "gemma4_base_ku500": {
      "correct": 151,
      "total": 200,
      "accuracy": 0.755,
      "invalid": 10,
      "confidence_interval_95": [
        0.655,
        0.835
      ]
    },
    "gemma4_r128_ku500": {
      "correct": 165,
      "total": 200,
      "accuracy": 0.825,
      "invalid": 10,
      "confidence_interval_95": [
        0.7148750000000001,
        0.9099999999999999
      ]
    },
    "gemma12_base_ku500": {
      "correct": 150,
      "total": 200,
      "accuracy": 0.75,
      "invalid": 10,
      "confidence_interval_95": [
        0.64,
        0.8400000000000001
      ]
    },
    "gemma12_r128_ku500": {
      "correct": 178,
      "total": 200,
      "accuracy": 0.89,
      "invalid": 0,
      "confidence_interval_95": [
        0.845,
        0.925
      ]
    },
    "qwen_ku500_bt": {
      "correct": 180,
      "total": 200,
      "accuracy": 0.9,
      "invalid": 0,
      "confidence_interval_95": [
        0.875,
        0.925
      ]
    },
    "gemma4_base_ku500_bt": {
      "correct": 149,
      "total": 200,
      "accuracy": 0.745,
      "invalid": 10,
      "confidence_interval_95": [
        0.645,
        0.825
      ]
    },
    "gemma4_r128_ku500_bt": {
      "correct": 165,
      "total": 200,
      "accuracy": 0.825,
      "invalid": 10,
      "confidence_interval_95": [
        0.7150000000000001,
        0.9
      ]
    },
    "gemma12_base_ku500_bt": {
      "correct": 150,
      "total": 200,
      "accuracy": 0.75,
      "invalid": 10,
      "confidence_interval_95": [
        0.645,
        0.835
      ]
    },
    "gemma12_r128_ku500_bt": {
      "correct": 177,
      "total": 200,
      "accuracy": 0.885,
      "invalid": 0,
      "confidence_interval_95": [
        0.8400000000000001,
        0.9199999999999999
      ]
    },
    "qwen_ku1000_bt": {
      "correct": 175,
      "total": 200,
      "accuracy": 0.875,
      "invalid": 0,
      "confidence_interval_95": [
        0.8400000000000001,
        0.9099999999999999
      ]
    },
    "gemma4_base_ku1000_bt": {
      "correct": 133,
      "total": 200,
      "accuracy": 0.665,
      "invalid": 0,
      "confidence_interval_95": [
        0.5900000000000001,
        0.74
      ]
    },
    "gemma4_r128_ku1000_bt": {
      "correct": 156,
      "total": 200,
      "accuracy": 0.78,
      "invalid": 0,
      "confidence_interval_95": [
        0.71,
        0.845
      ]
    },
    "gemma12_base_ku1000_bt": {
      "correct": 129,
      "total": 200,
      "accuracy": 0.645,
      "invalid": 10,
      "confidence_interval_95": [
        0.55,
        0.73
      ]
    },
    "gemma12_r128_ku1000_bt": {
      "correct": 168,
      "total": 200,
      "accuracy": 0.84,
      "invalid": 0,
      "confidence_interval_95": [
        0.79,
        0.885
      ]
    }
  },
  "paired_backtranslation_minus_raw": {
    "qwen_ku500": {
      "point": 0.015000000000000013,
      "confidence_interval_95": [
        0.0,
        0.030000000000000138
      ]
    },
    "gemma4_base_ku500": {
      "point": -0.010000000000000009,
      "confidence_interval_95": [
        -0.03500000000000003,
        0.015000000000000013
      ]
    },
    "gemma4_r128_ku500": {
      "point": 0.0,
      "confidence_interval_95": [
        -0.015000000000000013,
        0.015000000000000013
      ]
    },
    "gemma12_base_ku500": {
      "point": 0.0,
      "confidence_interval_95": [
        -0.020000000000000018,
        0.020000000000000018
      ]
    },
    "gemma12_r128_ku500": {
      "point": -0.0050000000000000044,
      "confidence_interval_95": [
        -0.015000000000000124,
        0.0
      ]
    },
    "qwen_ku1000": {
      "point": -0.0050000000000000044,
      "confidence_interval_95": [
        -0.02499999999999991,
        0.01000000000000012
      ]
    },
    "gemma4_base_ku1000": {
      "point": 0.0,
      "confidence_interval_95": [
        -0.030000000000000027,
        0.03499999999999992
      ]
    },
    "gemma4_r128_ku1000": {
      "point": -0.015000000000000013,
      "confidence_interval_95": [
        -0.03500000000000003,
        0.0050000000000000044
      ]
    },
    "gemma12_base_ku1000": {
      "point": 0.0,
      "confidence_interval_95": [
        -0.030000000000000138,
        0.030000000000000027
      ]
    },
    "gemma12_r128_ku1000": {
      "point": -0.0050000000000000044,
      "confidence_interval_95": [
        -0.02000000000000013,
        0.01000000000000012
      ]
    }
  },
  "quality": {
    "complete": true,
    "windows": 2517,
    "attempts": 3971,
    "changed_documents": 190,
    "inserted_windows": 1123,
    "rolled_back_documents": 0,
    "guarded_global_critical_failures": 0,
    "rejection_reasons": {
      "source_copy_exceeds_five": 1544,
      "nli_below_fixed_0.9": 1591,
      "number_unit_formula_or_operator_change": 315
    },
    "elapsed_seconds": 295.55666647595353,
    "numeric_guard_is_conservative": true,
    "raw_attempts_with_critical_flags": 315,
    "numeric_attempts": 1119,
    "formula_attempts": 592,
    "raw_numeric_signature_changes": 164,
    "raw_formula_signature_changes": 149
  },
  "metadata": {
    "qwen_ku500": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 2,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 163,
      "units": 163,
      "complete_papers": 20,
      "sentence_minhash_present_units": 163
    },
    "gemma4_base_ku500": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 0,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 162,
      "units": 162,
      "complete_papers": 19,
      "sentence_minhash_present_units": 162
    },
    "gemma4_r128_ku500": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 3,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 162,
      "units": 162,
      "complete_papers": 19,
      "sentence_minhash_present_units": 162
    },
    "gemma12_base_ku500": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 0,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 162,
      "units": 162,
      "complete_papers": 19,
      "sentence_minhash_present_units": 162
    },
    "gemma12_r128_ku500": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 6,
      "explicit_style_attributes_present": 1,
      "context_summary_present_units": 163,
      "units": 163,
      "complete_papers": 20,
      "sentence_minhash_present_units": 163
    },
    "qwen_ku1000": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 2,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 85,
      "units": 85,
      "complete_papers": 20,
      "sentence_minhash_present_units": 85
    },
    "gemma4_base_ku1000": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 0,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 85,
      "units": 85,
      "complete_papers": 20,
      "sentence_minhash_present_units": 85
    },
    "gemma4_r128_ku1000": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 3,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 85,
      "units": 85,
      "complete_papers": 20,
      "sentence_minhash_present_units": 85
    },
    "gemma12_base_ku1000": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 0,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 84,
      "units": 84,
      "complete_papers": 19,
      "sentence_minhash_present_units": 84
    },
    "gemma12_r128_ku1000": {
      "document_slots": 20,
      "exact_document_title_matches": 20,
      "nonempty_document_authors": 0,
      "author_attributes_present": 7,
      "explicit_style_attributes_present": 0,
      "context_summary_present_units": 85,
      "units": 85,
      "complete_papers": 20,
      "sentence_minhash_present_units": 85
    }
  },
  "translation": {
    "rounds": [
      {
        "complete": true,
        "windows": 2517,
        "seconds": 10.732607448007911,
        "forward": {
          "seconds": 5.453174700960517,
          "output_tokens": 92827,
          "input_tokens": 248100,
          "output_tokens_per_second": 17022.561185074366,
          "batch_size": 512,
          "num_beams": 1,
          "actual_largest_batch": 512,
          "inputs": 2517,
          "successful_outputs": 2517,
          "includes_tokenization_and_complete_outputs": true
        },
        "backward": {
          "seconds": 5.245763945975341,
          "output_tokens": 69895,
          "input_tokens": 279079,
          "output_tokens_per_second": 13324.084102874072,
          "batch_size": 512,
          "num_beams": 1,
          "actual_largest_batch": 512,
          "inputs": 2517,
          "successful_outputs": 2517,
          "includes_tokenization_and_complete_outputs": true
        },
        "active_gpus": 1
      },
      {
        "complete": true,
        "windows": 1454,
        "seconds": 6.765679658972658,
        "forward": {
          "seconds": 3.635976883000694,
          "output_tokens": 60874,
          "input_tokens": 152786,
          "output_tokens_per_second": 16742.130645715763,
          "batch_size": 512,
          "num_beams": 1,
          "actual_largest_batch": 512,
          "inputs": 1454,
          "successful_outputs": 1454,
          "includes_tokenization_and_complete_outputs": true
        },
        "backward": {
          "seconds": 3.102226751972921,
          "output_tokens": 45802,
          "input_tokens": 168476,
          "output_tokens_per_second": 14764.233456136413,
          "batch_size": 512,
          "num_beams": 1,
          "actual_largest_batch": 512,
          "inputs": 1454,
          "successful_outputs": 1454,
          "includes_tokenization_and_complete_outputs": true
        },
        "active_gpus": 1
      }
    ],
    "active_translation_seconds": 17.49828710698057,
    "active_translation_gpu_hours": 0.004860635307494603,
    "documents": 200,
    "document_versions_per_active_gpu_hour": 41146.88458350711,
    "setup_semantic_checks_QA_and_idle_excluded": true
  },
  "allocation": {
    "job_id": "2259268",
    "state": "COMPLETED",
    "allocation_seconds": 405,
    "allocated_gpus": 4,
    "reserved_gpu_hours": 0.45,
    "alloc_tres": "billing=288,cpu=288,gres/gpu:gh200=4,gres/gpu=4,node=1"
  },
  "protocol": {
    "papers": 20,
    "questions_per_condition": 200,
    "primary_conditions": 28,
    "primary_qa_slots": 5600,
    "prior_raw_conditions": [
      "no_context",
      "original",
      "qwen_ku1000",
      "qwen_ku500",
      "qwen_summary",
      "gemma4_base_ku1000",
      "gemma4_r128_ku1000",
      "gemma4_base_summary",
      "gemma4_r128_summary",
      "gemma12_base_ku1000",
      "gemma12_r128_ku1000",
      "gemma12_base_summary",
      "gemma12_r128_summary",
      "gemma12_previous_summary_lora",
      "gemma4_base_ku500",
      "gemma4_r128_ku500",
      "gemma12_base_ku500",
      "gemma12_r128_ku500"
    ],
    "ku_conditions": [
      "qwen_ku500",
      "gemma4_base_ku500",
      "gemma4_r128_ku500",
      "gemma12_base_ku500",
      "gemma12_r128_ku500",
      "qwen_ku1000",
      "gemma4_base_ku1000",
      "gemma4_r128_ku1000",
      "gemma12_base_ku1000",
      "gemma12_r128_ku1000"
    ],
    "translation_model": "google/translategemma-4b-it",
    "translation_revision": "10042cb0e6e7fdce748996a71dc3dc432a4e0c89",
    "pivot": "de",
    "maximum_roundtrips": 2,
    "greedy_temperature": 0,
    "translation_batch": 512,
    "targeted_fields": "context_summary and descriptive attribute string values only",
    "protected_fields": "entity names, aliases, IDs, relation predicates/targets, bibliographic and mathematical fields, JSON keys",
    "numeric_formula_guard_version": "1.1",
    "bidirectional_nli_minimum_entailment": 0.9,
    "source_copy_window_maximum_words": 5,
    "selection_minimum_copied_words": 6,
    "no_qa_feedback_to_generation_or_repair": true,
    "all_conditions_freshly_judged": true,
    "source_cohort_sha256": "2cc332c6367de7ae98d4fcbf04a3464545a917c087089ca5cae6756210260445",
    "frozen_questions_sha256": "cf5c0c51f2f590c582cd329d3a7febeeb60b200d15b1e55880074275edf5ff02",
    "original_metrics_sha256": "c73df07005527b1314b9ba750d3196afa74b2f1f15fec581aff716f18044bda9",
    "window_count": 2517,
    "demo_document_ids": [
      "fresh-arxiv-1599059",
      "fresh-arxiv-1599072",
      "fresh-arxiv-1599094",
      "fresh-bethgelab-298331",
      "fresh-bethgelab-298328"
    ],
    "demo_selection": "Fixed cohort indices 0,4,9,12,15 before this repair evaluation; no QA-based selection",
    "context": "Previous ten KUs for naming; no next-source lookahead; current context_summary follows existing schema",
    "failures": "Preserved; any original failed generation keeps ten wrong slots, including after paraphrasing"
  },
  "no_qa_based_output_filtering": true
}