{
  "report_date": "2026-10-24",
  "report_version": 3,
  "scope": "Aggregate project research record. Historical response observations used mismatched reference/candidate decoding settings and are not a validated matched greedy comparison or release-quality score.",
  "reference_model": {
    "id": "Qwen/Qwen3-0.6B",
    "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
    "license": "Apache-2.0"
  },
  "model_release_status": {
    "en": "unpublished",
    "tr": "unpublished"
  },
  "public_inference_enabled": false,
  "runs": {
    "en": {
      "run_id": "en-v2-005",
      "parameters": 640090112,
      "steps": 81,
      "selected_step": 81,
      "before_nll": 3.0174983390507157,
      "selected_saved_nll": 2.088429770654156,
      "wall_seconds": 108.78989913600003,
      "gpu": "NVIDIA A100-SXM4-40GB",
      "serialization_gate_passed": true,
      "all_parameters_trainable": true,
      "optimizer_state_saved": false,
      "fp32_masters_saved": false,
      "production_ready": false
    },
    "tr": {
      "run_id": "tr-v2-001",
      "parameters": 640090112,
      "steps": 81,
      "selected_step": 75,
      "before_nll": 3.0076933194620485,
      "selected_saved_nll": 2.410304774051039,
      "wall_seconds": 111.04285879600002,
      "gpu": "NVIDIA A100-SXM4-40GB",
      "serialization_gate_passed": true,
      "all_parameters_trainable": true,
      "optimizer_state_saved": false,
      "fp32_masters_saved": false,
      "production_ready": false
    }
  },
  "data": {
    "en": {
      "training_prefixes": 322,
      "training_conversations": 305,
      "training_families": 205,
      "development_cases": 32,
      "frozen_evaluation_cases": 100,
      "supervised_training_tokens": 10452,
      "supervised_development_tokens": 420,
      "later_turn_training_prefixes": 17,
      "later_turn_supervised_tokens": 495,
      "training_prefix_license_labels": {
        "Apache-2.0": 26,
        "CC-BY-SA-3.0": 73,
        "project-authored synthetic text": 223
      },
      "mask_and_eos_audit_passed": true
    },
    "tr": {
      "training_prefixes": 322,
      "training_conversations": 305,
      "training_families": 205,
      "development_cases": 32,
      "frozen_evaluation_cases": 100,
      "supervised_training_tokens": 14074,
      "supervised_development_tokens": 585,
      "later_turn_training_prefixes": 17,
      "later_turn_supervised_tokens": 806,
      "training_prefix_license_labels": {
        "Apache-2.0": 26,
        "CC-BY-SA-3.0": 73,
        "project-authored synthetic text": 223
      },
      "mask_and_eos_audit_passed": true
    }
  },
  "english_response_comparison": {
    "cases_per_model": 100,
    "turns_per_model": 120,
    "rubric_version": "user-priorities-v2-20261024",
    "review": "Assistant whole-response review; transparent post-hoc user-requested reassessment.",
    "independent_human_review": false,
    "coding_is_release_gate": false,
    "models": {
      "reference": {
        "primary_factual_accuracy_among_direct_answers": {
          "passed": 21,
          "total": 34,
          "fraction": 0.6176470588235294
        },
        "answerable_factual_coverage": {
          "passed": 34,
          "total": 34,
          "fraction": 1.0
        },
        "appropriate_uncertainty": {
          "passed": 17,
          "total": 20,
          "fraction": 0.85
        },
        "ambiguity_clarification": {
          "passed": 2,
          "total": 10,
          "fraction": 0.2
        },
        "friendly_adequate": {
          "passed": 9,
          "total": 10,
          "fraction": 0.9
        },
        "multi_turn_all_cases": {
          "passed": 15,
          "total": 20,
          "fraction": 0.75
        },
        "multi_turn_excluding_exact_format_diagnostics": {
          "passed": 14,
          "total": 18,
          "fraction": 0.7777777777777778
        },
        "coding_diagnostic_correct": {
          "passed": 2,
          "total": 3,
          "fraction": 0.6666666666666666
        },
        "generated_turns": 120,
        "hit_token_cap_turns": 2,
        "critical_fabrication_cases": 1
      },
      "candidate": {
        "primary_factual_accuracy_among_direct_answers": {
          "passed": 21,
          "total": 33,
          "fraction": 0.6363636363636364
        },
        "answerable_factual_coverage": {
          "passed": 33,
          "total": 34,
          "fraction": 0.9705882352941176
        },
        "appropriate_uncertainty": {
          "passed": 18,
          "total": 20,
          "fraction": 0.9
        },
        "ambiguity_clarification": {
          "passed": 1,
          "total": 10,
          "fraction": 0.1
        },
        "friendly_adequate": {
          "passed": 8,
          "total": 10,
          "fraction": 0.8
        },
        "multi_turn_all_cases": {
          "passed": 11,
          "total": 20,
          "fraction": 0.55
        },
        "multi_turn_excluding_exact_format_diagnostics": {
          "passed": 11,
          "total": 18,
          "fraction": 0.6111111111111112
        },
        "coding_diagnostic_correct": {
          "passed": 1,
          "total": 3,
          "fraction": 0.3333333333333333
        },
        "generated_turns": 120,
        "hit_token_cap_turns": 2,
        "critical_fabrication_cases": 2
      }
    },
    "limitations": [
      "Reference sampling was restored by model-specific defaults while candidate defaults are inferred as greedy; this was not a matched greedy comparison.",
      "Counts preserve a historical assistant review of a small authored test and do not establish controlled model quality or causal training effects.",
      "The original review notes are preserved as historical notes, not current release conclusions."
    ],
    "status": "historical preliminary whole-response review; mismatched reference/candidate decoding",
    "intended_decoding": "matched greedy",
    "reference_do_sample": true,
    "candidate_do_sample_inferred": false,
    "candidate_mode_inference_basis": "Saved candidate configuration; not an explicit per-call mode record.",
    "valid_under_claimed_greedy_protocol": false,
    "valid_for_release_quality_score": false,
    "supports_causal_training_claim": false,
    "original_review_limitations": [
      "Small, authored test; percentages are descriptive, not population guarantees.",
      "Some individual interpretations are subjective and documented; failure conclusion remains under lenient treatment of borderline cases.",
      "No training records or selected checkpoint were changed using these outputs.",
      "Per-turn hit-token-cap and raw responses are retained."
    ]
  },
  "turkish_generation_evaluation": {
    "run_id": "tr-eval-batch8-002",
    "status": "complete response collection with mismatched decoding; invalid as claimed matched greedy comparison",
    "intended_decoding": "matched greedy",
    "reference_do_sample": true,
    "candidate_do_sample_inferred": false,
    "valid_under_claimed_greedy_protocol": false,
    "valid_for_release_quality_score": false,
    "cases_per_model": 100,
    "turns_per_model": 120,
    "reference_completed_cases": 100,
    "reference_completed_turns": 120,
    "candidate_completed_cases": 100,
    "candidate_completed_turns": 120,
    "batch_size": 8,
    "max_new_tokens_per_turn": 128,
    "job_started_utc": "2026-10-24T14:47:49.541379+00:00",
    "job_stopped_utc": "2026-10-24T14:51:10.012068+00:00",
    "job_wall_seconds": 200.470689,
    "evaluation_stage_wall_seconds": 175.83801754800004,
    "returncode": 0,
    "comparative_quality_score": null,
    "corrected_greedy_comparison_available": false,
    "earlier_attempt": {
      "status": "historical partial reference sampled collection; 450-second limit",
      "reference_completed_cases": 96,
      "reference_completed_turns": 112,
      "reference_generated_tokens": 11385,
      "candidate_completed_cases": 0,
      "comparative_quality_score": null,
      "reference_do_sample": true,
      "valid_under_claimed_greedy_protocol": false
    }
  },
  "evidence_digests": {
    "english_run_manifest": {
      "source_filename": "run-manifest.json",
      "sha256": "1c5366d311e51160445ed0d333db07d4712b4acdc6e7421f8d5c459c1d05ecce"
    },
    "turkish_run_manifest": {
      "source_filename": "run-manifest.json",
      "sha256": "d3eac30b656d8c7338e4c48c957f0d565068e774665dceb98e3abd531ca222e8"
    },
    "english_evaluation_manifest": {
      "source_filename": "evaluation-manifest.json",
      "sha256": "2df4709fd28842d357310edddc972ca1177f7d06f9de343da2113e62ec076323"
    },
    "english_scores": {
      "source_filename": "EN_USER_PRIORITIES_V2_SCORES.json",
      "sha256": "25dfd135e5f36a7104186dc7c70662e5afceefd8b69a95c311821861b0c9a5ef"
    },
    "completion_objective_audit": {
      "source_filename": "TRAINING_OBJECTIVE_AUDIT.json",
      "sha256": "74ed71b350d926f3714e96d7b70fe093187655120f02622ad612c74389c9e212"
    },
    "data_manifest": {
      "source_filename": "manifest.json",
      "sha256": "206ada01cf664602528cbe4ad09bb7b83715117a1c728cdddb5be1eb6938a411"
    },
    "trainer_source": {
      "source_filename": "train_language.py",
      "sha256": "b9a99f5c65030c1f81dae4ba159af5061e5b214e503809d3392631ab6dfba8dd"
    },
    "evaluator_source": {
      "source_filename": "evaluate_language.py",
      "sha256": "ceaac86a029d7630a632f22b6858179f2f68b34f156d6e213fedbb33ee93631d"
    },
    "architecture_source": {
      "source_filename": "fork_model.py",
      "sha256": "7f4588982294b5f5dd01b4d902423948df07c0c6e611b83a5191ee18eddb4196"
    },
    "turkish_complete_evaluation_manifest": {
      "source_filename": "evaluation-manifest.json",
      "sha256": "78f95c754aa8a0e2c028a0fb8e7a8e72c8e280b4a53b508123f1e7a0ce207737"
    },
    "turkish_complete_job_status": {
      "source_filename": "evaluation-status.json",
      "sha256": "3bf0429177f739d577ac7e9d2a406494c3d9a4c3b86378748afce408bfa70421"
    },
    "turkish_generation_warning_log": {
      "source_filename": "evaluation.log",
      "sha256": "d38d234022099f6a8bdb2e51aa1203aa3b59cbd3400d553a6bda0e192bd31cd1"
    },
    "generation_mode_contract_check": {
      "source_filename": "EFFECTIVE_GENERATION_MODE_CHECK.json",
      "sha256": "ff81e83f0305d7653071d7841e23c8013a9c320b16e7d900f5429ae1e7ceba04"
    },
    "en_candidate_generation_configuration": {
      "source_filename": "generation_config.json",
      "sha256": "bb52bfdd308deaea4ec800bf0165e75770b0a4e5c105963bee1b0398f4043d3e"
    },
    "tr_candidate_generation_configuration": {
      "source_filename": "generation_config.json",
      "sha256": "bb52bfdd308deaea4ec800bf0165e75770b0a4e5c105963bee1b0398f4043d3e"
    },
    "turkish_candidate_scores": {
      "source_filename": "TR_USER_PRIORITIES_V2_SCORES.json",
      "sha256": "d16255b94b9ffdfb5cb59e5bf554e0f8e57c8888231ee5c3d17d8da1e36305ad"
    },
    "three_way_turkish_development_control": {
      "source_filename": "TR_THREE_WAY_DEVELOPMENT_CONTROL.json",
      "sha256": "2318ee5f0ebd7204d7c45ed27e545bd8024bd4f03093838997cc773fbf31ff0e"
    }
  },
  "protocol_correction": {
    "date": "2026-10-24",
    "supersedes": "Version 1 description of earlier response runs as greedy decoding.",
    "library": "Transformers 4.57.1",
    "cause": "Supplied GenerationConfig.do_sample=False matched the global default and was replaced by the reference model-specific do_sample=True when use_model_defaults was omitted. Candidate saved generation configuration omits do_sample and implies the global False default.",
    "reference_decoding": {
      "do_sample": true,
      "basis": "Runtime warning confirms restored reference sampling defaults."
    },
    "candidate_decoding": {
      "do_sample_inferred": false,
      "basis": "Saved candidate generation_config.json has _from_model_config=true and no do_sample field; no explicit per-call mode record was saved."
    },
    "matched_decoding_settings": false,
    "affected_records": [
      "English sequential response review",
      "Turkish sequential partial collection",
      "Turkish batched complete collection"
    ],
    "training_steps_nll_serialization_affected": false,
    "corrected_explicit_generation_arguments": {
      "use_model_defaults": false,
      "do_sample": false,
      "num_beams": 1
    },
    "contract_check": {
      "transformers": "4.57.1",
      "negative_control_restores_sampling": true,
      "fixed_effective_do_sample": false,
      "fixed_num_beams": 1,
      "repeat_equal_across_seeds": true,
      "fix": "use_model_defaults=False plus direct do_sample=False,num_beams=1",
      "scope": "Tiny CPU contract test, not a language-quality benchmark"
    },
    "contract_check_is_language_quality_benchmark": false,
    "completed_corrected_greedy_comparison_available": false,
    "corrected_greedy_frozen_cases": {
      "en_reference": 0,
      "en_candidate": 0,
      "tr_reference": 0,
      "tr_candidate": 0
    },
    "official_api_source": "https://huggingface.co/docs/transformers/v4.57.1/en/main_classes/text_generation"
  },
  "revision_history": [
    {
      "version": 1,
      "date": "2026-10-24",
      "record": "Reported intended greedy protocol and first partial Turkish collection; protocol label superseded."
    },
    {
      "version": 2,
      "date": "2026-10-24",
      "record": "Corrected reference sampling and inferred candidate defaults; preserved preliminary English review counts and earlier Turkish attempt; added complete Turkish coverage with mismatched settings and tiny CPU decoding contract check."
    },
    {
      "version": 3,
      "date": "2026-10-24",
      "record": "Added descriptive whole-response Turkish candidate review, with accuracy and answer coverage separate. Historical reference decoding mismatch remains unresolved; no release readiness or causal training claim."
    }
  ],
  "turkish_candidate_observations": {
    "reviewed_utc": "2026-10-24T15:49:07.222817+00:00",
    "cases": 100,
    "turns": 120,
    "rubric_version": "user-priorities-v2-20261024",
    "review": "Assistant whole-response review; not independent native-speaker assessment",
    "candidate_only": true,
    "matched_reference_comparison_complete": false,
    "coding_is_release_gate": false,
    "production_ready": false,
    "metrics": {
      "primary_factual_accuracy_among_direct_answers": {
        "passed": 8,
        "total": 25,
        "fraction": 0.32
      },
      "answerable_factual_coverage": {
        "passed": 25,
        "total": 34,
        "fraction": 0.7352941176470589
      },
      "appropriate_uncertainty": {
        "passed": 1,
        "total": 20,
        "fraction": 0.05
      },
      "ambiguity_clarification": {
        "passed": 1,
        "total": 10,
        "fraction": 0.1
      },
      "friendly_adequate": {
        "passed": 2,
        "total": 10,
        "fraction": 0.2
      },
      "multi_turn_all_cases": {
        "passed": 3,
        "total": 20,
        "fraction": 0.15
      },
      "multi_turn_excluding_exact_format_diagnostics": {
        "passed": 3,
        "total": 18,
        "fraction": 0.16666666666666666
      },
      "coding_diagnostic_correct": {
        "passed": 3,
        "total": 3,
        "fraction": 1.0
      },
      "generated_turns": 120,
      "hit_token_cap_turns": 19
    },
    "unresponsive_primary_factual_cases": 9,
    "limitations": [
      "Small authored test; descriptive percentages, no population guarantee",
      "Nine unresponsive factual cases excluded from direct-answer accuracy but retained as missing coverage; this distinction is explicit",
      "Lenient credit for imprecise roof-event wording, factual summary with extra sentence breaks, generic clarification and weak but respectful friendly replies",
      "Prior reference inherited sampling; do not draw matched model-improvement conclusions",
      "No data, weights or checkpoint selection changed from these observed holdout outputs"
    ]
  },
  "local_development_controls": {
    "three_way_turkish": {
      "cases": 4,
      "generated_turns_per_role": 5,
      "precision": "CPU FP32",
      "attention": "SDPA",
      "thinking": false,
      "decoding": "explicit greedy",
      "max_new_tokens_per_turn": 32,
      "first_turn_logit_max_abs_differences": [
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "untrained_migration_generations_identical_to_reference": true,
      "vocabulary_and_chat_template_equal": true,
      "selected_checkpoint_load_missing_unexpected_mismatched_keys": 0,
      "saved_tensor_edge_samples_exact": 310,
      "wall_seconds": 65.24,
      "peak_process_rss_mib": 5685.34,
      "scope": "Small observed development control, not a held-out quality or browser benchmark"
    },
    "sampling_ablation": {
      "same_development_cases": 4,
      "seed_range": [
        43,
        46
      ],
      "temperature": 0.7,
      "top_p": 0.8,
      "top_k": 20,
      "max_new_tokens": 32,
      "result": "Did not resolve basic errors in these observed prefixes; several responses truncate. Not release evidence.",
      "official_profile": "https://huggingface.co/Qwen/Qwen3-0.6B#best-practices"
    }
  }
}