{
  "schema_version": "1.0",
  "project": "小白看懂蒸馏原理：10个案例讲明白蒸馏是怎么回事",
  "retrieved_at": "2026-08-14T21:30:00+08:00",
  "integrity_note": "SHA-256 对应本次抓取的原始网页、README、模型卡或代码文件。动态网页日后重抓哈希可能变化。original_excerpt 均为来源中的连续逐字原文；中文解释不进入该字段。Prompt 档案中如有删节或仅规范缩进，会单独降级标注，不称字符级完整原文。",
  "sources": [
    {
      "id": "S01-anthropic-attack-disclosure",
      "institution_author": "Anthropic",
      "title": "Detecting and preventing distillation attacks",
      "url": "https://www.anthropic.com/news/detecting-and-preventing-distillation-attacks",
      "published_date": "2026-02-23",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "No explicit republication license on page; Anthropic copyright applies",
      "evidence_type": "Official company disclosure; allegations and an explicitly approximate prompt",
      "original_excerpt": "A prompt like the following (which approximates similar prompts we have seen used repetitively and at scale) may seem benign on its own: You are an expert data analyst combining statistical rigor with deep domain knowledge. Your goal is to deliver data-driven insights — not summaries or visualizations — grounded in real data and supported by complete and transparent reasoning.",
      "sha256": "0eb55977bd537f7b5ea907e54399cb17c97c401e49e4244652ca742806c3b110",
      "snapshot_scope": "HTML, 133031 bytes",
      "redistribution": "Limited quotation with attribution only; do not mirror the whole page",
      "notes": "Anthropic says the displayed prompt only approximates prompts it observed. It is not a leaked verbatim attack prompt. The page reports more than 16 million exchanges through approximately 24,000 fraudulent accounts and separates DeepSeek, Moonshot AI and MiniMax campaign scales. Treat attribution as Anthropic's claim, not independently adjudicated fact."
    },
    {
      "id": "S02-google-attack-disclosure",
      "institution_author": "Google Threat Intelligence Group and Google DeepMind",
      "title": "GTIG AI Threat Tracker: Distillation, Experimentation, and (Continued) Integration of AI for Adversarial Use",
      "url": "https://cloud.google.com/blog/topics/threat-intelligence/distillation-experimentation-integration-ai-adversarial-use",
      "published_date": "2026-02-13",
      "retrieved_at": "2026-08-14T21:14:00+08:00",
      "license": "No explicit republication license on page; Google copyright applies",
      "evidence_type": "Official threat-intelligence disclosure; partial quote from an observed attack instruction",
      "original_excerpt": "One identified attack instructed Gemini that the \"... language used in the thinking content must be strictly consistent with the main language of the user input.\" Analysis of this campaign revealed: Scale: Over 100,000 prompts identified.",
      "sha256": "ab8e791ef9928544aa01090e88165f888b86b7b0acdb56068bba51c63d09ec66",
      "snapshot_scope": "HTML, 409932 bytes",
      "redistribution": "Limited quotation with attribution only; do not mirror the whole page",
      "notes": "The ellipsis is in Google's publication, so this is only a partial prompt fragment. Google says it detected and disrupted the campaign. The page also gives a concise teacher/student and SFT definition."
    },
    {
      "id": "S03-validia-distillery",
      "institution_author": "Validia-AI; repository citation names Joe Vann",
      "title": "Distillery — Synthetic LLM Distillation Attack Prompt Dataset Generator",
      "url": "https://github.com/Validia-AI/distillery",
      "published_date": "2026-02-27",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "GPL-3.0",
      "evidence_type": "Third-party defensive research repository; synthetic seed prompts and generator",
      "original_excerpt": "This project generates realistic synthetic prompts that simulate real-world LLM distillation attacks, similar to those recently disclosed by:",
      "sha256": "b6f7c208fedba8bde57271917eba1ad5df5790091e2ce25de5b372aa24945aed",
      "related_sha256": {
        "seeds/cot_elicitation.json": "5de13f76f03f531360e63e8d7f38fc2a6cfb966e457c56a2280ef84752b14a7d",
        "seeds/reward_model_grading.json": "db6cf7e67ae2c7ca73958403057a9fa91be11e16bf95de43d9e625e39f0cce87",
        "seeds/tool_use_extraction.json": "60aa54e43fa7b014c50c63740dc5438105419472a435a6db50803f98b81e256d"
      },
      "snapshot_scope": "README plus three small seed files; 55 seed records total in those files; no generated 54K dataset downloaded",
      "redistribution": "Yes under GPL-3.0, preserving license and notices",
      "notes": "Repository plans 54,000 generated prompts: 41,500 attack and 12,500 benign. Its own README warns these are synthetic, not real traffic. GitHub API reports GPL-3.0 even though some secondary summaries have incorrectly called it MIT."
    },
    {
      "id": "S04-openai-sft-distillation-doc",
      "institution_author": "OpenAI",
      "title": "Supervised fine-tuning — Distilling from a larger model",
      "url": "https://platform.openai.com/docs/guides/distillation",
      "published_date": "Not stated on page; live documentation snapshot",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "No explicit content republication license on page",
      "evidence_type": "Official product documentation; legitimate response distillation workflow",
      "original_excerpt": "One way to build a training data set for a smaller model is to distill the results of a large model to create training data for supervised fine tuning.",
      "sha256": "efb78f5833a6d11cd4831b9faade6969d9d7657add64b7e9741ce26d0e25c99e",
      "snapshot_scope": "HTML, 518982 bytes; current URL resolves to the supervised fine-tuning guide",
      "redistribution": "Limited quotation with link; do not mirror the whole documentation page",
      "notes": "The official four-step flow is: optimize a large-model prompt against evals, capture good responses, build a dataset, fine-tune a smaller model. The page does not publish a universal distillation prompt."
    },
    {
      "id": "S05-openai-cookbook-wine",
      "institution_author": "OpenAI Cookbook contributors",
      "title": "Leveraging model distillation to fine-tune a model",
      "url": "https://github.com/openai/openai-cookbook/blob/main/examples/Leveraging_model_distillation_to_fine-tune_a_model.ipynb",
      "published_date": "2026-07-20 (latest file commit at retrieval)",
      "retrieved_at": "2026-08-14T21:20:00+08:00",
      "license": "MIT",
      "evidence_type": "Official runnable cookbook; complete prompt template, system message, structured output schema and evaluation flow",
      "original_excerpt": "Based on this wine review, guess the grape variety:",
      "sha256": "4df764d37dea995e11c1a2fbf4444d2932a29662557337229d4254fb99811c46",
      "snapshot_scope": "Notebook JSON, 34161 bytes",
      "redistribution": "Yes under MIT with license notice",
      "notes": "Uses a 500-row subset of a public wine-review dataset, obtains gpt-4o outputs, then distills into gpt-4o-mini. Dataset licensing must be checked separately before republishing review text."
    },
    {
      "id": "S06-microsoft-foundry-traces",
      "institution_author": "Microsoft Foundry",
      "title": "Traces → SFT Distillation",
      "url": "https://github.com/microsoft-foundry/fine-tuning/tree/main/Demos/TracesDistillation",
      "published_date": "2026-06-15 (README file commit at retrieval)",
      "retrieved_at": "2026-08-14T21:24:00+08:00",
      "license": "MIT",
      "evidence_type": "Official runnable demo; real agent traces, 20 prompt templates, full system prompt, tools and measured result",
      "original_excerpt": "The gpt-4.1-nano student, fine-tuned on traces from a gpt-4.1-mini teacher agent, matches the teacher's tool selection on every test row — at ~10× lower cost per token.",
      "sha256": "6832bfab2495a045d6bf55eb586ea08ec84d06066f2a571160bf19e654d86568",
      "related_sha256": {
        "fixtures/push_prompts.py": "e1b19bb69aaa8054699ddf629ec77294abfc8d265c2e7e6b9ea92dc533b",
        "fixtures/zava_system_prompt.md": "4fc14242b344839f6673d44c429b9bafb2a19536028d61b93aa83f413a024a10",
        "fixtures/zava_tools.json": "2493bb2265e1d59e3e9023204a87f10fda916507dbc6b6cb453f54bb671f347c"
      },
      "snapshot_scope": "README and three fixture files; no private traces downloaded",
      "redistribution": "Yes under MIT with license notice",
      "notes": "README reports baseline nano pass rate 60% and fine-tuned nano 100% on the included held-out structural tool-call test; this is a demo result, not a universal benchmark."
    },
    {
      "id": "S07-deepseek-r1-paper",
      "institution_author": "DeepSeek-AI; Daya Guo et al.",
      "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
      "url": "https://arxiv.org/abs/2501.12948",
      "published_date": "2025-01-22",
      "retrieved_at": "2026-08-14T21:18:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Primary research paper",
      "original_excerpt": "Moreover, the emergent reasoning patterns exhibited by these large-scale models can be systematically harnessed to guide and enhance the reasoning capabilities of smaller models.",
      "sha256": "6dda5a280d3ea97c487505c8869d23a3831ad661e0be81cf74b862569bc66b84",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation; do not assume the paper is openly relicensed beyond arXiv distribution terms",
      "notes": "Primary evidence for the R1 training/distillation claim. The paper describes 800k curated samples for distilled Qwen models in the associated repository/model card; it does not release those 800k samples."
    },
    {
      "id": "S08-deepseek-r1-model-card",
      "institution_author": "DeepSeek-AI",
      "title": "DeepSeek-R1 and DeepSeek-R1-Distill model card / repository",
      "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
      "published_date": "2025-01-20 repository created; README last relevant commit 2025-04-09",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "Repository and weights MIT; derived Qwen checkpoints also depend on Apache-2.0 bases; Llama variants retain relevant Llama licenses",
      "evidence_type": "Official model card and repository; released student checkpoints and usage guidance",
      "original_excerpt": "DeepSeek-R1-Distill models are fine-tuned based on open-source models, using samples generated by DeepSeek-R1.",
      "sha256": "d26d26ddb518fee60c6c6bf7a708bd751b1619d93a4944f188143693d956c77f",
      "snapshot_scope": "Hugging Face raw model card, 15994 bytes",
      "redistribution": "Model-card text/code under MIT with notice; verify base-model license before redistributing weights",
      "notes": "The card states Qwen distilled variants were fine-tuned with 800k samples curated with DeepSeek-R1, but those samples are not published here. It recommends a math inference directive; this is not the secret training prompt."
    },
    {
      "id": "S09-hinton-kd",
      "institution_author": "Geoffrey Hinton, Oriol Vinyals, Jeff Dean",
      "title": "Distilling the Knowledge in a Neural Network",
      "url": "https://arxiv.org/abs/1503.02531",
      "published_date": "2015-03-09",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Foundational research paper; probability/soft-target distillation",
      "original_excerpt": "We achieve some surprising results on MNIST and we show that we can significantly improve the acoustic model of a heavily used commercial system by distilling the knowledge in an ensemble of models into a single model.",
      "sha256": "fb175c757837377b7f00c50cbed6e9af27f625e4d66e4f9d20e67857fac6e84c",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation; do not redistribute the full paper as project-owned content",
      "notes": "Best primary source for teacher/student, soft targets, temperature and compression. This classic form often uses probability distributions rather than natural-language prompts."
    },
    {
      "id": "S10-sequence-level-kd",
      "institution_author": "Yoon Kim, Alexander M. Rush",
      "title": "Sequence-Level Knowledge Distillation",
      "url": "https://arxiv.org/abs/1606.07947",
      "published_date": "2016-06-25",
      "retrieved_at": "2026-08-14T21:18:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Primary research paper; teacher-generated whole-sequence targets",
      "original_excerpt": "Our best student model runs 10 times faster than its state-of-the-art teacher with little loss in performance.",
      "sha256": "938ac4ef4872be069e82d9236e24ad985270ac472307c113da2fed0611af402e",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation",
      "notes": "Useful bridge from soft-label KD to modern LLM response distillation: the teacher's full generated sequence becomes a target."
    },
    {
      "id": "S11-distilbert",
      "institution_author": "Victor Sanh, Lysandre Debut, Julien Chaumond, Thomas Wolf",
      "title": "DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter",
      "url": "https://arxiv.org/abs/1910.01108",
      "published_date": "2019-10-02",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Primary research paper; pretraining-stage distillation",
      "original_excerpt": "it is possible to reduce the size of a BERT model by 40%, while retaining 97% of its language understanding capabilities and being 60% faster.",
      "sha256": "87593071fff26bdfbe8d54907f55fc495719aac7e54b9f293ddc2c34ea21f257",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation",
      "notes": "Demonstrates a triple loss: language modeling, distillation and cosine-distance losses. No natural-language extraction prompt is involved."
    },
    {
      "id": "S12-tinybert",
      "institution_author": "Xiaoqi Jiao et al.",
      "title": "TinyBERT: Distilling BERT for Natural Language Understanding",
      "url": "https://arxiv.org/abs/1909.10351",
      "published_date": "2019-09-23",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Primary research paper; two-stage Transformer distillation",
      "original_excerpt": "TinyBERT with 4 layers is empirically effective and achieves more than 96.8% the performance of its teacher BERTBASE on GLUE benchmark, while being 7.5x smaller and 9.4x faster on inference.",
      "sha256": "5c137ff634cd4657a3e5e3245c97b3efbed1a173dfc90c8770b309bf6068b39a",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation",
      "notes": "Useful for explaining hidden-state and attention-map matching, another prompt-free form of distillation."
    },
    {
      "id": "S13-distilling-step-paper",
      "institution_author": "Cheng-Yu Hsieh et al.; Google Research",
      "title": "Distilling Step-by-Step! Outperforming Larger Language Models with Less Training Data and Smaller Model Sizes",
      "url": "https://arxiv.org/abs/2305.02301",
      "published_date": "2023-05-03",
      "retrieved_at": "2026-08-14T21:12:00+08:00",
      "license": "arXiv non-exclusive distribution license",
      "evidence_type": "Primary research paper; rationale distillation",
      "original_excerpt": "Our method extracts LLM rationales as additional supervision for training small models within a multi-task framework.",
      "sha256": "dae19a43e00dec6e1dca969f5c8c4d53bebacd6f19057c9bf454ae475b79c7e4",
      "snapshot_scope": "arXiv abstract HTML",
      "redistribution": "Quote with citation",
      "notes": "Primary evidence that label plus rationale can be more data-efficient than label-only distillation."
    },
    {
      "id": "S14-distilling-step-repo",
      "institution_author": "Google Research",
      "title": "google-research/distilling-step-by-step",
      "url": "https://github.com/google-research/distilling-step-by-step",
      "published_date": "2023-05-24 repository created",
      "retrieved_at": "2026-08-14T21:18:00+08:00",
      "license": "Apache-2.0",
      "evidence_type": "Official implementation and experiment commands",
      "original_excerpt": "Loss = alpha * label_prediction_loss + (1 - alpha) * rationale_generation_loss",
      "sha256": "ca4916db3401a91f90d6ddaa5e3dffa21e26e085d628c2ed4d58c79c1d4f69aa",
      "snapshot_scope": "README, 2814 bytes; datasets.zip not downloaded",
      "redistribution": "Yes under Apache-2.0 with notices",
      "notes": "Shows standard fine-tuning, standard distillation, and step-by-step distillation commands. The repository points to datasets.zip; the archive was not downloaded to avoid bulk local acquisition."
    },
    {
      "id": "S15-self-instruct",
      "institution_author": "Yizhong Wang et al.; University of Washington and Allen Institute for AI",
      "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions",
      "url": "https://github.com/yizhongw/self-instruct",
      "published_date": "2022-12-20 paper; repository updated 2023-03-27",
      "retrieved_at": "2026-08-14T21:18:00+08:00",
      "license": "Paper CC BY 4.0; repository Apache-2.0",
      "evidence_type": "Primary paper plus official data-generation code and prompt templates",
      "original_excerpt": "Our pipeline generates instructions, input, and output samples from a language model, then filters invalid or similar ones before using them to finetune the original model.",
      "sha256": "73f649b1d6c7f453c0dc5a9c790d817e8a903d30b6c2526405eb06196ea31290",
      "related_sha256": {
        "repository_README": "70874ae37f4999bde3ca4a4963ead0f7783cd76cbcb3a2e808fd41cbf234de7e",
        "self_instruct/bootstrap_instructions.py": "133e8a52b03dbccbcb956d0d4a843b8b02b9196d2e57ab01a192e36889b92637"
      },
      "snapshot_scope": "arXiv abstract HTML, repository README and bootstrap_instructions.py; no 52K/82K dataset downloaded",
      "redistribution": "Paper under CC BY 4.0; code under Apache-2.0; dataset terms should be checked per file/source model",
      "notes": "Not identical to teacher-student KD, but a key public method for producing instruction/input/output synthetic training triples. Authors warn 46% of 200 sampled data points may have problems."
    },
    {
      "id": "S16-stanford-alpaca",
      "institution_author": "Stanford Center for Research on Foundation Models / Tatsunori Lab",
      "title": "Stanford Alpaca: An Instruction-following LLaMA Model",
      "url": "https://github.com/tatsu-lab/stanford_alpaca",
      "published_date": "2023-03-10 repository created",
      "retrieved_at": "2026-08-14T21:18:00+08:00",
      "license": "Code Apache-2.0; dataset CC BY-NC 4.0; weight diff CC BY-NC 4.0",
      "evidence_type": "Official project repository; complete generation prompt, SFT format and original teacher-generated samples",
      "original_excerpt": "The current Alpaca model is fine-tuned from a 7B LLaMA model on 52K instruction-following data generated by the techniques in the Self-Instruct paper, with some modifications.",
      "sha256": "bc335b356f6de0a3294cf65d0a389edc4a796764e31e777b7a8d61a7cb7aa551",
      "related_sha256": {
        "prompt.txt": "c042d2a46d3bdc4fa4406deeed5951dc2dcd4ebf29ebf8237210c384f09b5b2d",
        "alpaca_data.json_byte_range_0_7999": "63512c685fabc8240f0742dc6dabd4c638577f48338e8e65c9b32f82dbae4093"
      },
      "snapshot_scope": "README, complete 1749-byte prompt.txt, and only bytes 0-7999 of the 52K-record dataset",
      "redistribution": "Generation prompt/code under Apache-2.0; dataset excerpts only under CC BY-NC 4.0 with attribution and non-commercial limitation",
      "notes": "The repository says outputs were generated by text-davinci-003 and the full 52K dataset cost less than $500. Only a byte-range sample was fetched; the 52K file was not downloaded."
    },
    {
      "id": "S17-openr1-math-220k",
      "institution_author": "Hugging Face Open R1 team",
      "title": "OpenR1-Math-220k",
      "url": "https://huggingface.co/datasets/open-r1/OpenR1-Math-220k",
      "sample_url": "https://datasets-server.huggingface.co/first-rows?dataset=open-r1%2FOpenR1-Math-220k&config=default&split=train",
      "published_date": "2025 (dataset card; exact publication date not stated in fetched card)",
      "retrieved_at": "2026-08-14T21:31:00+08:00",
      "license": "Apache-2.0",
      "evidence_type": "Open-source reasoning-distillation dataset card plus public first-rows API; one original input/reference/output record retained",
      "original_excerpt": "OpenR1-Math-220k is a large-scale dataset for mathematical reasoning. It consists of 220k math problems with two to four reasoning traces generated by DeepSeek R1 for problems from NuminaMath 1.5.",
      "sha256": "87a0390ba632618254d50c1614feb6a5a7803a28710118e1705bab18bda5e64e",
      "related_sha256": {
        "first_rows_api_response": "48086fa517baf97af8e847b1aad7be7ca47114afc2817c26906da1bd31b3544d"
      },
      "snapshot_scope": "Dataset card, 5131 bytes, and public first-rows response, 66027 bytes; only row 0 selected for the archive",
      "redistribution": "Yes under Apache-2.0 with attribution and license notice",
      "notes": "Hugging Face metadata reports 225,129 total examples across configs. Row 0 UUID 586fd646-76d6-5070-8c81-9993ab9d8559 contains one algebra word problem, a reference solution and two DeepSeek-R1 generations. The card discloses the exact prepended instruction and says at least one reasoning trace per problem is correct. This is direct evidence that some public reasoning-distillation corpora are math-heavy; it does not prove that the alleged Anthropic attack traffic was math-heavy."
    }
  ]
}