Sync KnowledgeBase notes from Nutstore WebDAV.
Replace demo notes with 146 vault notes and diagram assets; keep credentials only in local .env.
This commit is contained in:
secret-deus
@@ -0,0 +1,79 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 440" width="100%" role="img">
|
||||
<title>MoE 路由机制</title>
|
||||
<desc>混合专家模型中 Router 如何为每个 Token 选择专家，以及共享专家和路由专家的分工</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #26215C; }
|
||||
</style>
|
||||
|
||||
<!-- Input -->
|
||||
<rect x="220" y="16" width="240" height="36" rx="6" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<text class="tb" x="340" y="32" text-anchor="middle">输入：一批 Token（Attention 输出）</text>
|
||||
<text class="tc" x="340" y="48" text-anchor="middle">形状: (S, 4096)</text>
|
||||
|
||||
<!-- Always-active shared expert -->
|
||||
<line x1="220" y1="52" x2="160" y2="90" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<rect x="60" y="90" width="200" height="60" rx="8" fill="#E1F5EE" stroke="#0F6E56" stroke-width="0.5"/>
|
||||
<text class="tb" x="160" y="112" text-anchor="middle">共享专家(Shared Expert)</text>
|
||||
<text class="tc" x="160" y="128" text-anchor="middle">总是激活，捕获通用知识</text>
|
||||
<text class="tc" x="160" y="144" text-anchor="middle">所有 Token 都经过它</text>
|
||||
|
||||
<!-- Router -->
|
||||
<line x1="340" y1="52" x2="340" y2="78" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<rect x="230" y="80" width="220" height="40" rx="8" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tb" x="340" y="96" text-anchor="middle">Router (门控网络)</text>
|
||||
<text class="tc" x="340" y="112" text-anchor="middle">Softmax(Linear(token)) → 选 Top-6 Expert</text>
|
||||
|
||||
<!-- Experts grid -->
|
||||
<line x1="340" y1="120" x2="100" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<line x1="340" y1="120" x2="190" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<line x1="340" y1="120" x2="280" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<line x1="340" y1="120" x2="370" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<line x1="340" y1="120" x2="460" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<line x1="340" y1="120" x2="550" y2="180" stroke="#854F0B" stroke-width="0.5"/>
|
||||
|
||||
<!-- Expert boxes row 1 -->
|
||||
<rect x="30" y="180" width="130" height="44" rx="6" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<text class="tb" x="95" y="198" text-anchor="middle">Expert 0 (激活)</text>
|
||||
<text class="tc" x="95" y="216" text-anchor="middle">数学推理</text>
|
||||
|
||||
<rect x="170" y="180" width="130" height="44" rx="6" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tb" x="235" y="198" text-anchor="middle">Expert 1 (空闲)</text>
|
||||
<text class="tc" x="235" y="216" text-anchor="middle">法律知识</text>
|
||||
|
||||
<rect x="310" y="180" width="130" height="44" rx="6" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<text class="tb" x="375" y="198" text-anchor="middle">Expert 2 (激活)</text>
|
||||
<text class="tc" x="375" y="216" text-anchor="middle">代码生成</text>
|
||||
|
||||
<text class="tc" x="455" y="206" text-anchor="middle">...</text>
|
||||
|
||||
<rect x="490" y="180" width="130" height="44" rx="6" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<text class="tb" x="555" y="198" text-anchor="middle">Expert 383 (激活)</text>
|
||||
<text class="tc" x="555" y="216" text-anchor="middle">通用知识</text>
|
||||
|
||||
<!-- Combine -->
|
||||
<line x1="95" y1="224" x2="95" y2="270" stroke="#E24B4A" stroke-width="0.5"/>
|
||||
<line x1="375" y1="224" x2="375" y2="270" stroke="#E24B4A" stroke-width="0.5"/>
|
||||
<line x1="555" y1="224" x2="555" y2="270" stroke="#E24B4A" stroke-width="0.5"/>
|
||||
<line x1="160" y1="150" x2="160" y2="270" stroke="#0F6E56" stroke-width="0.5"/>
|
||||
|
||||
<!-- Merge -->
|
||||
<rect x="180" y="270" width="320" height="32" rx="6" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="290" text-anchor="middle">加权合并：Σ(gate_i × Expert_i) + Shared_Expert</text>
|
||||
|
||||
<line x1="340" y1="302" x2="340" y2="328" stroke="#888780" stroke-width="1"/>
|
||||
|
||||
<rect x="200" y="330" width="280" height="32" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="350" text-anchor="middle">输出: (S, 4096) — 和普通 FFN 输出形状相同</text>
|
||||
|
||||
<!-- Stats -->
|
||||
<rect x="60" y="380" width="560" height="48" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="400" text-anchor="middle">DeepSeek-V4: 384 专家, 每 Token 激活 6 个 + 1 共享 = 总参数 1.6T, 激活参数 49B</text>
|
||||
<text class="tc" x="340" y="420" text-anchor="middle">类比：拥有 384 本专业书的图书馆，每次只翻 6 本就能答出问题</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.9 KiB |
@@ -0,0 +1,90 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 520" width="100%" role="img">
|
||||
<title>Multi-Head Attention 数据流</title>
|
||||
<desc>输入通过 QKV 投影→拆分为多头→各头独立计算 Attention→合并→输出投影</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #26215C; }
|
||||
</style>
|
||||
|
||||
<!-- Input -->
|
||||
<rect x="240" y="16" width="200" height="32" rx="6" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="36" text-anchor="middle">输入 X: (S, 4096)</text>
|
||||
|
||||
<!-- Three projections -->
|
||||
<line x1="240" y1="48" x2="160" y2="74" stroke="#534AB7" stroke-width="1"/>
|
||||
<line x1="340" y1="48" x2="340" y2="74" stroke="#534AB7" stroke-width="1"/>
|
||||
<line x1="440" y1="48" x2="520" y2="74" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<rect x="100" y="74" width="120" height="32" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="160" y="94" text-anchor="middle">Q = X × W_Q</text>
|
||||
|
||||
<rect x="280" y="74" width="120" height="32" rx="6" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="94" text-anchor="middle">K = X × W_K</text>
|
||||
|
||||
<rect x="460" y="74" width="120" height="32" rx="6" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tm" x="520" y="94" text-anchor="middle">V = X × W_V</text>
|
||||
|
||||
<!-- Split into heads -->
|
||||
<text class="tc" x="160" y="120" text-anchor="middle">↓ reshape ↓</text>
|
||||
<text class="tc" x="340" y="120" text-anchor="middle">↓ reshape ↓</text>
|
||||
<text class="tc" x="520" y="120" text-anchor="middle">↓ reshape ↓</text>
|
||||
<text class="tc" x="340" y="134" text-anchor="middle">(S, 4096) → (S, 32, 128) → (32, S, 128)</text>
|
||||
|
||||
<!-- Heads row -->
|
||||
<text class="tb" x="50" y="160" text-anchor="middle">32 个 Head 独立计算</text>
|
||||
|
||||
<rect x="30" y="170" width="140" height="90" rx="6" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.3"/>
|
||||
<text class="tb" x="100" y="190" text-anchor="middle">Head 1 (dim=128)</text>
|
||||
<text class="tc" x="100" y="210" text-anchor="middle">Scores = Q₁K₁^T / √128</text>
|
||||
<text class="tc" x="100" y="226" text-anchor="middle">+ Causal Mask</text>
|
||||
<text class="tc" x="100" y="242" text-anchor="middle">→ Softmax → × V₁</text>
|
||||
<text class="tc" x="100" y="258" text-anchor="middle">→ (S, 128)</text>
|
||||
|
||||
<text class="tc" x="190" y="220" text-anchor="middle">...</text>
|
||||
|
||||
<rect x="220" y="170" width="140" height="90" rx="6" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tb" x="290" y="190" text-anchor="middle">Head 16 (dim=128)</text>
|
||||
<text class="tc" x="290" y="210" text-anchor="middle">Scores = Q₁₆K₁₆^T / √128</text>
|
||||
<text class="tc" x="290" y="226" text-anchor="middle">+ Causal Mask</text>
|
||||
<text class="tc" x="290" y="242" text-anchor="middle">→ Softmax → × V₁₆</text>
|
||||
<text class="tc" x="290" y="258" text-anchor="middle">→ (S, 128)</text>
|
||||
|
||||
<text class="tc" x="380" y="220" text-anchor="middle">...</text>
|
||||
|
||||
<rect x="410" y="170" width="140" height="90" rx="6" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.3"/>
|
||||
<text class="tb" x="480" y="190" text-anchor="middle">Head 32 (dim=128)</text>
|
||||
<text class="tc" x="480" y="210" text-anchor="middle">Scores = Q₃₂K₃₂^T / √128</text>
|
||||
<text class="tc" x="480" y="226" text-anchor="middle">+ Causal Mask</text>
|
||||
<text class="tc" x="480" y="242" text-anchor="middle">→ Softmax → × V₃₂</text>
|
||||
<text class="tc" x="480" y="258" text-anchor="middle">→ (S, 128)</text>
|
||||
|
||||
<!-- Merge -->
|
||||
<line x1="100" y1="260" x2="100" y2="290" stroke="#888780" stroke-width="0.5"/>
|
||||
<line x1="290" y1="260" x2="290" y2="290" stroke="#888780" stroke-width="0.5"/>
|
||||
<line x1="480" y1="260" x2="480" y2="290" stroke="#888780" stroke-width="0.5"/>
|
||||
|
||||
<text class="tb" x="340" y="288" text-anchor="middle">↓ Concatenate ↓</text>
|
||||
|
||||
<rect x="180" y="298" width="320" height="28" rx="6" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="316" text-anchor="middle">32 × (S,128) → 拼成 (S, 4096)</text>
|
||||
|
||||
<line x1="340" y1="326" x2="340" y2="348" stroke="#888780" stroke-width="0.5"/>
|
||||
|
||||
<!-- Output projection -->
|
||||
<rect x="200" y="350" width="280" height="32" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="370" text-anchor="middle">输出投影: (S, 4096) × W_O → (S, 4096)</text>
|
||||
|
||||
<line x1="340" y1="382" x2="340" y2="404" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<rect x="230" y="406" width="220" height="36" rx="6" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="432" text-anchor="middle">Attention 输出 (S, 4096)</text>
|
||||
|
||||
<!-- Bottom: comparison -->
|
||||
<rect x="60" y="460" width="560" height="48" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="478" text-anchor="middle">为什么多头：32 个头各看各的 → 一个头看语法，一个看语义，一个看位置...</text>
|
||||
<text class="tc" x="340" y="498" text-anchor="middle">最后拼起来再过一个线性层 → 融合所有头的信息 → 完整的理解</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.5 KiB |
@@ -0,0 +1,131 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 820" width="100%" role="img">
|
||||
<title>Decoder-Only Transformer 架构</title>
|
||||
<desc>LLaMA/GPT 类模型的一层完整架构：输入→Embedding→N层(Attention+FFN)→输出</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #26215C; }
|
||||
</style>
|
||||
|
||||
<!-- Input -->
|
||||
<rect x="260" y="20" width="160" height="32" rx="6" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="40" text-anchor="middle">"今天天气真好"</text>
|
||||
|
||||
<line x1="340" y1="52" x2="340" y2="70" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<!-- Tokenizer -->
|
||||
<rect x="240" y="70" width="200" height="28" rx="6" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="88" text-anchor="middle">Tokenizer: [今,天,天,气,真,好]</text>
|
||||
|
||||
<line x1="340" y1="98" x2="340" y2="116" stroke="#185FA5" stroke-width="1"/>
|
||||
|
||||
<!-- Embedding -->
|
||||
<rect x="230" y="116" width="220" height="32" rx="6" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="136" text-anchor="middle">Embedding: (6, 4096) 矩阵</text>
|
||||
|
||||
<line x1="340" y1="148" x2="340" y2="168" stroke="#854F0B" stroke-width="1"/>
|
||||
|
||||
<!-- Layer Box -->
|
||||
<rect x="60" y="168" width="560" height="380" rx="14" fill="transparent" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tb" x="340" y="192" text-anchor="middle">Transformer Layer × N（LLaMA-7B: 32 层, 70B: 80 层）</text>
|
||||
|
||||
<!-- Pre-Norm Attention -->
|
||||
<rect x="100" y="210" width="150" height="24" rx="4" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tc" x="175" y="226" text-anchor="middle">RMSNorm</text>
|
||||
|
||||
<line x1="175" y1="234" x2="175" y2="250" stroke="#888780" stroke-width="0.5"/>
|
||||
|
||||
<rect x="100" y="250" width="150" height="36" rx="6" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="175" y="266" text-anchor="middle">Multi-Head</text>
|
||||
<text class="tm" x="175" y="280" text-anchor="middle">Attention</text>
|
||||
|
||||
<line x1="175" y1="286" x2="175" y2="316" stroke="#185FA5" stroke-width="1"/>
|
||||
|
||||
<!-- Residual 1 -->
|
||||
<circle cx="175" cy="326" r="10" fill="#639922"/>
|
||||
<text class="tw" x="175" y="330" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="185" y1="326" x2="200" y2="326" stroke="#639922" stroke-width="1"/>
|
||||
|
||||
<!-- Residual line 1 -->
|
||||
<line x1="80" y1="198" x2="80" y2="326" stroke="#639922" stroke-width="1" stroke-dasharray="3,2"/>
|
||||
<line x1="80" y1="326" x2="165" y2="326" stroke="#639922" stroke-width="1" stroke-dasharray="3,2"/>
|
||||
<text class="tc" x="42" y="270" text-anchor="middle">残差 1</text>
|
||||
|
||||
<!-- Pre-Norm FFN -->
|
||||
<rect x="220" y="304" width="150" height="24" rx="4" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tc" x="295" y="320" text-anchor="middle">RMSNorm</text>
|
||||
|
||||
<line x1="295" y1="328" x2="295" y2="344" stroke="#888780" stroke-width="0.5"/>
|
||||
|
||||
<rect x="220" y="344" width="150" height="36" rx="6" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tm" x="295" y="360" text-anchor="middle">FFN (SwiGLU)</text>
|
||||
<text class="tm" x="295" y="374" text-anchor="middle">4096→11008→4096</text>
|
||||
|
||||
<line x1="295" y1="380" x2="295" y2="410" stroke="#854F0B" stroke-width="1"/>
|
||||
|
||||
<!-- Residual 2 -->
|
||||
<circle cx="295" cy="420" r="10" fill="#076E56"/>
|
||||
<text class="tw" x="295" y="424" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="175" y1="316" x2="285" y2="420" stroke="#0F6E56" stroke-width="1" stroke-dasharray="3,2"/>
|
||||
<text class="tc" x="235" y="380" text-anchor="middle">残差 2</text>
|
||||
|
||||
<line x1="305" y1="420" x2="320" y2="420" stroke="#0F6E56" stroke-width="1"/>
|
||||
|
||||
<!-- Right side detail -->
|
||||
<rect x="380" y="210" width="220" height="220" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
|
||||
<text class="tb" x="490" y="230" text-anchor="middle">Attention 细节</text>
|
||||
<text class="tc" x="400" y="248">输入 (6, 4096)</text>
|
||||
<text class="tc" x="400" y="264">↓ 投影</text>
|
||||
<text class="tc" x="400" y="280">Q(6,4096) K(6,4096) V(6,4096)</text>
|
||||
<text class="tc" x="400" y="296">↓ RoPE + 拆 32 头</text>
|
||||
<text class="tc" x="400" y="312">Q(32,6,128) K(32,6,128) V(32,6,128)</text>
|
||||
<text class="tc" x="400" y="328">↓ Scores=QK^T/√128</text>
|
||||
<text class="tc" x="400" y="344">Scores(32,6,6) + Causal Mask</text>
|
||||
<text class="tc" x="400" y="360">↓ Softmax</text>
|
||||
<text class="tc" x="400" y="376">Weights(32,6,6) × V</text>
|
||||
<text class="tc" x="400" y="392">↓ 合并 32 头</text>
|
||||
<text class="tc" x="400" y="408">输出 (6, 4096)</text>
|
||||
|
||||
<rect x="380" y="440" width="220" height="90" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tb" x="490" y="458" text-anchor="middle">参数分布（每层 LLaMA-7B）</text>
|
||||
<text class="tc" x="400" y="478">QKV 投影: 3×(4096→4096) = 50M</text>
|
||||
<text class="tc" x="400" y="494">Output 投影: (4096→4096) = 17M</text>
|
||||
<text class="tc" x="400" y="510">FFN up+gate: 2×(4096→11008) = 90M</text>
|
||||
<text class="tc" x="400" y="526">FFN down: (11008→4096) = 45M</text>
|
||||
|
||||
<!-- Output after layers -->
|
||||
<line x1="295" y1="430" x2="295" y2="470" stroke="#0F6E56" stroke-width="1"/>
|
||||
<line x1="295" y1="470" x2="340" y2="470" stroke="#0F6E56" stroke-width="1"/>
|
||||
<line x1="340" y1="470" x2="340" y2="500" stroke="#0F6E56" stroke-width="1"/>
|
||||
|
||||
<text class="tc" x="340" y="558" text-anchor="middle">↑ 上面这一层重复 32 次（LLaMA-7B）↑</text>
|
||||
|
||||
<line x1="340" y1="568" x2="340" y2="586" stroke="#0F6E56" stroke-width="1"/>
|
||||
|
||||
<!-- Post-Layer Norm -->
|
||||
<rect x="240" y="586" width="200" height="28" rx="6" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tc" x="340" y="604" text-anchor="middle">RMSNorm (最后一层后)</text>
|
||||
|
||||
<line x1="340" y1="614" x2="340" y2="632" stroke="#888780" stroke-width="1"/>
|
||||
|
||||
<!-- LM Head -->
|
||||
<rect x="220" y="632" width="240" height="36" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="652" text-anchor="middle">LM Head (4096 → 32000)</text>
|
||||
<text class="tc" x="340" y="666" text-anchor="middle">每个 token 预测下一个 token 的概率分布</text>
|
||||
|
||||
<line x1="340" y1="668" x2="340" y2="686" stroke="#534AB7" stroke-width="1"/>
|
||||
|
||||
<!-- Output -->
|
||||
<rect x="180" y="686" width="320" height="44" rx="6" fill="#26215C"/>
|
||||
<text class="tw" x="340" y="714" text-anchor="middle">输出: [天,天,气,真,好] 的下一个预测 → "！"</text>
|
||||
|
||||
<!-- Bottom note -->
|
||||
<text class="tc" x="340" y="760" text-anchor="middle">Decoder-Only：每个 token 只能看到它前面的 token（Causal Mask）</text>
|
||||
<text class="tc" x="340" y="778" text-anchor="middle">自回归生成：输出"！"后拼回序列 → Forward("今天天气真好！") → 预测下一个...</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.2 KiB |
@@ -0,0 +1,108 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 700" width="100%" role="img">
|
||||
<title>反向传播原理</title>
|
||||
<desc>反向传播将最终误差逐层分配回每个参数，告诉模型每个权重该怎么调</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 10px; fill: #888780; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 11px; font-weight: 500; fill: #26215C; }
|
||||
.tr { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #A32D2D; }
|
||||
.tg { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #0F6E56; }
|
||||
</style>
|
||||
|
||||
<text class="th" x="340" y="28" text-anchor="middle">问题：从错误到修正，中间差了最关键的一步</text>
|
||||
|
||||
<rect x="40" y="46" width="600" height="120" rx="10" fill="#FCEBEB" stroke="#F09595" stroke-width="0.3"/>
|
||||
|
||||
<text class="tb" x="340" y="72" text-anchor="middle">训练时你知道模型预测错了（Loss 很大），但问题来了：</text>
|
||||
|
||||
<text class="tr" x="80" y="98" text-anchor="middle">模型有几亿个参数，每个都对最终预测有贡献</text>
|
||||
<text class="tr" x="80" y="118" text-anchor="middle">你怎么知道哪些参数该调大、哪些该调小、各调多少？</text>
|
||||
|
||||
<text class="tc" x="80" y="144" text-anchor="middle">如果随机试 → 几亿个参数，每个试一次 → 永远训不完</text>
|
||||
<text class="tc" x="80" y="160" text-anchor="middle">如果只看最后一层的输出 → 前面的参数根本不知道自己的责任</text>
|
||||
|
||||
<text class="th" x="340" y="200" text-anchor="middle">方案：反向传播 — 沿着计算路径，把误差"回传"给每个参与者</text>
|
||||
|
||||
<text class="ts" x="340" y="222" text-anchor="middle">类比：团队项目搞砸了，但不需要开会扯皮——数学直接算出每人多大责任</text>
|
||||
|
||||
<rect x="40" y="240" width="600" height="170" rx="10" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="264" text-anchor="middle">前向传播：产生预测和误差</text>
|
||||
|
||||
<rect x="70" y="278" width="80" height="32" rx="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="110" y="298" text-anchor="middle">输入 x</text>
|
||||
|
||||
<line x1="150" y1="294" x2="180" y2="294" stroke="#534AB7" stroke-width="1.5" marker-end="url(#af)"/>
|
||||
|
||||
<rect x="185" y="278" width="80" height="32" rx="5" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="225" y="298" text-anchor="middle">Layer 1</text>
|
||||
|
||||
<line x1="265" y1="294" x2="295" y2="294" stroke="#185FA5" stroke-width="1.5" marker-end="url(#af)"/>
|
||||
|
||||
<rect x="300" y="278" width="80" height="32" rx="5" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="298" text-anchor="middle">Layer 2</text>
|
||||
|
||||
<line x1="380" y1="294" x2="410" y2="294" stroke="#854F0B" stroke-width="1.5" marker-end="url(#af)"/>
|
||||
|
||||
<rect x="415" y="278" width="80" height="32" rx="5" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tm" x="455" y="298" text-anchor="middle">...</text>
|
||||
|
||||
<line x1="495" y1="294" x2="525" y2="294" stroke="#3B6D11" stroke-width="1.5" marker-end="url(#af)"/>
|
||||
|
||||
<rect x="530" y="278" width="80" height="32" rx="5" fill="#F1EFE8" stroke="#888780" stroke-width="0.5"/>
|
||||
<text class="tm" x="570" y="298" text-anchor="middle">预测 y'</text>
|
||||
|
||||
<text class="tc" x="340" y="334" text-anchor="middle">↓ 对比真实答案 y ↓</text>
|
||||
|
||||
<rect x="250" y="346" width="180" height="32" rx="6" fill="#FCEBEB" stroke="#E24B4A" stroke-width="0.5"/>
|
||||
<text class="tr" x="340" y="366" text-anchor="middle">Loss = |y' - y|（误差）</text>
|
||||
|
||||
<text class="tc" x="340" y="398" text-anchor="middle">现在知道了"有多错"，但不知道"每层各错多少"</text>
|
||||
|
||||
<rect x="40" y="418" width="600" height="170" rx="10" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="442" text-anchor="middle">反向传播：沿原路把误差分配回去</text>
|
||||
|
||||
<rect x="530" y="458" width="80" height="32" rx="5" fill="#F1EFE8" stroke="#888780" stroke-width="0.5"/>
|
||||
<text class="tm" x="570" y="478" text-anchor="middle">dL/dy'</text>
|
||||
|
||||
<line x1="525" y1="474" x2="495" y2="474" stroke="#534AB7" stroke-width="1.5" marker-end="url(#ab)"/>
|
||||
|
||||
<rect x="415" y="458" width="80" height="32" rx="5" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tm" x="455" y="478" text-anchor="middle">dL/dW_n</text>
|
||||
|
||||
<line x1="410" y1="474" x2="380" y2="474" stroke="#534AB7" stroke-width="1.5" marker-end="url(#ab)"/>
|
||||
|
||||
<rect x="300" y="458" width="80" height="32" rx="5" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tm" x="340" y="478" text-anchor="middle">dL/dW_2</text>
|
||||
|
||||
<line x1="295" y1="474" x2="265" y2="474" stroke="#534AB7" stroke-width="1.5" marker-end="url(#ab)"/>
|
||||
|
||||
<rect x="185" y="458" width="80" height="32" rx="5" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="225" y="478" text-anchor="middle">dL/dW_1</text>
|
||||
|
||||
<line x1="180" y1="474" x2="150" y2="474" stroke="#534AB7" stroke-width="1.5" marker-end="url(#ab)"/>
|
||||
|
||||
<rect x="70" y="458" width="80" height="32" rx="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="110" y="478" text-anchor="middle">dL/dx</text>
|
||||
|
||||
<text class="tc" x="340" y="512" text-anchor="middle">Layer 1 的权重 W_1 只知道输入 x 是什么、Layer 2 传回来的 dL/dW_2 是什么</text>
|
||||
<text class="tc" x="340" y="528" text-anchor="middle">但 dL/dW_2 里已经包含了后面所有层的误差 → W_1 还是能收到正确的修正信号</text>
|
||||
|
||||
<text class="tb" x="340" y="556" text-anchor="middle">类比：</text>
|
||||
|
||||
<rect x="140" y="566" width="400" height="48" rx="6" fill="#26215C"/>
|
||||
<text class="tb" x="340" y="584" text-anchor="middle" fill="#fff">前向：原料流到各车间 → 做出来一个产品 → 质检发现有 10 个缺陷</text>
|
||||
<text class="tb" x="340" y="602" text-anchor="middle" fill="#fff">反向：从最后车间往回问 → "你造成的缺陷是 3 个" → "你造成 5 个" → "你造成 2 个"</text>
|
||||
|
||||
<defs>
|
||||
<marker id="af" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto"><path d="M2 1L8 5L2 9" fill="none" stroke="#534AB7" stroke-width="1.2" stroke-linecap="round"/></marker>
|
||||
<marker id="ab" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto"><path d="M8 1L2 5L8 9" fill="none" stroke="#534AB7" stroke-width="1.2" stroke-linecap="round"/></marker>
|
||||
</defs>
|
||||
|
||||
<text class="th" x="340" y="642" text-anchor="middle">反向传播对 GPU 意味着什么</text>
|
||||
|
||||
<rect x="40" y="658" width="600" height="34" rx="8" fill="#EAF3DE" stroke="#97C459" stroke-width="0.3"/>
|
||||
<text class="tg" x="340" y="680" text-anchor="middle">前向时必须存下每层的激活值（反向要用） → 这就是 O(n²) 显存的来源 → FlashAttention 等优化技术的动机</text>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.1 KiB |
@@ -0,0 +1,105 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 700" width="100%" role="img">
|
||||
<title>LLM 推理两阶段：Prefill + Decode</title>
|
||||
<desc>Prefill 阶段并行处理输入 Prompt，Decode 阶段逐 token 自回归生成，KV Cache 起关键作用</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 11px; fill: #ffffff; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 11px; font-weight: 500; fill: #26215C; }
|
||||
</style>
|
||||
|
||||
<!-- ===== PREFILL ===== -->
|
||||
<rect x="20" y="20" width="640" height="220" rx="10" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
<rect x="20" y="20" width="640" height="32" rx="10" fill="#185FA5"/>
|
||||
<rect x="20" y="40" width="640" height="12" fill="#185FA5"/>
|
||||
<text class="tw" x="340" y="41" text-anchor="middle">Prefill 阶段（一次搞定所有输入 token）</text>
|
||||
|
||||
<!-- Prompt -->
|
||||
<rect x="60" y="66" width="240" height="44" rx="6" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tb" x="180" y="84" text-anchor="middle">输入 Prompt：6 个 token</text>
|
||||
<text class="tc" x="180" y="100" text-anchor="middle">"Tell me a joke"</text>
|
||||
|
||||
<line x1="300" y1="88" x2="338" y2="88" stroke="#185FA5" stroke-width="1.5" marker-end="url(#arr-blue)"/>
|
||||
|
||||
<!-- Prefill processing -->
|
||||
<rect x="340" y="66" width="300" height="100" rx="6" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tm" x="490" y="86" text-anchor="middle">一次 Forward，并行处理所有 6 个 token</text>
|
||||
<text class="tc" x="490" y="104" text-anchor="middle">算 Attention: Q(6,4096) × K^T(6,4096) → 6×6 矩阵</text>
|
||||
<text class="tc" x="490" y="120" text-anchor="middle">算完把每层的 K 和 V 存入 KV Cache</text>
|
||||
<text class="tc" x="490" y="136" text-anchor="middle">瓶颈：Compute-bound（算力密集型）</text>
|
||||
<text class="tc" x="490" y="152" text-anchor="middle">GPU 利用率高 → 几千核心全部跑满</text>
|
||||
|
||||
<line x1="640" y1="88" x2="640" y2="278" stroke="#185FA5" stroke-width="1"/>
|
||||
|
||||
<text class="ts" x="340" y="196" text-anchor="middle">Prefill 结束后，KV Cache 里已经存了 6 个 token 的 K 和 V → 下一个 token 直接用</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arr-blue" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#185FA5" stroke-width="1.5" stroke-linecap="round"/>
|
||||
</marker>
|
||||
<marker id="arr-amber" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#854F0B" stroke-width="1.5" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<!-- Arrow to decode -->
|
||||
<line x1="340" y1="240" x2="340" y2="270" stroke="#854F0B" stroke-width="1.5" marker-end="url(#arr-amber)"/>
|
||||
|
||||
<!-- ===== DECODE ===== -->
|
||||
<rect x="20" y="272" width="640" height="200" rx="10" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<rect x="20" y="272" width="640" height="32" rx="10" fill="#854F0B"/>
|
||||
<rect x="20" y="292" width="640" height="12" fill="#854F0B"/>
|
||||
<text class="tw" x="340" y="293" text-anchor="middle">Decode 阶段（每次只生成 1 个 token，反复循环）</text>
|
||||
|
||||
<!-- Decode loop -->
|
||||
<rect x="60" y="316" width="560" height="144" rx="6" fill="#ffffff" stroke="#EF9F27" stroke-width="0.3"/>
|
||||
|
||||
<text class="tb" x="340" y="338" text-anchor="middle">每步 Decode：只处理 1 个新 token</text>
|
||||
|
||||
<text class="tc" x="80" y="358">Step 1:</text>
|
||||
<text class="tm" x="140" y="358">新 token "Why" → Q_new(1,4096)</text>
|
||||
<text class="tm" x="140" y="376">从 KV Cache 取出前 6 个 token 的 K 和 V</text>
|
||||
<text class="tm" x="140" y="394">Q_new × K_cache^T → Score(1,7) → 选出下一个 token "don't"</text>
|
||||
|
||||
<text class="tc" x="80" y="416">Step 2:</text>
|
||||
<text class="tm" x="140" y="416">新 token "don't" → Q_new2 × K_cache^T → 下一个 "scientists"</text>
|
||||
|
||||
<text class="tc" x="80" y="434">Step N:</text>
|
||||
<text class="tm" x="140" y="434">...重复直到生成 EOS token 或达到最大长度</text>
|
||||
|
||||
<text class="tc" x="340" y="454" text-anchor="middle">瓶颈：Memory-bandwidth-bound（显存带宽密集型）→ GPU 核心大部分在等数据</text>
|
||||
|
||||
<!-- Arrow to KV Cache explanation -->
|
||||
<line x1="340" y1="472" x2="340" y2="498" stroke="#534AB7" stroke-width="1.5" marker-end="url(#arr-purple)"/>
|
||||
|
||||
<!-- ===== KV CACHE ===== -->
|
||||
<rect x="20" y="500" width="640" height="185" rx="10" fill="#E1F5EE" stroke="#5DCAA5" stroke-width="0.5"/>
|
||||
<rect x="20" y="500" width="640" height="32" rx="10" fill="#0F6E56"/>
|
||||
<rect x="20" y="520" width="640" height="12" fill="#0F6E56"/>
|
||||
<text class="tw" x="340" y="521" text-anchor="middle">KV Cache：推理显存的沉默杀手</text>
|
||||
|
||||
<rect x="60" y="546" width="560" height="124" rx="6" fill="#ffffff" stroke="#5DCAA5" stroke-width="0.3"/>
|
||||
|
||||
<text class="tb" x="340" y="566" text-anchor="middle">以 LLaMA-7B (32 层, 32 头, 128 维, BF16) 为例</text>
|
||||
|
||||
<text class="tc" x="80" y="588">每层 KV Cache:</text>
|
||||
<text class="tm" x="200" y="588">32 头 × 128 维 × 2 (K+V) × 2 bytes (BF16) = 16KB / token</text>
|
||||
|
||||
<text class="tc" x="80" y="608">32 层总计:</text>
|
||||
<text class="tm" x="200" y="608">32 × 16KB = 512KB / token</text>
|
||||
|
||||
<text class="tc" x="80" y="628">4K 上下文:</text>
|
||||
<text class="tm" x="200" y="628">512KB × 4096 ≈ 2.1 GB KV Cache</text>
|
||||
|
||||
<text class="tc" x="80" y="648">128K 上下文:</text>
|
||||
<text class="tm" x="200" y="648">512KB × 131072 ≈ 68 GB KV Cache ← 比模型权重还大!</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arr-purple" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#534AB7" stroke-width="1.5" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.0 KiB |
@@ -0,0 +1,90 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 660" width="100%" role="img">
|
||||
<title>残差 + 主干的角色分工</title>
|
||||
<desc>残差保留原始信号，Attention/FFN 学习需要的修正部分</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 10px; fill: #888780; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 11px; font-weight: 500; fill: #26215C; }
|
||||
.tr { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #A32D2D; }
|
||||
.tg { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #0F6E56; }
|
||||
</style>
|
||||
|
||||
<text class="th" x="340" y="28" text-anchor="middle">用个生活例子说明残差和主干的分工</text>
|
||||
|
||||
<rect x="40" y="46" width="600" height="130" rx="10" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<text class="tb" x="340" y="70" text-anchor="middle">想象你在写一段文字，每次修改有三种方式：</text>
|
||||
|
||||
<rect x="60" y="84" width="560" height="40" rx="6" fill="#FCEBEB" stroke="#F09595" stroke-width="0.3"/>
|
||||
<text class="tr" x="80" y="100">方式 A：扔掉旧的，从头重写（没有残差）</text>
|
||||
<text class="tc" x="80" y="118">→ 每改一次可能全盘推翻，改几版之后初稿完全丢了，质量反而变差</text>
|
||||
|
||||
<rect x="60" y="132" width="560" height="34" rx="6" fill="#EAF3DE" stroke="#97C459" stroke-width="0.3"/>
|
||||
<text class="tg" x="80" y="148">方式 B：保留上一版，在它基础上修改（有残差）</text>
|
||||
<text class="tc" x="80" y="164">→ 每次只改需要改的部分，好的保留，差的修正 → 越来越完善</text>
|
||||
|
||||
<text class="th" x="340" y="210" text-anchor="middle">Attention/FFN 层学到的是什么？是"需要的修正"，不是"全部重算"</text>
|
||||
|
||||
<rect x="40" y="228" width="600" height="120" rx="10" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
|
||||
<rect x="70" y="248" width="90" height="28" rx="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="115" y="267" text-anchor="middle">输入 x = 10</text>
|
||||
|
||||
<line x1="160" y1="262" x2="190" y2="262" stroke="#534AB7" stroke-width="1.5" marker-end="url(#a1)"/>
|
||||
|
||||
<rect x="195" y="244" width="100" height="36" rx="6" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="245" y="260" text-anchor="middle">Attention</text>
|
||||
<text class="tm" x="245" y="274" text-anchor="middle">or FFN 层</text>
|
||||
|
||||
<line x1="295" y1="262" x2="330" y2="262" stroke="#185FA5" stroke-width="1.5" marker-end="url(#a2)"/>
|
||||
|
||||
<circle cx="345" cy="262" r="10" fill="#639922"/>
|
||||
<text class="tm" x="345" y="266" text-anchor="middle" fill="#fff">+</text>
|
||||
|
||||
<line x1="355" y1="262" x2="390" y2="262" stroke="#639922" stroke-width="1.5" marker-end="url(#a3)"/>
|
||||
|
||||
<rect x="395" y="248" width="90" height="28" rx="5" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tm" x="440" y="267" text-anchor="middle">输出 = 10 + 3 = 13</text>
|
||||
|
||||
<line x1="70" y1="248" x2="70" y2="290" stroke="#639922" stroke-width="1.5" stroke-dasharray="4,3"/>
|
||||
<line x1="70" y1="290" x2="335" y2="290" stroke="#639922" stroke-width="1.5" stroke-dasharray="4,3"/>
|
||||
<line x1="335" y1="290" x2="335" y2="272" stroke="#639922" stroke-width="1.5" stroke-dasharray="4,3"/>
|
||||
<text class="tc" x="200" y="302">残差直通：10 原样传到输出端</text>
|
||||
|
||||
<text class="tc" x="245" y="322" text-anchor="middle">层只负责算出 "多出来的 +3"</text>
|
||||
<text class="tc" x="340" y="340" text-anchor="middle">不需要重新算出 13</text>
|
||||
|
||||
<defs>
|
||||
<marker id="a1" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto"><path d="M2 1L8 5L2 9" fill="none" stroke="#534AB7" stroke-width="1.2" stroke-linecap="round"/></marker>
|
||||
<marker id="a2" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto"><path d="M2 1L8 5L2 9" fill="none" stroke="#185FA5" stroke-width="1.2" stroke-linecap="round"/></marker>
|
||||
<marker id="a3" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto"><path d="M2 1L8 5L2 9" fill="none" stroke="#639922" stroke-width="1.2" stroke-linecap="round"/></marker>
|
||||
</defs>
|
||||
|
||||
<text class="th" x="340" y="380" text-anchor="middle">训练初值到收敛：残差让模型从"不做改变"逐步学起</text>
|
||||
|
||||
<rect x="40" y="396" width="280" height="130" rx="8" fill="#FCEBEB" stroke="#F09595" stroke-width="0.3"/>
|
||||
<text class="tr" x="180" y="418" text-anchor="middle">如果没有残差</text>
|
||||
<text class="tc" x="180" y="438" text-anchor="middle">初始化时 F(x) 的输出是随机的</text>
|
||||
<text class="tc" x="180" y="456" text-anchor="middle">→ 第一层输出乱七八糟</text>
|
||||
<text class="tc" x="180" y="474" text-anchor="middle">→ 第二层输入乱七八糟</text>
|
||||
<text class="tc" x="180" y="492" text-anchor="middle">→ ... 越传越乱</text>
|
||||
<text class="tr" x="180" y="514" text-anchor="middle">深层网络训练极慢甚至训不动</text>
|
||||
|
||||
<rect x="360" y="396" width="280" height="130" rx="8" fill="#EAF3DE" stroke="#97C459" stroke-width="0.3"/>
|
||||
<text class="tg" x="500" y="418" text-anchor="middle">有残差</text>
|
||||
<text class="tc" x="500" y="438" text-anchor="middle">初始化时 F(x) ≈ 0（权重都是接近 0 的小数）</text>
|
||||
<text class="tc" x="500" y="456" text-anchor="middle">→ 第一层输出 ≈ x</text>
|
||||
<text class="tc" x="500" y="474" text-anchor="middle">→ 第二层输入 ≈ x</text>
|
||||
<text class="tc" x="500" y="492" text-anchor="middle">→ ... 每一层输入都和原始输入差不多</text>
|
||||
<text class="tg" x="500" y="514" text-anchor="middle">然后逐步学出需要改的地方，平稳收敛</text>
|
||||
|
||||
<text class="th" x="340" y="556" text-anchor="middle">残差 + 主干 = 每层只学"增量"，整个网络是逐层累积的修正</text>
|
||||
|
||||
<rect x="40" y="574" width="600" height="74" rx="8" fill="#26215C"/>
|
||||
<text class="tb" x="340" y="598" text-anchor="middle" fill="#fff">残差负责"保存已有的好东西"，Attention/FFN 负责"找出需要改进的地方"</text>
|
||||
<text class="tc" x="340" y="618" text-anchor="middle" fill="rgba(255,255,255,0.7)">两者缺一不可——没有残差深层训不动，没有主干信息永远不变</text>
|
||||
<text class="tc" x="340" y="636" text-anchor="middle" fill="rgba(255,255,255,0.7)">"最差情况 F(x)=0，输出还是 x"只是下限保障——真正靠的是 F(x) 学出来的修正</text>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.6 KiB |
@@ -0,0 +1,181 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 880" width="100%" role="img">
|
||||
<title>残差连接原理详解</title>
|
||||
<desc>残差连接如何解决梯度消失问题，以及 Pre-Norm 与 Post-Norm 的区别</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tn { font-family: -apple-system, sans-serif; font-size: 10px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.tr { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #A32D2D; }
|
||||
.tg { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #0F6E56; }
|
||||
.edge { fill: none; stroke: #B4B2A9; stroke-width: 0.8; }
|
||||
</style>
|
||||
|
||||
<text class="th" x="340" y="24" text-anchor="middle">问题：没有残差连接会怎样？</text>
|
||||
|
||||
<rect x="40" y="38" width="600" height="200" rx="10" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<text class="tr" x="55" y="60">没有残差连接：output = Layer(input)，梯度逐层衰减</text>
|
||||
|
||||
<rect x="60" y="80" width="120" height="36" rx="6" fill="#E24B4A"/>
|
||||
<text class="tw" x="120" y="102" text-anchor="middle">Layer 1</text>
|
||||
|
||||
<line x1="180" y1="98" x2="220" y2="98" stroke="#E24B4A" stroke-width="1.5" marker-end="url(#arrow-red)"/>
|
||||
<text class="tc" x="200" y="88" text-anchor="middle">x₁</text>
|
||||
|
||||
<rect x="220" y="80" width="120" height="36" rx="6" fill="#E24B4A"/>
|
||||
<text class="tw" x="280" y="102" text-anchor="middle">Layer 2</text>
|
||||
|
||||
<line x1="340" y1="98" x2="380" y2="98" stroke="#E24B4A" stroke-width="1.5" marker-end="url(#arrow-red)"/>
|
||||
<text class="tc" x="360" y="88" text-anchor="middle">x₂</text>
|
||||
|
||||
<rect x="380" y="80" width="120" height="36" rx="6" fill="#E24B4A"/>
|
||||
<text class="tw" x="440" y="102" text-anchor="middle">Layer 3</text>
|
||||
|
||||
<line x1="500" y1="98" x2="540" y2="98" stroke="#E24B4A" stroke-width="1.5" marker-end="url(#arrow-red)"/>
|
||||
<text class="tc" x="520" y="88" text-anchor="middle">x₃</text>
|
||||
|
||||
<rect x="540" y="80" width="80" height="36" rx="6" fill="#A32D2D" stroke="#791F1F" stroke-width="0.5"/>
|
||||
<text class="tw" x="580" y="102" text-anchor="middle">Loss</text>
|
||||
|
||||
<text class="ts" x="340" y="140" text-anchor="middle">反向传播时梯度怎么走：</text>
|
||||
|
||||
<text class="tm" x="120" y="162" text-anchor="middle">dL/dx₁</text>
|
||||
<text class="tc" x="120" y="178" text-anchor="middle">= dL/dx₃ * dx₃/dx₂ * dx₂/dx₁</text>
|
||||
<text class="tr" x="120" y="196" text-anchor="middle">三层链式乘法 → 梯度越来越小 → 梯度消失！</text>
|
||||
|
||||
<text class="tn" x="340" y="218" text-anchor="middle">Layer 1 几乎收不到有效梯度 → 模型前几层永远学不到东西</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arrow-red" viewBox="0 0 10 10" refX="8" refY="5"
|
||||
markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#E24B4A" stroke-width="1.5" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<text class="th" x="340" y="258" text-anchor="middle">方案：加一条"短路"，让输入直接绕过 Layer 传到输出</text>
|
||||
|
||||
<rect x="40" y="274" width="600" height="215" rx="10" fill="#EAF3DE" stroke="#97C459" stroke-width="0.5"/>
|
||||
<text class="tg" x="55" y="296">有残差连接：output = Layer(input) + input，梯度走两条路</text>
|
||||
|
||||
<rect x="60" y="312" width="120" height="36" rx="6" fill="#3B6D11"/>
|
||||
<text class="tw" x="120" y="334" text-anchor="middle">Layer 1</text>
|
||||
|
||||
<line x1="60" y1="312" x2="170" y2="360" stroke="#639922" stroke-width="1.5"/>
|
||||
<circle cx="180" cy="360" r="10" fill="#639922"/>
|
||||
<text class="tw" x="180" y="364" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="190" y1="360" x2="220" y2="330" stroke="#3B6D11" stroke-width="1.5"/>
|
||||
|
||||
<rect x="220" y="312" width="120" height="36" rx="6" fill="#3B6D11"/>
|
||||
<text class="tw" x="280" y="334" text-anchor="middle">Layer 2</text>
|
||||
|
||||
<line x1="220" y1="312" x2="330" y2="360" stroke="#639922" stroke-width="1.5"/>
|
||||
<circle cx="340" cy="360" r="10" fill="#639922"/>
|
||||
<text class="tw" x="340" y="364" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="350" y1="360" x2="380" y2="330" stroke="#3B6D11" stroke-width="1.5"/>
|
||||
|
||||
<rect x="380" y="312" width="120" height="36" rx="6" fill="#3B6D11"/>
|
||||
<text class="tw" x="440" y="334" text-anchor="middle">Layer 3</text>
|
||||
|
||||
<line x1="380" y1="312" x2="490" y2="360" stroke="#639922" stroke-width="1.5"/>
|
||||
<circle cx="500" cy="360" r="10" fill="#639922"/>
|
||||
<text class="tw" x="500" y="364" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="510" y1="360" x2="540" y2="330" stroke="#3B6D11" stroke-width="1.5"/>
|
||||
|
||||
<rect x="540" y="312" width="80" height="36" rx="6" fill="#27500A" stroke="#173404" stroke-width="0.5"/>
|
||||
<text class="tw" x="580" y="334" text-anchor="middle">Loss</text>
|
||||
|
||||
<text class="ts" x="340" y="390" text-anchor="middle">反向传播时梯度怎么走：</text>
|
||||
|
||||
<text class="tm" x="150" y="412" text-anchor="middle">路径 1（通过 Layer 1）：</text>
|
||||
<text class="tc" x="150" y="428" text-anchor="middle">dL/dx₀ = dL/dx₃ * dx₃/dx₂ * dx₂/dx₁ * dx₁/dx₀</text>
|
||||
<text class="tc" x="150" y="442" text-anchor="middle">← 还是三层连乘，可能消失</text>
|
||||
|
||||
<text class="tm" x="480" y="412" text-anchor="middle">路径 2（通过残差短路）：</text>
|
||||
<text class="tc" x="480" y="428" text-anchor="middle">d(x₁ + x₀)/dx₀ = 1 ← 导数是常数！</text>
|
||||
<text class="tg" x="480" y="445" text-anchor="middle">不管层多深，梯度至少能原封不动传回来！</text>
|
||||
|
||||
<text class="tb" x="340" y="472" text-anchor="middle">残差连接 = 给梯度开了一条"高速公路"，跳过中间的乘法</text>
|
||||
|
||||
<text class="th" x="340" y="510" text-anchor="middle">用数字感受一下</text>
|
||||
|
||||
<rect x="40" y="524" width="290" height="100" rx="8" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<text class="tr" x="185" y="544" text-anchor="middle">没有残差连接</text>
|
||||
<text class="tm" x="185" y="564" text-anchor="middle">每层梯度缩放 0.5x</text>
|
||||
<text class="tm" x="185" y="584" text-anchor="middle">1.0 → 0.5 → 0.25 → 0.125 → 0.06 → 0.03</text>
|
||||
<text class="tr" x="185" y="604" text-anchor="middle">→ 5 层后梯度只剩 3%，前几层训不动</text>
|
||||
<text class="tc" x="185" y="618">这就是 100 层网络训不了的原因</text>
|
||||
|
||||
<rect x="350" y="524" width="290" height="100" rx="8" fill="#EAF3DE" stroke="#97C459" stroke-width="0.5"/>
|
||||
<text class="tg" x="495" y="544" text-anchor="middle">有残差连接</text>
|
||||
<text class="tm" x="495" y="564" text-anchor="middle">每层梯度 = 走 Layer(0.5x) + 走残差(1.0x) = 1.5x</text>
|
||||
<text class="tm" x="495" y="584" text-anchor="middle">1.0 → 1.5 → 2.25 → 3.38 → 5.06 → 7.59</text>
|
||||
<text class="tg" x="495" y="604" text-anchor="middle">→ 梯度不但没消失，还在累积！浅层学得动</text>
|
||||
<text class="tc" x="495" y="618">LLaMA-70B 的 80 层就是这样堆出来的</text>
|
||||
|
||||
<text class="th" x="340" y="660" text-anchor="middle">LLaMA 一层里的两个残差连接（Pre-Norm）</text>
|
||||
|
||||
<rect x="40" y="678" width="600" height="155" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<rect x="70" y="698" width="80" height="28" rx="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="110" y="716" text-anchor="middle">输入 x</text>
|
||||
|
||||
<line x1="150" y1="712" x2="178" y2="712" stroke="#534AB7" stroke-width="1" marker-end="url(#arrow-purple)"/>
|
||||
|
||||
<rect x="180" y="690" width="100" height="18" rx="4" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tc" x="230" y="703" text-anchor="middle">RMSNorm</text>
|
||||
<rect x="180" y="712" width="100" height="28" rx="5" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="230" y="730" text-anchor="middle">Attention</text>
|
||||
|
||||
<line x1="280" y1="726" x2="308" y2="726" stroke="#185FA5" stroke-width="1" marker-end="url(#arrow-blue)"/>
|
||||
|
||||
<circle cx="320" cy="726" r="10" fill="#639922"/>
|
||||
<text class="tw" x="320" y="730" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="70" y1="698" x2="310" y2="716" stroke="#639922" stroke-width="1.5" stroke-dasharray="4,3"/>
|
||||
<text class="tc" x="200" y="772">残差 1：x 直接跳过 Attention，加到 Attention 的输出上</text>
|
||||
|
||||
<line x1="330" y1="726" x2="360" y2="726" stroke="#639922" stroke-width="1" marker-end="url(#arrow-green)"/>
|
||||
|
||||
<rect x="362" y="690" width="100" height="18" rx="4" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.3"/>
|
||||
<text class="tc" x="412" y="703" text-anchor="middle">RMSNorm</text>
|
||||
<rect x="362" y="712" width="100" height="28" rx="5" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tm" x="412" y="730" text-anchor="middle">FFN (SwiGLU)</text>
|
||||
|
||||
<line x1="462" y1="726" x2="490" y2="726" stroke="#854F0B" stroke-width="1" marker-end="url(#arrow-amber)"/>
|
||||
|
||||
<circle cx="502" cy="726" r="10" fill="#076E56"/>
|
||||
<text class="tw" x="502" y="730" text-anchor="middle" font-size="9">+</text>
|
||||
|
||||
<line x1="310" y1="736" x2="492" y2="726" stroke="#0F6E56" stroke-width="1.5" stroke-dasharray="4,3"/>
|
||||
<text class="tc" x="480" y="772">残差 2：Attention 的输出跳过 FFN，加到 FFN 的输出上</text>
|
||||
|
||||
<line x1="512" y1="726" x2="540" y2="726" stroke="#0F6E56" stroke-width="1"/>
|
||||
|
||||
<rect x="542" y="712" width="80" height="28" rx="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="582" y="730" text-anchor="middle">输出 h'</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arrow-purple" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#534AB7" stroke-width="1.2" stroke-linecap="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-blue" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#185FA5" stroke-width="1.2" stroke-linecap="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-green" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#639922" stroke-width="1.2" stroke-linecap="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-amber" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#854F0B" stroke-width="1.2" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<rect x="80" y="800" width="520" height="44" rx="10" fill="#26215C"/>
|
||||
<text class="tw" x="340" y="820" text-anchor="middle">残差连接的本质：y = F(x) + x — 最差情况 F(x)=0，输出还是 x，不会退化</text>
|
||||
<text class="tw" x="340" y="836" text-anchor="middle">有了它，100 层甚至 1000 层的网络都能训练</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,120 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 830" width="100%" role="img">
|
||||
<title>注意力矩阵完整推导</title>
|
||||
<desc>三步：Q×K^T 得出原始分数 → 除以 √d 缩放 → Softmax 归一化为概率</desc>
|
||||
<defs>
|
||||
<marker id="arr" viewBox="0 0 10 10" refX="8" refY="5"
|
||||
markerWidth="6" markerHeight="6" orient="auto-start-reverse">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="context-stroke"
|
||||
stroke-width="1.2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tn { font-family: -apple-system, sans-serif; font-size: 10px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 10px; font-weight: 500; fill: #26215C; }
|
||||
.tp { font-family: -apple-system, monospace; font-size: 9px; fill: #534AB7; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.edge { fill: none; stroke: #B4B2A9; stroke-width: 0.8; }
|
||||
</style>
|
||||
|
||||
<text class="th" x="340" y="24" text-anchor="middle">第 1 步：Q 和 K 是从哪来的</text>
|
||||
|
||||
<rect x="40" y="38" width="600" height="60" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="ts" x="340" y="58" text-anchor="middle">输入：4 个 token 的 Embedding，每个 4096 维（简化为 4 维示意）</text>
|
||||
|
||||
<rect x="80" y="70" width="100" height="20" rx="4" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<text class="tm" x="130" y="84" text-anchor="middle">[我]</text>
|
||||
<rect x="195" y="70" width="100" height="20" rx="4" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
<text class="tm" x="245" y="84" text-anchor="middle">[爱]</text>
|
||||
<rect x="310" y="70" width="100" height="20" rx="4" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<text class="tm" x="360" y="84" text-anchor="middle">[吃]</text>
|
||||
<rect x="425" y="70" width="100" height="20" rx="4" fill="#EAF3DE" stroke="#97C459" stroke-width="0.5"/>
|
||||
<text class="tm" x="475" y="84" text-anchor="middle">[苹果]</text>
|
||||
|
||||
<line x1="160" y1="98" x2="130" y2="124" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="520" y1="98" x2="490" y2="124" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<rect x="80" y="124" width="240" height="32" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tm" x="200" y="144" text-anchor="middle">X × W_Q → Q (每个 token 的"提问"能力)</text>
|
||||
|
||||
<rect x="360" y="124" width="240" height="32" rx="6" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tm" x="480" y="144" text-anchor="middle">X × W_K → K (每个 token 的"标签"能力)</text>
|
||||
|
||||
<rect x="40" y="170" width="280" height="130" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tb" x="180" y="188" text-anchor="middle">Q 矩阵 (4 token × 4 dim)</text>
|
||||
<line x1="60" y1="195" x2="300" y2="195" stroke="#534AB7" stroke-width="0.3"/>
|
||||
<text class="tp" x="100" y="210" text-anchor="middle">[0.5, 0.3, -0.2, 0.8] ← "我"的 Q</text>
|
||||
<text class="tp" x="100" y="228" text-anchor="middle">[0.1, -0.4, 0.7, 0.2] ← "爱"的 Q</text>
|
||||
<text class="tp" x="100" y="246" text-anchor="middle">[-0.3, 0.6, 0.1, -0.5] ← "吃"的 Q</text>
|
||||
<text class="tp" x="100" y="264" text-anchor="middle">[0.7, -0.1, -0.3, 0.4] ← "苹果"的 Q</text>
|
||||
|
||||
<rect x="360" y="170" width="280" height="130" rx="6" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tb" x="500" y="188" text-anchor="middle">K 矩阵 (4 token × 4 dim)</text>
|
||||
<line x1="380" y1="195" x2="620" y2="195" stroke="#185FA5" stroke-width="0.3"/>
|
||||
<text class="tp" x="420" y="210" text-anchor="middle">[ 0.3, -0.5, 0.2, 0.1] ← "我"的 K</text>
|
||||
<text class="tp" x="420" y="228" text-anchor="middle">[-0.2, 0.6, 0.2, -0.3] ← "爱"的 K</text>
|
||||
<text class="tp" x="420" y="246" text-anchor="middle">[ 0.4, -0.1, -0.4, 0.5] ← "吃"的 K</text>
|
||||
<text class="tp" x="420" y="264" text-anchor="middle">[ 0.1, 0.3, 0.6, -0.2] ← "苹果"的 K</text>
|
||||
|
||||
<text class="th" x="340" y="325" text-anchor="middle" font-size="15" fill="#534AB7">↓ Q × K^T ↓</text>
|
||||
|
||||
<text class="th" x="340" y="356" text-anchor="middle">第 2 步：Q × K^T = 原始注意力分数</text>
|
||||
|
||||
<rect x="60" y="372" width="560" height="120" rx="8" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<text class="ts" x="90" y="396">核心操作：Scores[i][j] = Q 的第 i 行 · K 的第 j 行（两个 4 维向量的点积）</text>
|
||||
|
||||
<rect x="90" y="408" width="500" height="75" rx="6" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="426" text-anchor="middle">以 Scores[1][2] 为例：「爱」的 Q · 「吃」的 K</text>
|
||||
<text class="tp" x="340" y="444" text-anchor="middle">(0.1)×(0.4) + (-0.4)×(-0.1) + (0.7)×(-0.4) + (0.2)×(0.5)</text>
|
||||
<text class="tp" x="340" y="462" text-anchor="middle">= 0.04 + 0.04 + (-0.28) + 0.10 = -0.10</text>
|
||||
<text class="tc" x="340" y="477" text-anchor="middle">「爱」和「吃」的注意力分数是 -0.10，说明关系不太密切</text>
|
||||
|
||||
<text class="th" x="340" y="518" text-anchor="middle" font-size="15" fill="#534AB7">↓ 算出 4×4 矩阵后 ↓</text>
|
||||
|
||||
<text class="th" x="340" y="548" text-anchor="middle">第 3 步：缩放 + Causal Mask + Softmax = 注意力权重</text>
|
||||
|
||||
<rect x="40" y="564" width="200" height="150" rx="6" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
<text class="tb" x="140" y="582" text-anchor="middle">原始 Scores (Q×K^T)</text>
|
||||
<line x1="50" y1="590" x2="230" y2="590" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
<text class="tp" x="140" y="608" text-anchor="middle">[ 0.35 -0.18 -0.25 0.15] "我"</text>
|
||||
<text class="tp" x="140" y="624" text-anchor="middle">[ 0.12 0.42 -0.10 0.08] "爱"</text>
|
||||
<text class="tp" x="140" y="640" text-anchor="middle">[-0.20 0.33 0.55 0.41] "吃"</text>
|
||||
<text class="tp" x="140" y="656" text-anchor="middle">[ 0.28 -0.15 0.38 0.62] "苹果"</text>
|
||||
<text class="tc" x="140" y="676">每行 = 这个 token 看所有 token</text>
|
||||
<text class="tc" x="140" y="690">值越大 = 越关注</text>
|
||||
|
||||
<text class="tn" x="255" y="640" text-anchor="middle" font-size="16" fill="#888780">→</text>
|
||||
<text class="tc" x="255" y="654" text-anchor="middle">÷√d</text>
|
||||
|
||||
<rect x="270" y="564" width="200" height="150" rx="6" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<text class="tb" x="370" y="582" text-anchor="middle">除以 √4 = 2</text>
|
||||
<line x1="280" y1="590" x2="460" y2="590" stroke="#EF9F27" stroke-width="0.3"/>
|
||||
<text class="tp" x="370" y="608" text-anchor="middle">[ 0.18 -0.09 -0.13 0.08]</text>
|
||||
<text class="tp" x="370" y="624" text-anchor="middle">[ 0.06 0.21 -0.05 0.04]</text>
|
||||
<text class="tp" x="370" y="640" text-anchor="middle">[-0.10 0.17 0.28 0.21]</text>
|
||||
<text class="tp" x="370" y="656" text-anchor="middle">[ 0.14 -0.08 0.19 0.31]</text>
|
||||
<text class="tc" x="370" y="676">除以 √d 防止分数太大</text>
|
||||
<text class="tc" x="370" y="690">→ Softmax 后梯度更稳定</text>
|
||||
|
||||
<text class="tn" x="485" y="640" text-anchor="middle" font-size="16" fill="#888780">→</text>
|
||||
<text class="tc" x="485" y="654" text-anchor="middle">Softmax</text>
|
||||
|
||||
<rect x="500" y="564" width="150" height="150" rx="6" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tb" x="575" y="582" text-anchor="middle">注意力权重</text>
|
||||
<line x1="510" y1="590" x2="640" y2="590" stroke="#3B6D11" stroke-width="0.3"/>
|
||||
<text class="tp" x="575" y="608" text-anchor="middle">[0.28 0.19 0.18 0.35]</text>
|
||||
<text class="tp" x="575" y="624" text-anchor="middle">[0.22 0.31 0.24 0.23]</text>
|
||||
<text class="tp" x="575" y="640" text-anchor="middle">[0.15 0.19 0.35 0.31]</text>
|
||||
<text class="tp" x="575" y="656" text-anchor="middle">[0.21 0.16 0.25 0.38]</text>
|
||||
<text class="tc" x="575" y="676">每行加起来 = 1</text>
|
||||
<text class="tc" x="575" y="690">→ 概率分布！</text>
|
||||
|
||||
<rect x="140" y="740" width="400" height="44" rx="10" fill="#26215C"/>
|
||||
<text class="tw" x="340" y="766" text-anchor="middle" font-size="12">最后一步：注意力权重 × V → 加权求和 → 每个 token 的输出</text>
|
||||
|
||||
<text class="tc" x="340" y="808" text-anchor="middle">这个 4×4 的注意力矩阵 = O(n²) 显存的来源 —— 序列每长一倍，这个矩阵大四倍</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 8.5 KiB |
@@ -0,0 +1,166 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 880" width="100%" role="img">
|
||||
<title>GPU 集群运维知识图谱</title>
|
||||
<desc>22个文件、9个模块的完整双向互联知识结构</desc>
|
||||
<defs>
|
||||
<marker id="arr" viewBox="0 0 10 10" refX="8" refY="5"
|
||||
markerWidth="6" markerHeight="6" orient="auto-start-reverse">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="context-stroke"
|
||||
stroke-width="1.2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<style>
|
||||
.t { font-family: -apple-system, sans-serif; font-size: 14px; fill: #26215C; }
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 10px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #ffffff; }
|
||||
.tws { font-family: -apple-system, sans-serif; font-size: 11px; fill: rgba(255,255,255,0.9); }
|
||||
.edge { fill: none; stroke: #B4B2A9; stroke-width: 0.8; }
|
||||
</style>
|
||||
|
||||
<rect x="140" y="24" width="400" height="44" rx="22"
|
||||
fill="#26215C" stroke="none"/>
|
||||
<text class="tw" x="340" y="50" text-anchor="middle" dominant-baseline="central">GPU 集群运维知识总览</text>
|
||||
|
||||
<line x1="200" y1="68" x2="130" y2="120" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="275" y1="68" x2="220" y2="120" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="340" y1="68" x2="340" y2="120" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="405" y1="68" x2="460" y2="120" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="480" y1="68" x2="560" y2="120" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<rect x="46" y="120" width="170" height="160" rx="12" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<rect x="46" y="120" width="170" height="32" rx="12" fill="#534AB7"/>
|
||||
<rect x="46" y="140" width="170" height="12" fill="#534AB7"/>
|
||||
<text class="tw" x="131" y="140" text-anchor="middle" dominant-baseline="central">硬件基础</text>
|
||||
<text class="tc" x="55" y="168" dominant-baseline="central">3 篇笔记</text>
|
||||
<text class="ts" x="55" y="188" dominant-baseline="central">NVIDIA GPU 架构演进</text>
|
||||
<text class="ts" x="55" y="208" dominant-baseline="central">GPU 服务器硬件选型指南</text>
|
||||
<text class="ts" x="55" y="228" dominant-baseline="central">NVLink 与 NVSwitch 拓扑详解</text>
|
||||
<text class="tc" x="55" y="258" dominant-baseline="central">硬件 → 调度 → 网络</text>
|
||||
|
||||
<rect x="250" y="120" width="180" height="160" rx="12" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
<rect x="250" y="120" width="180" height="32" rx="12" fill="#185FA5"/>
|
||||
<rect x="250" y="140" width="180" height="12" fill="#185FA5"/>
|
||||
<text class="tw" x="340" y="140" text-anchor="middle" dominant-baseline="central">集群调度</text>
|
||||
<text class="tc" x="259" y="168" dominant-baseline="central">4 篇笔记</text>
|
||||
<text class="ts" x="259" y="188" dominant-baseline="central">K8s GPU 调度机制详解</text>
|
||||
<text class="ts" x="259" y="208" dominant-baseline="central">Device Plugin 与 DRA 对比</text>
|
||||
<text class="ts" x="259" y="228" dominant-baseline="central">GPU 资源分配与隔离策略</text>
|
||||
<text class="ts" x="259" y="248" dominant-baseline="central">Volcano 调度器实战</text>
|
||||
|
||||
<rect x="464" y="120" width="180" height="160" rx="12" fill="#E1F5EE" stroke="#5DCAA5" stroke-width="0.5"/>
|
||||
<rect x="464" y="120" width="180" height="32" rx="12" fill="#0F6E56"/>
|
||||
<rect x="464" y="140" width="180" height="12" fill="#0F6E56"/>
|
||||
<text class="tw" x="554" y="140" text-anchor="middle" dominant-baseline="central">网络互联</text>
|
||||
<text class="tc" x="473" y="168" dominant-baseline="central">3 篇笔记</text>
|
||||
<text class="ts" x="473" y="188" dominant-baseline="central">RDMA 与 InfiniBand 详解</text>
|
||||
<text class="ts" x="473" y="208" dominant-baseline="central">NCCL 通信原理与调优</text>
|
||||
<text class="ts" x="473" y="228" dominant-baseline="central">GPU 集群网络拓扑设计</text>
|
||||
<text class="tc" x="473" y="258" dominant-baseline="central">调度 → 存储 → 排障</text>
|
||||
|
||||
<line x1="216" y1="200" x2="250" y2="200" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="430" y1="200" x2="464" y2="200" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<rect x="46" y="320" width="170" height="160" rx="12" fill="#EAF3DE" stroke="#97C459" stroke-width="0.5"/>
|
||||
<rect x="46" y="320" width="170" height="32" rx="12" fill="#3B6D11"/>
|
||||
<rect x="46" y="340" width="170" height="12" fill="#3B6D11"/>
|
||||
<text class="tw" x="131" y="340" text-anchor="middle" dominant-baseline="central">存储体系</text>
|
||||
<text class="tc" x="55" y="368" dominant-baseline="central">2 篇笔记</text>
|
||||
<text class="ts" x="55" y="388" dominant-baseline="central">分布式文件系统选型</text>
|
||||
<text class="ts" x="55" y="408" dominant-baseline="central">训练数据流水线设计</text>
|
||||
<text class="tc" x="55" y="458" dominant-baseline="central">网络 → 性能</text>
|
||||
|
||||
<rect x="250" y="320" width="180" height="160" rx="12" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<rect x="250" y="320" width="180" height="32" rx="12" fill="#854F0B"/>
|
||||
<rect x="250" y="340" width="180" height="12" fill="#854F0B"/>
|
||||
<text class="tw" x="340" y="340" text-anchor="middle" dominant-baseline="central">监控可观测</text>
|
||||
<text class="tc" x="259" y="368" dominant-baseline="central">2 篇笔记</text>
|
||||
<text class="ts" x="259" y="388" dominant-baseline="central">DCGM 监控体系详解</text>
|
||||
<text class="ts" x="259" y="408" dominant-baseline="central">GPU 集群可观测性方案</text>
|
||||
<text class="tc" x="259" y="458" dominant-baseline="central">排障 → 性能</text>
|
||||
|
||||
<rect x="464" y="320" width="180" height="160" rx="12" fill="#FCEBEB" stroke="#F09595" stroke-width="0.5"/>
|
||||
<rect x="464" y="320" width="180" height="32" rx="12" fill="#A32D2D"/>
|
||||
<rect x="464" y="340" width="180" height="12" fill="#A32D2D"/>
|
||||
<text class="tw" x="554" y="340" text-anchor="middle" dominant-baseline="central">故障排查</text>
|
||||
<text class="tc" x="473" y="368" dominant-baseline="central">2 篇笔记</text>
|
||||
<text class="ts" x="473" y="388" dominant-baseline="central">GPU Xid 错误排查手册</text>
|
||||
<text class="ts" x="473" y="408" dominant-baseline="central">NCCL 通信故障诊断指南</text>
|
||||
<text class="tc" x="473" y="458" dominant-baseline="central">监控 → 硬件 → 自动化</text>
|
||||
|
||||
<line x1="216" y1="400" x2="250" y2="400" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="430" y1="400" x2="464" y2="400" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<line x1="131" y1="280" x2="131" y2="320" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="340" y1="280" x2="340" y2="320" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="554" y1="280" x2="554" y2="320" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<rect x="46" y="520" width="170" height="160" rx="12" fill="#FBEAF0" stroke="#ED93B1" stroke-width="0.5"/>
|
||||
<rect x="46" y="520" width="170" height="32" rx="12" fill="#993556"/>
|
||||
<rect x="46" y="540" width="170" height="12" fill="#993556"/>
|
||||
<text class="tw" x="131" y="540" text-anchor="middle" dominant-baseline="central">训练与推理</text>
|
||||
<text class="tc" x="55" y="568" dominant-baseline="central">2 篇笔记</text>
|
||||
<text class="ts" x="55" y="588" dominant-baseline="central">分布式训练框架对比</text>
|
||||
<text class="ts" x="55" y="608" dominant-baseline="central">PyTorch 分布式训练实战</text>
|
||||
<text class="tc" x="55" y="658" dominant-baseline="central">网络 → 性能 → 硬件</text>
|
||||
|
||||
<rect x="250" y="520" width="180" height="160" rx="12" fill="#FAECE7" stroke="#F0997B" stroke-width="0.5"/>
|
||||
<rect x="250" y="520" width="180" height="32" rx="12" fill="#993C1D"/>
|
||||
<rect x="250" y="540" width="180" height="12" fill="#993C1D"/>
|
||||
<text class="tw" x="340" y="540" text-anchor="middle" dominant-baseline="central">性能优化</text>
|
||||
<text class="tc" x="259" y="568" dominant-baseline="central">1 篇笔记</text>
|
||||
<text class="ts" x="259" y="588" dominant-baseline="central">GPU 集群性能调优指南</text>
|
||||
<text class="tc" x="259" y="658" dominant-baseline="central">硬件 → 存储 → 自动化</text>
|
||||
|
||||
<rect x="464" y="520" width="180" height="160" rx="12" fill="#F1EFE8" stroke="#888780" stroke-width="0.5"/>
|
||||
<rect x="464" y="520" width="180" height="32" rx="12" fill="#444441"/>
|
||||
<rect x="464" y="540" width="180" height="12" fill="#444441"/>
|
||||
<text class="tw" x="554" y="540" text-anchor="middle" dominant-baseline="central">运维自动化</text>
|
||||
<text class="tc" x="473" y="568" dominant-baseline="central">2 篇笔记</text>
|
||||
<text class="ts" x="473" y="588" dominant-baseline="central">GPU 驱动与固件管理</text>
|
||||
<text class="ts" x="473" y="608" dominant-baseline="central">集群自动化部署方案</text>
|
||||
<text class="tc" x="473" y="658" dominant-baseline="central">调度 → 排障 → 硬件</text>
|
||||
|
||||
<line x1="216" y1="600" x2="250" y2="600" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="430" y1="600" x2="464" y2="600" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<line x1="131" y1="480" x2="131" y2="520" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="340" y1="480" x2="340" y2="520" class="edge" marker-end="url(#arr)"/>
|
||||
<line x1="554" y1="480" x2="554" y2="520" class="edge" marker-end="url(#arr)"/>
|
||||
|
||||
<rect x="140" y="730" width="400" height="64" rx="12" fill="#F1EFE8" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<text class="th" x="230" y="756" text-anchor="middle">22</text>
|
||||
<text class="ts" x="230" y="776" text-anchor="middle">知识节点</text>
|
||||
|
||||
<line x1="280" y1="740" x2="280" y2="790" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<text class="th" x="340" y="756" text-anchor="middle">9</text>
|
||||
<text class="ts" x="340" y="776" text-anchor="middle">模块</text>
|
||||
|
||||
<line x1="400" y1="740" x2="400" y2="790" stroke="#D3D1C7" stroke-width="0.5"/>
|
||||
|
||||
<text class="th" x="450" y="756" text-anchor="middle">10,306</text>
|
||||
<text class="ts" x="450" y="776" text-anchor="middle">总行数</text>
|
||||
|
||||
<rect x="140" y="808" width="400" height="56" rx="8" fill="transparent" stroke="#D3D1C7" stroke-width="0.3"/>
|
||||
|
||||
<circle cx="170" cy="836" r="5" fill="#EEEDFE" stroke="#534AB7" stroke-width="0.5"/>
|
||||
<text class="tc" x="182" y="840" dominant-baseline="central">硬件基础 (3)</text>
|
||||
|
||||
<circle cx="280" cy="836" r="5" fill="#E6F1FB" stroke="#185FA5" stroke-width="0.5"/>
|
||||
<text class="tc" x="292" y="840" dominant-baseline="central">集群调度 (4)</text>
|
||||
|
||||
<circle cx="390" cy="836" r="5" fill="#E1F5EE" stroke="#0F6E56" stroke-width="0.5"/>
|
||||
<text class="tc" x="402" y="840" dominant-baseline="central">网络互联 (3)</text>
|
||||
|
||||
<circle cx="170" cy="856" r="5" fill="#EAF3DE" stroke="#3B6D11" stroke-width="0.5"/>
|
||||
<text class="tc" x="182" y="860" dominant-baseline="central">存储体系 (2)</text>
|
||||
|
||||
<circle cx="280" cy="856" r="5" fill="#FAEEDA" stroke="#854F0B" stroke-width="0.5"/>
|
||||
<text class="tc" x="292" y="860" dominant-baseline="central">监控可观测 (2)</text>
|
||||
|
||||
<circle cx="390" cy="856" r="5" fill="#FCEBEB" stroke="#A32D2D" stroke-width="0.5"/>
|
||||
<text class="tc" x="402" y="860" dominant-baseline="central">故障排查 (2)</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,122 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 680 600" width="100%" role="img">
|
||||
<title>LLM 训练三阶段流水线</title>
|
||||
<desc>预训练→SFT→RLHF：从万亿 token 的原始数据到对齐人类偏好的完整流程</desc>
|
||||
<style>
|
||||
.th { font-family: -apple-system, sans-serif; font-size: 13px; font-weight: 500; fill: #26215C; }
|
||||
.ts { font-family: -apple-system, sans-serif; font-size: 11px; fill: #5F5E5A; }
|
||||
.tm { font-family: -apple-system, monospace; font-size: 10px; fill: #26215C; }
|
||||
.tc { font-family: -apple-system, sans-serif; font-size: 9px; fill: #888780; }
|
||||
.tw { font-family: -apple-system, sans-serif; font-size: 10px; fill: #ffffff; }
|
||||
.tb { font-family: -apple-system, sans-serif; font-size: 11px; font-weight: 500; fill: #26215C; }
|
||||
</style>
|
||||
|
||||
<!-- ===== Phase 1: Pre-training ===== -->
|
||||
<rect x="20" y="20" width="640" height="160" rx="10" fill="#E6F1FB" stroke="#85B7EB" stroke-width="0.5"/>
|
||||
<rect x="20" y="20" width="640" height="30" rx="10" fill="#185FA5"/>
|
||||
<rect x="20" y="38" width="640" height="12" fill="#185FA5"/>
|
||||
<text class="tw" x="340" y="40" text-anchor="middle">阶段 1：预训练 (Pre-training) — 耗时：数周到数月</text>
|
||||
|
||||
<text class="ts" x="50" y="68">数据：</text>
|
||||
<rect x="100" y="56" width="120" height="22" rx="4" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tc" x="160" y="71" text-anchor="middle">网页 (Common Crawl)</text>
|
||||
<rect x="230" y="56" width="80" height="22" rx="4" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tc" x="270" y="71" text-anchor="middle">代码 (GitHub)</text>
|
||||
<rect x="320" y="56" width="80" height="22" rx="4" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tc" x="360" y="71" text-anchor="middle">书籍</text>
|
||||
<rect x="410" y="56" width="80" height="22" rx="4" fill="#ffffff" stroke="#85B7EB" stroke-width="0.3"/>
|
||||
<text class="tc" x="450" y="71" text-anchor="middle">论文</text>
|
||||
<text class="tc" x="510" y="71">≈ 数万亿 Token</text>
|
||||
|
||||
<text class="ts" x="50" y="94">目标：</text>
|
||||
<text class="tc" x="100" y="94">Next-Token Prediction — 给定前面的 token，预测下一个</text>
|
||||
<text class="tc" x="100" y="108">Loss = Cross-Entropy(预测, 真值) ｜ 优化器：AdamW / Muon</text>
|
||||
|
||||
<text class="ts" x="50" y="128">产出：</text>
|
||||
<rect x="100" y="116" width="160" height="28" rx="6" fill="#378ADD" stroke="#0C447C" stroke-width="0.5"/>
|
||||
<text class="tw" x="180" y="134" text-anchor="middle">Base Model（基座模型）</text>
|
||||
<text class="tc" x="270" y="134">会续写但不会对话，不懂指令</text>
|
||||
|
||||
<text class="ts" x="50" y="158">典型：</text>
|
||||
<text class="tc" x="100" y="158">LLaMA-7B: 1T tokens, ~26 天 (1024×A100) ｜ DeepSeek-V3: 14.8T tokens, ~2 月 (2048×H800)</text>
|
||||
|
||||
<!-- Arrow 1→2 -->
|
||||
<line x1="340" y1="180" x2="340" y2="206" stroke="#888780" stroke-width="1.5" marker-end="url(#arr)"/>
|
||||
|
||||
<!-- ===== Phase 2: SFT ===== -->
|
||||
<rect x="20" y="208" width="640" height="120" rx="10" fill="#FAEEDA" stroke="#EF9F27" stroke-width="0.5"/>
|
||||
<rect x="20" y="208" width="640" height="30" rx="10" fill="#854F0B"/>
|
||||
<rect x="20" y="226" width="640" height="12" fill="#854F0B"/>
|
||||
<text class="tw" x="340" y="228" text-anchor="middle">阶段 2：有监督微调 (SFT) — 耗时：几小时到几天</text>
|
||||
|
||||
<text class="ts" x="50" y="256">数据：</text>
|
||||
<rect x="100" y="244" width="200" height="22" rx="4" fill="#ffffff" stroke="#EF9F27" stroke-width="0.3"/>
|
||||
<text class="tc" x="200" y="259" text-anchor="middle">(指令 Instruction, 回答 Response) 对</text>
|
||||
<text class="tc" x="310" y="259">≈ 1 万 - 100 万 对</text>
|
||||
|
||||
<text class="ts" x="50" y="280">目标：</text>
|
||||
<text class="tc" x="100" y="280">仍然是 Next-Token Prediction，但 Loss 只在 Response 部分计算 ｜ 可全量微调或 LoRA</text>
|
||||
|
||||
<text class="ts" x="50" y="300">产出：</text>
|
||||
<rect x="100" y="288" width="160" height="28" rx="6" fill="#BA7517" stroke="#633806" stroke-width="0.5"/>
|
||||
<text class="tw" x="180" y="306" text-anchor="middle">Instruction Model</text>
|
||||
<text class="tc" x="270" y="306">会回答问题、遵循指令，但对齐不够好</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arr" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#888780" stroke-width="1.2" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<!-- Arrow 2→3 -->
|
||||
<line x1="340" y1="328" x2="340" y2="354" stroke="#888780" stroke-width="1.5" marker-end="url(#arr)"/>
|
||||
|
||||
<!-- ===== Phase 3: RLHF ===== -->
|
||||
<rect x="20" y="356" width="640" height="225" rx="10" fill="#EEEDFE" stroke="#AFA9EC" stroke-width="0.5"/>
|
||||
<rect x="20" y="356" width="640" height="30" rx="10" fill="#534AB7"/>
|
||||
<rect x="20" y="374" width="640" height="12" fill="#534AB7"/>
|
||||
<text class="tw" x="340" y="376" text-anchor="middle">阶段 3：人类偏好对齐 (RLHF/DPO/GRPO) — 耗时：几天到几周</text>
|
||||
|
||||
<!-- Sub-step 3a -->
|
||||
<rect x="40" y="396" width="180" height="80" rx="6" fill="#EEEDFE" stroke="#7F77DD" stroke-width="0.3"/>
|
||||
<text class="tb" x="130" y="414" text-anchor="middle">3a. 训练 Reward Model</text>
|
||||
<text class="tc" x="55" y="432">数据：(提问, 回答A, 回答B, A>B)</text>
|
||||
<text class="tc" x="55" y="448">训练目标：预测人类偏好</text>
|
||||
<text class="tc" x="55" y="464">Bradley-Terry 模型</text>
|
||||
|
||||
<line x1="220" y1="436" x2="248" y2="436" stroke="#888780" stroke-width="0.8" marker-end="url(#arr-small)"/>
|
||||
|
||||
<!-- Sub-step 3b -->
|
||||
<rect x="250" y="396" width="180" height="80" rx="6" fill="#EEEDFE" stroke="#7F77DD" stroke-width="0.3"/>
|
||||
<text class="tb" x="340" y="414" text-anchor="middle">3b. PPO / GRPO 训练</text>
|
||||
<text class="tc" x="265" y="432">Policy 生成回答</text>
|
||||
<text class="tc" x="265" y="448">Reward Model 打分</text>
|
||||
<text class="tc" x="265" y="464">GRPO 用组内相对优势</text>
|
||||
|
||||
<line x1="430" y1="436" x2="458" y2="436" stroke="#888780" stroke-width="0.8" marker-end="url(#arr-small)"/>
|
||||
|
||||
<!-- Sub-step 3c -->
|
||||
<rect x="460" y="396" width="180" height="80" rx="6" fill="#EEEDFE" stroke="#7F77DD" stroke-width="0.3"/>
|
||||
<text class="tb" x="550" y="414" text-anchor="middle">3c. 迭代 + DPO</text>
|
||||
<text class="tc" x="475" y="432">重复 3a-3b 多轮</text>
|
||||
<text class="tc" x="475" y="448">或用 DPO 直接优化</text>
|
||||
<text class="tc" x="475" y="464">(省去 Reward Model)</text>
|
||||
|
||||
<text class="ts" x="50" y="498">产出：</text>
|
||||
<rect x="100" y="486" width="180" height="28" rx="6" fill="#7F77DD" stroke="#3C3489" stroke-width="0.5"/>
|
||||
<text class="tw" x="190" y="504" text-anchor="middle">Aligned Model（对齐模型）</text>
|
||||
<text class="tc" x="290" y="504">更安全、更有用、更符合人类偏好</text>
|
||||
|
||||
<text class="ts" x="50" y="532">选择：</text>
|
||||
<text class="tc" x="100" y="532">PPO（经典，需 4 个模型加载）｜ GRPO（DeepSeek 提出，去 Critic，省一半显存）</text>
|
||||
<text class="tc" x="100" y="548">DPO（更简单，不需要 Reward Model，直接用偏好对训练）｜ OPD（DeepSeek-V4，多教师蒸馏替代 RL）</text>
|
||||
|
||||
<text class="ts" x="50" y="566">关键挑战：</text>
|
||||
<text class="tc" x="100" y="566">RL 训练不稳定（loss spike）｜ Reward Hacking（刷分不干正事）｜ 分布偏移（生成越来越偏）</text>
|
||||
|
||||
<defs>
|
||||
<marker id="arr-small" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="#888780" stroke-width="1" stroke-linecap="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.5 KiB |
Reference in New Issue
Block a user