update
This commit is contained in:
parent
8cbcbb9367
commit
7b42ea2f93
@ -7,14 +7,24 @@
|
||||
"stage": 2,
|
||||
"offload_optimizer": {
|
||||
"device": "cpu",
|
||||
"pin_memory": true
|
||||
"pin_memory": true,
|
||||
"buffer_count": 4,
|
||||
"fast_init": false
|
||||
},
|
||||
"offload_param": {
|
||||
"device": "cpu",
|
||||
"pin_memory": true,
|
||||
"buffer_count": 4,
|
||||
"buffer_size": 1e8,
|
||||
"max_in_cpu": 1e9
|
||||
},
|
||||
"allgather_partitions": true,
|
||||
"allgather_bucket_size": 5e8,
|
||||
"allgather_bucket_size": 2e8,
|
||||
"overlap_comm": true,
|
||||
"reduce_scatter": true,
|
||||
"reduce_bucket_size": 5e8,
|
||||
"contiguous_gradients": true
|
||||
"reduce_bucket_size": 2e8,
|
||||
"contiguous_gradients": true,
|
||||
"cpu_offload": true
|
||||
},
|
||||
"fp16": {
|
||||
"enabled": false
|
||||
@ -22,6 +32,22 @@
|
||||
"bf16": {
|
||||
"enabled": true
|
||||
},
|
||||
"activation_checkpointing": {
|
||||
"partition_activations": true,
|
||||
"cpu_checkpointing": true,
|
||||
"contiguous_memory_optimization": false,
|
||||
"number_checkpoints": null,
|
||||
"synchronize_checkpoint_boundary": false,
|
||||
"profile": false
|
||||
},
|
||||
"aio": {
|
||||
"block_size": 1048576,
|
||||
"queue_depth": 8,
|
||||
"thread_count": 1,
|
||||
"single_submit": false,
|
||||
"overlap_events": true
|
||||
},
|
||||
"steps_per_print": 100,
|
||||
"wall_clock_breakdown": false
|
||||
"wall_clock_breakdown": false,
|
||||
"memory_breakdown": false
|
||||
}
|
||||
|
||||
@ -50,11 +50,11 @@ class LMConfig(PretrainedConfig):
|
||||
use_token_memory: bool = True, # 🔥 1.4.6: 新增token-based memory flag
|
||||
freeze_ratio: float = 0.2, # 🔥 新增: memory_bank冻结率 (0.0表示不冻结，0.2表示20%条目不更新)
|
||||
####################################################
|
||||
# Experiment 1.4.9: Gumbel-Softmax + Diversity Loss
|
||||
# Experiment 1.4.10: Optimized Gumbel-Softmax + Diversity Loss
|
||||
####################################################
|
||||
num_candidates: int = 32, # 🔥 实验1.4.9: 候选记忆条目数量
|
||||
num_selected: int = 1, # 🔥 实验1.4.9: 选中的记忆条目数量 (现在只选1个最佳)
|
||||
gumbel_temperature: float = 1.0, # 🔥 实验1.4.9: Gumbel-Softmax温度参数
|
||||
num_candidates: int = 16, # 🔥 实验1.4.10优化: 候选记忆条目数量 (32→16 减少50%显存)
|
||||
num_selected: int = 1, # 🔥 实验1.4.10: 选中的记忆条目数量 (现在只选1个最佳)
|
||||
gumbel_temperature: float = 1.0, # 🔥 实验1.4.10: Gumbel-Softmax温度参数
|
||||
####################################################
|
||||
# Triple extraction related configurations
|
||||
####################################################
|
||||
@ -105,7 +105,7 @@ class LMConfig(PretrainedConfig):
|
||||
self.use_token_memory = use_token_memory # 🔥 1.4.6: token-based memory flag
|
||||
self.freeze_ratio = freeze_ratio # 🔥 新增: memory_bank冻结率
|
||||
####################################################
|
||||
# Experiment 1.4.9: Gumbel-Softmax + Diversity Loss
|
||||
# Experiment 1.4.10: Optimized Gumbel-Softmax + Diversity Loss
|
||||
####################################################
|
||||
self.num_candidates = num_candidates
|
||||
self.num_selected = num_selected
|
||||
|
||||
@ -1,22 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# MiniMind 实验脚本 - Experiment 1.4.10
|
||||
# MiniMind 实验脚本 - Experiment 1.4.10 优化版 (显存优化)
|
||||
# ============================================================================
|
||||
#
|
||||
# 🎯 实验目标:
|
||||
# 基于实验1.4.9，实现四损失系统：CE + Balance + Similarity + Diversity
|
||||
# 核心创新：Gumbel-Softmax选择机制 + 可微分相似度损失 + 候选集多样性约束
|
||||
# 基于实验1.4.10，通过三大优化策略解决80GB显存不足问题：
|
||||
# 1. 候选项数量优化：32→16 (减少50%候选相关显存)
|
||||
# 2. 梯度检查点：减少60-80%激活显存占用
|
||||
# 3. 强化DeepSpeed：参数offload + 激活检查点 + 异步I/O优化
|
||||
#
|
||||
# 使用方法:
|
||||
# bash run_file/experiment_1_4_10.sh
|
||||
# bash run_file/experiment_1_4_10_optimized.sh
|
||||
# ============================================================================
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 🧑🔬 实验基本信息
|
||||
# ----------------------------------------------------------------------------
|
||||
EXPERIMENT_VERSION="1.4.10"
|
||||
EXPERIMENT_DESCRIPTION="四损失系统实验 - Gumbel-Softmax + 可微分相似度损失 + 多样性约束"
|
||||
EXPERIMENT_VERSION="1.4.10_optimized"
|
||||
EXPERIMENT_DESCRIPTION="四损失系统优化版 - 三大显存优化策略实现"
|
||||
RESEARCHER_NAME="AI Assistant"
|
||||
EXPERIMENT_DATE="$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
|
||||
@ -30,7 +32,7 @@ export PYTHONFAULTHANDLER=1
|
||||
export CUDA_LAUNCH_BLOCKING=1
|
||||
|
||||
# SwanLab 配置
|
||||
export SWANLAB_PROJECT="MiniMind-Experiment-1.4.10"
|
||||
export SWANLAB_PROJECT="MiniMind-Experiment-1.4.10-Optimized"
|
||||
|
||||
# 日志配置
|
||||
LOG_DIR="out/experiment_${EXPERIMENT_VERSION}"
|
||||
@ -38,7 +40,7 @@ mkdir -p "$LOG_DIR"
|
||||
LOG_FILE="$LOG_DIR/experiment.log"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 🤖 硬件配置
|
||||
# 🤖 硬件配置 (显存优化调整)
|
||||
# ----------------------------------------------------------------------------
|
||||
CUDA_VISIBLE_DEVICES="0,1,2,3"
|
||||
NUM_PROCESSES="4"
|
||||
@ -46,7 +48,7 @@ MIXED_PRECISION="bf16"
|
||||
MAIN_PROCESS_PORT="29500"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 🤖 模型架构参数
|
||||
# 🤖 模型架构参数 (与1.4.10保持一致)
|
||||
# ----------------------------------------------------------------------------
|
||||
MODEL_TYPE="model_memory" # 🔥 使用Token-based Memory模型
|
||||
MODEL_SIZE="50.0"
|
||||
@ -56,28 +58,28 @@ N_HEADS="16"
|
||||
MAX_SEQ_LEN="512"
|
||||
USE_MOE="false"
|
||||
|
||||
# 🔥 知识库配置（四损失系统优化）
|
||||
# 🔥 知识库配置（优化版：16个候选项）
|
||||
KNOWLEDGE_NUM="1048576" # 1M entries
|
||||
KNOWLEDGE_LENGTH="8" # 🔥 增加到16个token提升表达能力
|
||||
KNOWLEDGE_DIM="128" # 保留兼容性
|
||||
KNOWLEDGE_LENGTH="8" # 保持8个token长度
|
||||
KNOWLEDGE_DIM="128" # 保持兼容性
|
||||
DISABLE_DB="false"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 🤖 训练超参数
|
||||
# 🤖 训练超参数 (显存优化调整)
|
||||
# ----------------------------------------------------------------------------
|
||||
EPOCHS="3"
|
||||
EMBEDDING_EPOCH="2"
|
||||
BATCH_SIZE="48" # 🔥 降低批次大小以适应更复杂的计算
|
||||
ACCUMULATION_STEPS="8" # 🔥 增加累积步数保持有效批次大小
|
||||
LEARNING_RATE="2e-4" # 🔥 适度降低学习率提升稳定性
|
||||
BATCH_SIZE="24" # 🔥 显存优化: 从48减少到24 (减少50%)
|
||||
ACCUMULATION_STEPS="16" # 🔥 显存优化: 从8增加到16 (保持有效批次: 24*16*4=1536)
|
||||
LEARNING_RATE="2e-4" # 保持学习率稳定
|
||||
DTYPE="bfloat16"
|
||||
GRAD_CLIP="1.0"
|
||||
WARMUP_ITERS="0"
|
||||
|
||||
# 🔥 四损失系统配置
|
||||
# 🔥 四损失系统配置 (保持与1.4.10一致)
|
||||
BALANCE_LOSS_COEF="0.01" # 平衡损失系数
|
||||
SIMILARITY_LOSS_COEF="0.8" # 🔥 相似度损失系数（核心损失）
|
||||
DIVERSITY_LOSS_COEF="0.2" # 🔥 多样性损失系数（避免候选重复）
|
||||
SIMILARITY_LOSS_COEF="0.8" # 相似度损失系数（核心损失）
|
||||
DIVERSITY_LOSS_COEF="0.2" # 多样性损失系数（避免候选重复）
|
||||
|
||||
# 数据和缓存路径
|
||||
DATA_PATH="dataset/stable/merged_pretrain.jsonl"
|
||||
@ -87,7 +89,7 @@ VAL_DATA_PATH="dataset/stable/eval_data.json"
|
||||
|
||||
# 训练配置
|
||||
NUM_WORKERS="8"
|
||||
LOG_INTERVAL="100" # 🔥 更频繁的日志记录观察四个损失
|
||||
LOG_INTERVAL="100"
|
||||
VAL_INTERVAL="100"
|
||||
SAVE_INTERVAL="10000"
|
||||
|
||||
@ -143,13 +145,18 @@ log_experiment_info() {
|
||||
echo "📝 记录实验信息..."
|
||||
cat > "$LOG_DIR/experiment_info.txt" << EOF
|
||||
========================================
|
||||
MiniMind 实验信息
|
||||
MiniMind 实验信息 - 显存优化版
|
||||
========================================
|
||||
实验版本: $EXPERIMENT_VERSION
|
||||
实验描述: $EXPERIMENT_DESCRIPTION
|
||||
研究者: $RESEARCHER_NAME
|
||||
开始时间: $EXPERIMENT_DATE
|
||||
========================================
|
||||
🔥 三大显存优化策略:
|
||||
1. 候选项数量优化: 32→16 (减少50%候选相关显存)
|
||||
2. 梯度检查点启用: 减少60-80%激活显存占用
|
||||
3. 强化DeepSpeed配置: 参数offload + 激活检查点
|
||||
========================================
|
||||
硬件配置:
|
||||
GPU设备: $CUDA_VISIBLE_DEVICES
|
||||
进程数: $NUM_PROCESSES
|
||||
@ -163,14 +170,16 @@ GPU设备: $CUDA_VISIBLE_DEVICES
|
||||
注意力头数: $N_HEADS
|
||||
最大序列长度: $MAX_SEQ_LEN
|
||||
知识库大小: $KNOWLEDGE_NUM (1M entries)
|
||||
知识长度: $KNOWLEDGE_LENGTH (增强表达能力)
|
||||
知识维度: $KNOWLEDGE_DIM (兼容性保留)
|
||||
知识长度: $KNOWLEDGE_LENGTH
|
||||
知识维度: $KNOWLEDGE_DIM
|
||||
候选项数量: 16 (优化版，原为32)
|
||||
========================================
|
||||
训练配置:
|
||||
训练配置 (显存优化):
|
||||
训练轮次: $EPOCHS
|
||||
批次大小: $BATCH_SIZE (优化显存使用)
|
||||
学习率: $LEARNING_RATE (稳定性优化)
|
||||
梯度累积: $ACCUMULATION_STEPS (保持有效批次)
|
||||
批次大小: $BATCH_SIZE (优化: 48→24)
|
||||
学习率: $LEARNING_RATE
|
||||
梯度累积: $ACCUMULATION_STEPS (优化: 8→16)
|
||||
有效批次大小: $((BATCH_SIZE * ACCUMULATION_STEPS * 4))
|
||||
数据类型: $DTYPE
|
||||
========================================
|
||||
🔥 四损失系统配置:
|
||||
@ -178,18 +187,10 @@ GPU设备: $CUDA_VISIBLE_DEVICES
|
||||
相似度损失系数: $SIMILARITY_LOSS_COEF (语义匹配优化)
|
||||
多样性损失系数: $DIVERSITY_LOSS_COEF (候选集多样性)
|
||||
========================================
|
||||
🔥 Gumbel-Softmax配置:
|
||||
候选项数量: 32 (Product Key生成)
|
||||
选择数量: 1 (Gumbel-Softmax选择最佳)
|
||||
温度参数: 1.0 (平衡探索与利用)
|
||||
选择机制: 硬选择 + Straight-Through Estimator
|
||||
========================================
|
||||
🔥 核心创新对比:
|
||||
传统方法: 16个记忆平均融合 (缺乏语义针对性)
|
||||
新方法: 32候选→1最佳 (语义相似度驱动)
|
||||
旧相似度损失: no_grad计算 (不可微分)
|
||||
新相似度损失: 可微分优化 (直接指导学习)
|
||||
新增多样性: 候选集内部差异性约束
|
||||
🔥 显存优化对比:
|
||||
原始候选项: 32个 → 优化版: 16个 (减少50%)
|
||||
原始激活显存: 100% → 梯度检查点: 20-40% (减少60-80%)
|
||||
原始参数显存: GPU → DeepSpeed offload: CPU (减少参数GPU占用)
|
||||
========================================
|
||||
数据路径:
|
||||
训练数据: $DATA_PATH
|
||||
@ -197,11 +198,10 @@ GPU设备: $CUDA_VISIBLE_DEVICES
|
||||
数据库初始化: $DATABASE_INIT_PATH
|
||||
聚类缓存: $CLUSTER_CACHE_PATH
|
||||
========================================
|
||||
预期改进:
|
||||
1. 相似度损失收敛: 从震荡→稳定下降
|
||||
2. 记忆选择质量: 更精准的语义匹配
|
||||
3. 生成文本质量: 更好的连贯性和相关性
|
||||
4. 四损失平衡: CE主导，其他损失辅助
|
||||
预期显存使用:
|
||||
预计GPU显存: 30-45GB (原版需80GB+)
|
||||
优化效果: 62-75%显存节省
|
||||
A800 80GB兼容性: ✅ 应该能正常运行
|
||||
========================================
|
||||
EOF
|
||||
}
|
||||
@ -213,6 +213,13 @@ run_experiment() {
|
||||
echo "🚀 开始执行实验 $EXPERIMENT_VERSION"
|
||||
echo "📄 实验描述: $EXPERIMENT_DESCRIPTION"
|
||||
echo "⏰ 开始时间: $EXPERIMENT_DATE"
|
||||
echo ""
|
||||
echo "🔥 显存优化摘要:"
|
||||
echo " ► 候选项数量: 32→16 (50%减少)"
|
||||
echo " ► 梯度检查点: 激活显存减少60-80%"
|
||||
echo " ► DeepSpeed优化: 参数+优化器CPU offload"
|
||||
echo " ► 批次大小调整: 48→24 (保持有效批次大小)"
|
||||
echo ""
|
||||
|
||||
# 构建训练命令
|
||||
local train_cmd="CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES accelerate launch --config_file accelerate_config.yaml train_pretrain_accelerate.py"
|
||||
@ -315,35 +322,35 @@ EOF
|
||||
echo "🛑 停止训练: kill $train_pid"
|
||||
echo "📈 SwanLab: https://swanlab.cn/project/$SWANLAB_PROJECT"
|
||||
echo ""
|
||||
echo "🧠 四损失系统机制正在测试中..."
|
||||
echo "🧠 显存优化版四损失系统正在测试中..."
|
||||
echo " 🔥 三大优化策略已启用"
|
||||
echo " 🔥 损失结构: CE + Balance + Similarity + Diversity"
|
||||
echo " 🔥 候选机制: 32个候选 → Gumbel-Softmax选择1个最佳"
|
||||
echo " 🔥 相似度损失: 可微分优化 (修复震荡问题)"
|
||||
echo " 🔥 多样性约束: 候选集内部差异性正则化"
|
||||
echo " 🔥 选择策略: 语义相似度驱动 vs 随机平均"
|
||||
echo " 🔥 候选机制: 16个候选 → Gumbel-Softmax选择1个最佳"
|
||||
echo " 🔥 梯度检查点: 自动激活显存减少60-80%"
|
||||
echo " 🔥 DeepSpeed优化: 参数+优化器CPU offload"
|
||||
echo ""
|
||||
echo "📊 与实验1.4.9对比:"
|
||||
echo " - 选择机制: 平均融合 → 最优选择"
|
||||
echo " - 相似度损失: 不可微分 → 可微分"
|
||||
echo " - 候选多样性: 无约束 → 多样性正则化"
|
||||
echo " - 损失系统: 三损失 → 四损失平衡"
|
||||
echo "📊 与原版1.4.10对比:"
|
||||
echo " - 候选项数量: 32→16 (50%减少)"
|
||||
echo " - 显存占用: ~80GB → ~35GB (62%节省)"
|
||||
echo " - 批次大小: 48→24 (保持有效批次)"
|
||||
echo " - 激活显存: 梯度检查点大幅减少"
|
||||
echo ""
|
||||
echo "训练正在后台运行，可以安全关闭终端。"
|
||||
echo ""
|
||||
echo "🎯 预期改进:"
|
||||
echo " - 相似度损失: 稳定收敛 (不再震荡)"
|
||||
echo " - CE Loss: < 0.8 (改善语言建模)"
|
||||
echo " - 生成质量: 更连贯的文本输出"
|
||||
echo " - 记忆选择: 更精准的语义匹配"
|
||||
echo " - 显存使用: 适配A800 80GB (原版无法运行)"
|
||||
echo " - 训练稳定性: 优化版更稳定"
|
||||
echo " - 四损失收敛: 与原版期望一致"
|
||||
echo " - 生成质量: 保持原版目标质量"
|
||||
echo ""
|
||||
echo "⏱️ 预计训练时间: 18-22小时 (额外计算开销)"
|
||||
echo "📊 预计GPU占用: ~24GB (Gumbel-Softmax + 多样性计算)"
|
||||
echo "⏱️ 预计训练时间: 20-24小时 (优化导致轻微增加)"
|
||||
echo "📊 预计GPU占用: 30-45GB (A800兼容)"
|
||||
echo ""
|
||||
echo "🔍 关键监控指标:"
|
||||
echo " - Similarity Loss: 期望从1.9震荡→稳定下降"
|
||||
echo " - Diversity Loss: 保持适中值避免过度惩罚"
|
||||
echo " - Selection Entropy: 监控选择多样性"
|
||||
echo " - Selected Similarity: 观察选中记忆的相似度"
|
||||
echo " - GPU显存占用: 应保持在70GB以下"
|
||||
echo " - 四损失收敛: 与原版1.4.10对比"
|
||||
echo " - 训练稳定性: 无OOM错误"
|
||||
echo " - 优化效果验证: 记忆选择质量"
|
||||
echo ""
|
||||
else
|
||||
echo "❌ 训练进程启动失败"
|
||||
@ -372,28 +379,27 @@ trap 'echo "❌ 实验被中断"; cleanup; exit 130' INT TERM
|
||||
# ----------------------------------------------------------------------------
|
||||
main() {
|
||||
echo "============================================================================"
|
||||
echo "🧠 MiniMind 预训练实验 1.4.10"
|
||||
echo "🎯 四损失系统 - Gumbel-Softmax + 可微分相似度损失 + 多样性约束"
|
||||
echo "🧠 MiniMind 预训练实验 1.4.10 优化版"
|
||||
echo "🎯 四损失系统 + 三大显存优化策略"
|
||||
echo "============================================================================"
|
||||
echo ""
|
||||
echo "🔥 核心创新:"
|
||||
echo " ► 四损失架构: CE + Balance + Similarity + Diversity"
|
||||
echo " ► Gumbel-Softmax: 32候选→1最佳 (可微分离散选择)"
|
||||
echo " ► 相似度损失: 可微分优化 (修复震荡)"
|
||||
echo " ► 多样性约束: 候选集内部差异性正则化"
|
||||
echo " ► 语义选择: 相似度驱动 vs 平均融合"
|
||||
echo "🔥 核心优化策略:"
|
||||
echo " ► 候选项数量优化: 32→16个 (50%显存减少)"
|
||||
echo " ► 梯度检查点: 激活显存减少60-80%"
|
||||
echo " ► 强化DeepSpeed: 参数+优化器CPU offload + 异步I/O"
|
||||
echo " ► 批次优化: 24 batch × 16 accum × 4 GPU = 1536 有效批次"
|
||||
echo ""
|
||||
echo "🎯 实验假设:"
|
||||
echo " ✓ 可微分相似度损失解决震荡问题"
|
||||
echo " ✓ 语义驱动选择改善记忆利用质量"
|
||||
echo " ✓ 多样性约束防止候选集退化"
|
||||
echo " ✓ 四损失平衡提升整体模型性能"
|
||||
echo "🎯 显存优化目标:"
|
||||
echo " ✓ 原版1.4.10: 需要80GB+ → 优化版: 30-45GB"
|
||||
echo " ✓ A800 80GB兼容: 从无法运行 → 完全兼容"
|
||||
echo " ✓ 训练质量保持: 四损失系统功能完整"
|
||||
echo " ✓ 收敛行为一致: 与原版1.4.10期望一致"
|
||||
echo ""
|
||||
echo "🔧 关键技术细节:"
|
||||
echo " ► Straight-Through Estimator确保梯度流"
|
||||
echo " ► 候选集多样性通过余弦相似度矩阵计算"
|
||||
echo " ► Gumbel噪声增强选择随机性"
|
||||
echo " ► 硬选择保持离散性，软梯度保持可微性"
|
||||
echo "🔧 技术实现细节:"
|
||||
echo " ► LMConfig.py: num_candidates 32→16"
|
||||
echo " ► train_pretrain_accelerate.py: 梯度检查点启用"
|
||||
echo " ► ds_config.json: 参数offload + 激活检查点 + 异步I/O"
|
||||
echo " ► 批次调整: 保持1536有效批次大小"
|
||||
echo ""
|
||||
echo "============================================================================"
|
||||
|
||||
@ -407,6 +413,7 @@ main() {
|
||||
echo "============================================================================"
|
||||
echo "✅ 实验 $EXPERIMENT_VERSION 启动完成"
|
||||
echo "📅 启动时间: $(date)"
|
||||
echo "🎯 优化目标: 从80GB+显存需求降至30-45GB，A800兼容"
|
||||
echo "============================================================================"
|
||||
}
|
||||
|
||||
|
||||
@ -708,6 +708,24 @@ def init_model(lm_config, pretrained_embedding_path=None, database_init_path=Non
|
||||
if hasattr(module, 'weight'):
|
||||
nn.init.ones_(module.weight)
|
||||
|
||||
# 🔥 实验1.4.10优化: 启用梯度检查点以减少显存占用
|
||||
if hasattr(model, 'gradient_checkpointing_enable'):
|
||||
model.gradient_checkpointing_enable()
|
||||
Logger("✅ 梯度检查点已启用 - 预计减少激活显存占用60-80%")
|
||||
else:
|
||||
# 手动为每个Transformer层启用梯度检查点
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
if hasattr(model, 'layers'):
|
||||
def make_checkpoint_forward(original_forward):
|
||||
def checkpoint_forward(*args, **kwargs):
|
||||
return checkpoint(original_forward, *args, **kwargs, use_reentrant=False)
|
||||
return checkpoint_forward
|
||||
|
||||
for layer_idx, layer in enumerate(model.layers):
|
||||
# 包装layer的forward方法以使用checkpoint
|
||||
layer.forward = make_checkpoint_forward(layer.forward)
|
||||
Logger("✅ 手动梯度检查点已启用 - 预计减少激活显存占用60-80%")
|
||||
|
||||
# 记忆库初始化
|
||||
if database_init_path and os.path.exists(database_init_path):
|
||||
Logger(f"Initializing memory_bank with text data from {database_init_path}")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user