124 lines
4.4 KiB
JSON
124 lines
4.4 KiB
JSON
{
|
|
"version": "0.2.0",
|
|
"configurations": [
|
|
{
|
|
"name": "MiniMind Training (Direct Python)",
|
|
"type": "python",
|
|
"request": "launch",
|
|
"program": "${workspaceFolder}/train_pretrain_accelerate.py",
|
|
"args": [
|
|
"--out_dir", "out",
|
|
"--epochs", "3",
|
|
"--embedding_epoch", "2",
|
|
"--batch_size", "128",
|
|
"--learning_rate", "8e-5",
|
|
"--dtype", "bfloat16",
|
|
"--use_swanlab",
|
|
"--swanlab_project", "MiniMind-Pretrain",
|
|
"--num_workers", "1",
|
|
"--accumulation_steps", "16",
|
|
"--grad_clip", "0.5",
|
|
"--warmup_iters", "0",
|
|
"--log_interval", "1",
|
|
"--save_interval", "10000",
|
|
"--dim", "512",
|
|
"--n_layers", "8",
|
|
"--max_seq_len", "512",
|
|
"--data_path", "./dataset/stable/merged_pretrain.jsonl",
|
|
"--profile",
|
|
"--profile_interval", "10",
|
|
"--use_flash_attn",
|
|
"--knowledge_num", "1048576",
|
|
"--knowledge_length", "32",
|
|
"--database_init_path", "./dataset/stable/sentence_trex_data.json",
|
|
"--fast_clustering",
|
|
"--cluster_cache_path", "./cache/cluster_tokens_single.pt",
|
|
"--memory_monitor_interval", "10",
|
|
"--model_type", "model",
|
|
"--model_size", "538"
|
|
],
|
|
"env": {
|
|
"CUDA_VISIBLE_DEVICES": "0",
|
|
"NCCL_DEBUG": "INFO",
|
|
"PYTHONFAULTHANDLER": "1"
|
|
},
|
|
"cwd": "${workspaceFolder}",
|
|
"console": "integratedTerminal",
|
|
"justMyCode": false,
|
|
"stopOnEntry": false,
|
|
"python": "${workspaceFolder}/.venv/bin/python"
|
|
},
|
|
{
|
|
"name": "MiniMind Training (Direct Python - Simple)",
|
|
"type": "python",
|
|
"request": "launch",
|
|
"program": "${workspaceFolder}/train_pretrain_accelerate.py",
|
|
"args": [
|
|
"--epochs", "1",
|
|
"--batch_size", "32",
|
|
"--learning_rate", "1e-4",
|
|
"--log_interval", "10",
|
|
"--profile_interval", "2",
|
|
"--model_type", "model_original"
|
|
],
|
|
"env": {
|
|
"CUDA_VISIBLE_DEVICES": "0"
|
|
},
|
|
"cwd": "${workspaceFolder}",
|
|
"console": "integratedTerminal",
|
|
"justMyCode": false,
|
|
"stopOnEntry": false,
|
|
"python": "${workspaceFolder}/.venv/bin/python"
|
|
},
|
|
{
|
|
"name": "MiniMind Test (Direct Python)",
|
|
"type": "python",
|
|
"request": "launch",
|
|
"program": "${workspaceFolder}/test.py",
|
|
"env": {
|
|
"CUDA_VISIBLE_DEVICES": "0"
|
|
},
|
|
"cwd": "${workspaceFolder}",
|
|
"console": "integratedTerminal",
|
|
"justMyCode": false,
|
|
"python": "${workspaceFolder}/.venv/bin/python"
|
|
},
|
|
{
|
|
"name": "MiniMind Training Debug (Accelerate)",
|
|
"type": "python",
|
|
"request": "launch",
|
|
"module": "accelerate.commands.launch",
|
|
"args": [
|
|
"--num_processes=1",
|
|
"--mixed_precision=bf16",
|
|
"${workspaceFolder}/train_pretrain_accelerate.py",
|
|
"--epochs", "1",
|
|
"--batch_size", "32",
|
|
"--learning_rate", "1e-4",
|
|
"--log_interval", "10",
|
|
"--profile_interval", "2",
|
|
"--model_type", "model_original"
|
|
],
|
|
"env": {
|
|
"CUDA_VISIBLE_DEVICES": "0"
|
|
},
|
|
"cwd": "${workspaceFolder}",
|
|
"console": "integratedTerminal",
|
|
"justMyCode": false,
|
|
"stopOnEntry": false,
|
|
"python": "${workspaceFolder}/.venv/bin/python"
|
|
},
|
|
{
|
|
"name": "MiniMind Test Only",
|
|
"type": "python",
|
|
"request": "launch",
|
|
"program": "${workspaceFolder}/test.py",
|
|
"env": {
|
|
"CUDA_VISIBLE_DEVICES": "0"
|
|
},
|
|
"cwd": "${workspaceFolder}",
|
|
"console": "integratedTerminal",
|
|
"justMyCode": false
|
|
}
|
|
]
|
|
} |