root@22318190fe62:~/axolotl# pwd
/home/axolotl
root@22318190fe62:~/axolotl# accelerate launch -m axolotl.cli.train examples/tiny-llama/lora.yml
The following values were not passed to `accelerate launch` and had defaults used instead:
        `--num_processes` was set to a value of `1`
        `--num_machines` was set to a value of `1`
        `--mixed_precision` was set to a value of `'no'`
        `--dynamo_backend` was set to a value of `'no'`
To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
WARNING: BNB_CUDA_VERSION=118 environment variable detected; loading libbitsandbytes_cuda118.so.
This can be used to load a bitsandbytes version that is different from the PyTorch CUDA version.
If this was unintended set the BNB_CUDA_VERSION variable to an empty string: export BNB_CUDA_VERSION=
If you use the manual override make sure the right libcudart.so is in your LD_LIBRARY_PATH
For example by adding the following to your .bashrc: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:<path_to_cuda_dir/lib64

[2024-05-25 11:14:41,377] [INFO] [datasets.<module>:58] [PID:96] PyTorch version 2.1.2+cu118 available.
[2024-05-25 11:14:42,465] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
df: /home/.triton/autotune: No such file or directory
[2024-05-25 11:14:42,539] [INFO] [root.spawn:38] [PID:96] gcc -pthread -B /root/miniconda3/envs/py3.10/compiler_compat -Wno-unused-result -Wsign-compare -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/miniconda3/envs/py3.10/include -fPIC -O2 -isystem /root/miniconda3/envs/py3.10/include -fPIC -c /tmp/tmp4lzmy1iw/test.c -o /tmp/tmp4lzmy1iw/test.o
[2024-05-25 11:14:42,561] [INFO] [root.spawn:38] [PID:96] gcc -pthread -B /root/miniconda3/envs/py3.10/compiler_compat /tmp/tmp4lzmy1iw/test.o -laio -o /tmp/tmp4lzmy1iw/a.out
 [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
 [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
 [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
[2024-05-25 11:14:44,099] [DEBUG] [axolotl.normalize_config:79] [PID:96] [RANK:0] bf16 support detected, enabling for this configuration.
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
config.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 560/560 [00:00<00:00, 3.09MB/s]
[2024-05-25 11:14:44,621] [INFO] [axolotl.normalize_config:182] [PID:96] [RANK:0] GPU memory usage baseline: 0.000GB (+0.313GB misc)
                                 dP            dP   dP 
                                 88            88   88 
      .d8888b. dP.  .dP .d8888b. 88 .d8888b. d8888P 88 
      88'  `88  `8bd8'  88'  `88 88 88'  `88   88   88 
      88.  .88  .d88b.  88.  .88 88 88.  .88   88   88 
      `88888P8 dP'  `dP `88888P' dP `88888P'   dP   dP 
                                                       
                                                       

****************************************
**** Axolotl Dependency Versions *****
  accelerate: 0.30.1         
        peft: 0.10.0         
transformers: 4.40.2         
         trl: 0.8.5          
       torch: 2.1.2+cu118    
bitsandbytes: 0.43.1         
****************************************
[2024-05-25 11:14:44,657] [WARNING] [axolotl.scripts.check_user_token:487] [PID:96] [RANK:0] Error verifying HuggingFace token. Remember to log in using `huggingface-cli login` and get your access token from https://huggingface.co/settings/tokens if you want to use gated models or datasets.
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
tokenizer_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 776/776 [00:00<00:00, 2.61MB/s]
tokenizer.model: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 500k/500k [00:00<00:00, 15.9MB/s]
special_tokens_map.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 414/414 [00:00<00:00, 1.41MB/s]
tokenizer.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.84M/1.84M [00:00<00:00, 8.50MB/s]
[2024-05-25 11:14:47,333] [DEBUG] [axolotl.load_tokenizer:280] [PID:96] [RANK:0] EOS: 2 / </s>
[2024-05-25 11:14:47,333] [DEBUG] [axolotl.load_tokenizer:281] [PID:96] [RANK:0] BOS: 1 / <s>
[2024-05-25 11:14:47,334] [DEBUG] [axolotl.load_tokenizer:282] [PID:96] [RANK:0] PAD: 2 / </s>
[2024-05-25 11:14:47,334] [DEBUG] [axolotl.load_tokenizer:283] [PID:96] [RANK:0] UNK: 0 / <unk>
[2024-05-25 11:14:47,334] [INFO] [axolotl.load_tokenizer:294] [PID:96] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.
[2024-05-25 11:14:47,334] [INFO] [axolotl.load_tokenized_prepared_datasets:183] [PID:96] [RANK:0] Unable to find prepared dataset in last_run_prepared/b679ea8e13fdec9db52fe0332ca58c81
[2024-05-25 11:14:47,334] [INFO] [axolotl.load_tokenized_prepared_datasets:184] [PID:96] [RANK:0] Loading raw datasets...
[2024-05-25 11:14:47,334] [WARNING] [axolotl.load_tokenized_prepared_datasets:186] [PID:96] [RANK:0] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset.
[2024-05-25 11:14:47,334] [INFO] [axolotl.load_tokenized_prepared_datasets:193] [PID:96] [RANK:0] No seed provided, using default seed of 42
Downloading readme: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 28.0/28.0 [00:00<00:00, 93.1kB/s]
Downloading data: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.76M/1.76M [00:00<00:00, 5.00MB/s]
Generating train split: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:00<00:00, 62978.96 examples/s]
Tokenizing Prompts (num_proc=64): 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:01<00:00, 1451.75 examples/s]
[2024-05-25 11:14:59,020] [INFO] [axolotl.load_tokenized_prepared_datasets:410] [PID:96] [RANK:0] merging datasets
Dropping Long Sequences (num_proc=64): 100%|█████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:00<00:00, 4458.76 examples/s]
Add position_id column (Sample Packing) (num_proc=64): 100%|█████████████████████████████████████████████████████████████████████████| 2000/2000 [00:00<00:00, 2937.27 examples/s]
[2024-05-25 11:15:02,316] [INFO] [axolotl.load_tokenized_prepared_datasets:423] [PID:96] [RANK:0] Saving merged prepared dataset to disk... last_run_prepared/b679ea8e13fdec9db52fe0332ca58c81
Saving the dataset (1/1 shards): 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:00<00:00, 71343.83 examples/s]
[2024-05-25 11:15:02,368] [DEBUG] [axolotl.calculate_total_num_steps:299] [PID:96] [RANK:0] total_num_tokens: 414_041
[2024-05-25 11:15:02,386] [DEBUG] [axolotl.calculate_total_num_steps:312] [PID:96] [RANK:0] `total_supervised_tokens: 294_246`
[2024-05-25 11:15:07,864] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 1.0 total_num_tokens per device: 414041
[2024-05-25 11:15:07,864] [DEBUG] [axolotl.calculate_total_num_steps:365] [PID:96] [RANK:0] data_loader_len: 12
[2024-05-25 11:15:07,864] [INFO] [axolotl.calc_sample_packing_eff_est:371] [PID:96] [RANK:0] sample_packing_eff_est across ranks: [0.9719637357271634]
[2024-05-25 11:15:07,864] [DEBUG] [axolotl.calculate_total_num_steps:383] [PID:96] [RANK:0] sample_packing_eff_est: 0.98
[2024-05-25 11:15:07,864] [DEBUG] [axolotl.calculate_total_num_steps:391] [PID:96] [RANK:0] total_num_steps: 48
[2024-05-25 11:15:07,873] [DEBUG] [axolotl.train.train:56] [PID:96] [RANK:0] loading tokenizer... TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
[2024-05-25 11:15:08,464] [DEBUG] [axolotl.load_tokenizer:280] [PID:96] [RANK:0] EOS: 2 / </s>
[2024-05-25 11:15:08,464] [DEBUG] [axolotl.load_tokenizer:281] [PID:96] [RANK:0] BOS: 1 / <s>
[2024-05-25 11:15:08,464] [DEBUG] [axolotl.load_tokenizer:282] [PID:96] [RANK:0] PAD: 2 / </s>
[2024-05-25 11:15:08,464] [DEBUG] [axolotl.load_tokenizer:283] [PID:96] [RANK:0] UNK: 0 / <unk>
[2024-05-25 11:15:08,464] [INFO] [axolotl.load_tokenizer:294] [PID:96] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.
[2024-05-25 11:15:08,464] [DEBUG] [axolotl.train.train:85] [PID:96] [RANK:0] loading model and peft_config...
[2024-05-25 11:15:08,762] [INFO] [axolotl.load_model:360] [PID:96] [RANK:0] patching with flash attention for sample packing
[2024-05-25 11:15:08,762] [INFO] [axolotl.load_model:419] [PID:96] [RANK:0] patching _expand_mask
`low_cpu_mem_usage` was None, now set to True since model is quantized.
model.safetensors: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.40G/4.40G [00:44<00:00, 99.4MB/s]
generation_config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 129/129 [00:00<00:00, 1.07MB/s]
[2024-05-25 11:15:58,242] [WARNING] [axolotl.load_model:712] [PID:96] [RANK:0] increasing model.config.max_position_embeddings from 2048 to 4096
[2024-05-25 11:15:58,242] [INFO] [axolotl.load_model:734] [PID:96] [RANK:0] GPU memory usage after model load: 1.225GB (+0.025GB cache, +0.546GB misc)
[2024-05-25 11:15:58,251] [INFO] [axolotl.load_model:785] [PID:96] [RANK:0] converting PEFT model w/ prepare_model_for_kbit_training
[2024-05-25 11:15:58,253] [INFO] [axolotl.load_model:794] [PID:96] [RANK:0] converting modules to torch.bfloat16 for flash attention
[2024-05-25 11:15:58,256] [INFO] [axolotl.load_lora:951] [PID:96] [RANK:0] found linear modules: ['q_proj', 'up_proj', 'v_proj', 'down_proj', 'o_proj', 'k_proj', 'gate_proj']
trainable params: 25,231,360 || all params: 1,125,279,744 || trainable%: 2.2422299996542017
[2024-05-25 11:15:58,674] [INFO] [axolotl.load_model:843] [PID:96] [RANK:0] GPU memory usage after adapters: 1.319GB (+0.515GB cache, +0.546GB misc)
[2024-05-25 11:15:58,703] [INFO] [axolotl.train.train:119] [PID:96] [RANK:0] Pre-saving adapter config to ./outputs/lora-out
[2024-05-25 11:15:58,706] [INFO] [axolotl.train.train:156] [PID:96] [RANK:0] Starting trainer...
[2024-05-25 11:15:58,941] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041
[2024-05-25 11:15:58,943] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041
  0%|                                                                                                                                                      | 0/48 [00:00<?, ?it/s][2024-05-25 11:15:58,989] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:316: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization
  warnings.warn(f"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization")
{'loss': 1.4615, 'grad_norm': 0.16869854927062988, 'learning_rate': 2e-05, 'epoch': 0.08}                                                                                         
  2%|██▉                                                                                                                                           | 1/48 [00:07<05:58,  7.63s/it[2024-05-25 11:16:35,436] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.4899382591247559, 'eval_runtime': 28.84, 'eval_samples_per_second': 3.467, 'eval_steps_per_second': 1.734, 'epoch': 0.08}                                         
  2%|██▉                                                                                                                                           | 1/48 [00:36<05:58,  7.63s/it[2024-05-25 11:16:42,739] [INFO] [axolotl.callbacks.on_step_end:126] [PID:96] [RANK:0] GPU memory usage while training: 1.391GB (+5.191GB cache, +0.580GB misc)                    
{'loss': 1.4241, 'grad_norm': 0.19149836897850037, 'learning_rate': 4e-05, 'epoch': 0.16}                                                                                         
{'loss': 1.3848, 'grad_norm': 0.16426080465316772, 'learning_rate': 6e-05, 'epoch': 0.24}                                                                                         
  6%|████████▉                                                                                                                                     | 3/48 [00:51<12:26, 16.59s/it[2024-05-25 11:17:19,236] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.4848034381866455, 'eval_runtime': 29.2169, 'eval_samples_per_second': 3.423, 'eval_steps_per_second': 1.711, 'epoch': 0.24}                                       
{'loss': 1.2218, 'grad_norm': 0.14049963653087616, 'learning_rate': 8e-05, 'epoch': 0.32}                                                                                         
{'loss': 1.3626, 'grad_norm': 0.1581335961818695, 'learning_rate': 0.0001, 'epoch': 0.4}                                                                                          
{'loss': 1.3668, 'grad_norm': 0.1626354455947876, 'learning_rate': 0.00012, 'epoch': 0.48}                                                                                        
 12%|█████████████████▊                                                                                                                            | 6/48 [01:42<10:13, 14.61s/it[2024-05-25 11:18:10,846] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.4397631883621216, 'eval_runtime': 29.4158, 'eval_samples_per_second': 3.4, 'eval_steps_per_second': 1.7, 'epoch': 0.48}                                           
{'loss': 1.3009, 'grad_norm': 0.14702440798282623, 'learning_rate': 0.00014, 'epoch': 0.56}                                                                                       
{'loss': 1.3332, 'grad_norm': 0.13431227207183838, 'learning_rate': 0.00016, 'epoch': 0.64}                                                                                       
{'loss': 1.2687, 'grad_norm': 0.11134211719036102, 'learning_rate': 0.00018, 'epoch': 0.72}                                                                                       
 19%|██████████████████████████▋                                                                                                                   | 9/48 [02:34<09:13, 14.20s/it[2024-05-25 11:19:02,655] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.33953857421875, 'eval_runtime': 29.501, 'eval_samples_per_second': 3.39, 'eval_steps_per_second': 1.695, 'epoch': 0.72}                                           
{'loss': 1.2957, 'grad_norm': 0.10139326751232147, 'learning_rate': 0.0002, 'epoch': 0.8}                                                                                         
{'loss': 1.2933, 'grad_norm': 0.11713045090436935, 'learning_rate': 0.000199658449300667, 'epoch': 0.88}                                                                          
{'loss': 1.2273, 'grad_norm': 0.10887009650468826, 'learning_rate': 0.00019863613034027224, 'epoch': 0.96}                                                                        
 25%|███████████████████████████████████▎                                                                                                         | 12/48 [03:26<08:27, 14.11s/it[2024-05-25 11:19:54,560] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2948062419891357, 'eval_runtime': 29.5289, 'eval_samples_per_second': 3.387, 'eval_steps_per_second': 1.693, 'epoch': 0.96}                                       
 25%|███████████████████████████████████▎                                                                                                         | 12/48 [03:55<08:27, 14.11s/it/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:316: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization
  warnings.warn(f"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization")
{'loss': 1.2662, 'grad_norm': 0.11043684184551239, 'learning_rate': 0.00019694002659393305, 'epoch': 1.04}                                                                        
 27%|██████████████████████████████████████▏                                                                                                      | 13/48 [04:03<12:25, 21.29s/it][2024-05-25 11:20:02,854] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041
{'loss': 1.2585, 'grad_norm': 0.11361780762672424, 'learning_rate': 0.00019458172417006347, 'epoch': 1.08}                                                                        
{'loss': 1.2513, 'grad_norm': 0.1074887067079544, 'learning_rate': 0.00019157733266550575, 'epoch': 1.16}                                                                         
 31%|████████████████████████████████████████████                                                                                                 | 15/48 [04:18<07:48, 14.20s/it[2024-05-25 11:20:47,217] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2809884548187256, 'eval_runtime': 29.4793, 'eval_samples_per_second': 3.392, 'eval_steps_per_second': 1.696, 'epoch': 1.16}                                       
{'loss': 1.1642, 'grad_norm': 0.09892746061086655, 'learning_rate': 0.0001879473751206489, 'epoch': 1.24}                                                                         
{'loss': 1.1603, 'grad_norm': 0.09089738875627518, 'learning_rate': 0.00018371664782625287, 'epoch': 1.32}                                                                        
{'loss': 1.2269, 'grad_norm': 0.09244850277900696, 'learning_rate': 0.00017891405093963938, 'epoch': 1.4}                                                                         
 38%|████████████████████████████████████████████████████▉                                                                                        | 18/48 [05:10<07:02, 14.08s/it[2024-05-25 11:21:38,866] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2540477514266968, 'eval_runtime': 29.3745, 'eval_samples_per_second': 3.404, 'eval_steps_per_second': 1.702, 'epoch': 1.4}                                        
{'loss': 1.2267, 'grad_norm': 0.09758879989385605, 'learning_rate': 0.00017357239106731317, 'epoch': 1.48}                                                                        
{'loss': 1.192, 'grad_norm': 0.08069184422492981, 'learning_rate': 0.00016772815716257412, 'epoch': 1.56}                                                                         
{'loss': 1.1356, 'grad_norm': 0.07329125702381134, 'learning_rate': 0.0001614212712689668, 'epoch': 1.64}                                                                         
 44%|█████████████████████████████████████████████████████████████▋                                                                               | 21/48 [06:02<06:18, 14.02s/it[2024-05-25 11:22:30,367] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2350471019744873, 'eval_runtime': 29.2682, 'eval_samples_per_second': 3.417, 'eval_steps_per_second': 1.708, 'epoch': 1.64}                                       
{'loss': 1.2026, 'grad_norm': 0.0817209780216217, 'learning_rate': 0.00015469481581224272, 'epoch': 1.72}                                                                         
{'loss': 1.2479, 'grad_norm': 0.07559554278850555, 'learning_rate': 0.00014759473930370736, 'epoch': 1.8}                                                                         
{'loss': 1.2701, 'grad_norm': 0.07286789268255234, 'learning_rate': 0.00014016954246529696, 'epoch': 1.88}                                                                        
 50%|██████████████████████████████████████████████████████████████████████▌                                                                      | 24/48 [06:53<05:35, 13.97s/it[2024-05-25 11:23:21,837] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.227484941482544, 'eval_runtime': 29.2809, 'eval_samples_per_second': 3.415, 'eval_steps_per_second': 1.708, 'epoch': 1.88}                                        
 50%|██████████████████████████████████████████████████████████████████████▌                                                                      | 24/48 [07:22<05:35, 13.97s/it/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:316: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization
  warnings.warn(f"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization")
{'loss': 1.204, 'grad_norm': 0.07036563754081726, 'learning_rate': 0.00013246994692046836, 'epoch': 1.96}                                                                         
{'loss': 1.1928, 'grad_norm': 0.07845467329025269, 'learning_rate': 0.00012454854871407994, 'epoch': 2.04}                                                                        
 54%|████████████████████████████████████████████████████████████████████████████▍                                                                | 26/48 [07:37<06:06, 16.68s/it][2024-05-25 11:23:36,588] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041
{'loss': 1.148, 'grad_norm': 0.0651242807507515, 'learning_rate': 0.00011645945902807341, 'epoch': 2.08}                                                                          
 56%|███████████████████████████████████████████████████████████████████████████████▎                                                             | 27/48 [07:45<04:51, 13.90s/it[2024-05-25 11:24:13,265] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2253979444503784, 'eval_runtime': 29.2812, 'eval_samples_per_second': 3.415, 'eval_steps_per_second': 1.708, 'epoch': 2.08}                                       
{'loss': 1.169, 'grad_norm': 0.07599511742591858, 'learning_rate': 0.00010825793454723325, 'epoch': 2.16}                                                                         
{'loss': 1.2047, 'grad_norm': 0.06874757260084152, 'learning_rate': 0.0001, 'epoch': 2.24}                                                                                        
{'loss': 1.1529, 'grad_norm': 0.06863673776388168, 'learning_rate': 9.174206545276677e-05, 'epoch': 2.32}                                                                         
 62%|████████████████████████████████████████████████████████████████████████████████████████▏                                                    | 30/48 [08:36<04:10, 13.93s/it[2024-05-25 11:25:04,763] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.220930576324463, 'eval_runtime': 29.2937, 'eval_samples_per_second': 3.414, 'eval_steps_per_second': 1.707, 'epoch': 2.32}                                        
{'loss': 1.1821, 'grad_norm': 0.06745310127735138, 'learning_rate': 8.35405409719266e-05, 'epoch': 2.4}                                                                           
{'loss': 1.2093, 'grad_norm': 0.06782731413841248, 'learning_rate': 7.54514512859201e-05, 'epoch': 2.48}                                                                          
{'loss': 1.1945, 'grad_norm': 0.06917714327573776, 'learning_rate': 6.753005307953167e-05, 'epoch': 2.56}                                                                         
 69%|████████████████████████████████████████████████████████████████████████████████████████████████▉                                            | 33/48 [09:28<03:29, 13.95s/it[2024-05-25 11:25:56,248] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2193293571472168, 'eval_runtime': 29.2683, 'eval_samples_per_second': 3.417, 'eval_steps_per_second': 1.708, 'epoch': 2.56}                                       
{'loss': 1.1886, 'grad_norm': 0.06958159059286118, 'learning_rate': 5.983045753470308e-05, 'epoch': 2.64}                                                                         
{'loss': 1.1788, 'grad_norm': 0.0727023035287857, 'learning_rate': 5.240526069629265e-05, 'epoch': 2.72}                                                                          
{'loss': 1.112, 'grad_norm': 0.07052871584892273, 'learning_rate': 4.530518418775733e-05, 'epoch': 2.8}                                                                           
 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                   | 36/48 [10:19<02:47, 13.96s/it[2024-05-25 11:26:47,758] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2152072191238403, 'eval_runtime': 29.2768, 'eval_samples_per_second': 3.416, 'eval_steps_per_second': 1.708, 'epoch': 2.8}                                        
 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                   | 36/48 [10:48<02:47, 13.96s/it/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
  warnings.warn(
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:316: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization
  warnings.warn(f"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization")
{'loss': 1.1843, 'grad_norm': 0.0690612718462944, 'learning_rate': 3.857872873103322e-05, 'epoch': 2.88}                                                                          
{'loss': 1.1661, 'grad_norm': 0.06867097318172455, 'learning_rate': 3.227184283742591e-05, 'epoch': 2.96}                                                                         
{'loss': 1.1506, 'grad_norm': 0.06810767203569412, 'learning_rate': 2.6427608932686843e-05, 'epoch': 3.04}                                                                        
 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                          | 39/48 [11:12<02:07, 14.14s/it[2024-05-25 11:27:40,491] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2144311666488647, 'eval_runtime': 29.3187, 'eval_samples_per_second': 3.411, 'eval_steps_per_second': 1.705, 'epoch': 3.04}                                       
 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                          | 39/48 [11:41<02:07, 14.14s/it[2024-05-25 11:27:40,519] [INFO] [axolotl.utils.samplers.multipack._len_est:184] [PID:96] [RANK:0] packing_efficiency_estimate: 0.98 total_num_tokens per device: 414041           
{'loss': 1.1548, 'grad_norm': 0.06440620869398117, 'learning_rate': 2.1085949060360654e-05, 'epoch': 3.08}                                                                        
{'loss': 1.146, 'grad_norm': 0.06522033363580704, 'learning_rate': 1.6283352173747145e-05, 'epoch': 3.16}                                                                         
{'loss': 1.189, 'grad_norm': 0.06824471801519394, 'learning_rate': 1.2052624879351104e-05, 'epoch': 3.24}                                                                         
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                 | 42/48 [12:03<01:24, 14.03s/it[2024-05-25 11:28:32,058] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2129244804382324, 'eval_runtime': 29.2818, 'eval_samples_per_second': 3.415, 'eval_steps_per_second': 1.708, 'epoch': 3.24}                                       
{'loss': 1.1736, 'grad_norm': 0.069525808095932, 'learning_rate': 8.422667334494249e-06, 'epoch': 3.32}                                                                           
{'loss': 1.0965, 'grad_norm': 0.06717149913311005, 'learning_rate': 5.418275829936537e-06, 'epoch': 3.4}                                                                          
{'loss': 1.1006, 'grad_norm': 0.06725575774908066, 'learning_rate': 3.059973406066963e-06, 'epoch': 3.48}                                                                         
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏        | 45/48 [12:55<00:41, 13.98s/it[2024-05-25 11:29:23,559] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.211127519607544, 'eval_runtime': 29.2906, 'eval_samples_per_second': 3.414, 'eval_steps_per_second': 1.707, 'epoch': 3.48}                                        
{'loss': 1.1261, 'grad_norm': 0.06806794553995132, 'learning_rate': 1.3638696597277679e-06, 'epoch': 3.56}                                                                        
{'loss': 1.1464, 'grad_norm': 0.06902754306793213, 'learning_rate': 3.415506993330153e-07, 'epoch': 3.64}                                                                         
{'loss': 1.1887, 'grad_norm': 0.07014337927103043, 'learning_rate': 0.0, 'epoch': 3.72}                                                                                           
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [13:46<00:00, 13.97s/it[2024-05-25 11:30:15,098] [INFO] [accelerate.accelerator.gather_for_metrics:2380] [PID:96] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.
{'eval_loss': 1.2139402627944946, 'eval_runtime': 29.3147, 'eval_samples_per_second': 3.411, 'eval_steps_per_second': 1.706, 'epoch': 3.72}                                       
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [14:16<00:00, 13.97s/it/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
{'train_runtime': 856.8959, 'train_samples_per_second': 8.869, 'train_steps_per_second': 0.056, 'train_loss': 1.2187986423571904, 'epoch': 3.72}                                  
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [14:16<00:00, 17.85s/it]
[2024-05-25 11:30:15,880] [INFO] [axolotl.train.train:173] [PID:96] [RANK:0] Training Completed!!! Saving pre-trained model to ./outputs/lora-out
/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
(PeftModelForCausalLM(   (base_model): LoraModel(     (model): LlamaForCausalLM(       (model): LlamaModel(         (embed_tokens): Embedding(32000, 2048)         (layers): ModuleList(           (0-21): 22 x LlamaDecoderLayer(             (self_attn): LlamaAttention(               (q_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=2048, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=2048, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (k_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=256, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=256, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (v_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=256, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=256, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (o_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=2048, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=2048, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (rotary_emb): LlamaRotaryEmbedding()             )             (mlp): LlamaMLP(               (gate_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=5632, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=5632, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (up_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=2048, out_features=5632, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=2048, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=5632, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (down_proj): lora.Linear8bitLt(                 (base_layer): Linear8bitLt(in_features=5632, out_features=2048, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=5632, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=2048, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (act_fn): SiLU()             )             (input_layernorm): LlamaRMSNorm()             (post_attention_layernorm): LlamaRMSNorm()           )         )         (norm): LlamaRMSNorm()       )       (lm_head): Linear(in_features=2048, out_features=32000, bias=False)     )   ) ), LlamaTokenizer(name_or_path='TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '</s>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={    0: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),    1: AddedToken("<s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),    2: AddedToken("</s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), })
root@22318190fe62:~/axolotl#