Tweaking Mistral 7b Tutorial with qlora with axolotl for environment friendly LLM coaching

# 1. Verify GPU availability
!nvidia-smi


# 2. Set up git-lfs (for dealing with giant mannequin information)
!sudo apt-get -y set up git-lfs
!git lfs set up


# 3. Clone Axolotl and set up from supply
!git clone https://github.com/OpenAccess-AI-Collective/axolotl.git
%cd axolotl
!pip set up -e .


# (Non-compulsory) When you want a selected PyTorch model, set up it BEFORE Axolotl:
# !pip set up torch==2.0.1+cu118 --extra-index-url https://obtain.pytorch.org/whl/cu118


# Return to /content material listing
%cd /content material

import os


# Create a small JSONL dataset
os.makedirs("knowledge", exist_ok=True)
with open("knowledge/sample_instructions.jsonl", "w") as f:
    f.write('{"instruction": "Clarify quantum computing in easy phrases.", "enter": "", "output": "Quantum computing makes use of qubits..."}n')
    f.write('{"instruction": "What's the capital of France?", "enter": "", "output": "The capital of France is Paris."}n')


# Write a QLoRA config for Mistral 7B
config_text = """
base_model: mistralai/mistral-7b-v0.1
tokenizer: mistralai/mistral-7b-v0.1


# We'll use QLoRA to reduce reminiscence utilization
train_type: qlora
bits: 4
double_quant: true
quant_type: nf4


lora_r: 8
lora_alpha: 16
lora_dropout: 0.05
target_modules:
  - q_proj
  - k_proj
  - v_proj


knowledge:
  datasets:
    - path: /content material/knowledge/sample_instructions.jsonl
  val_set_size: 0
  max_seq_length: 512
  cutoff_len: 512


training_arguments:
  output_dir: /content material/mistral-7b-qlora-output
  num_train_epochs: 1
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 4
  learning_rate: 0.0002
  fp16: true
  logging_steps: 10
  save_strategy: "epoch"
  evaluation_strategy: "no"


wandb:
  enabled: false
"""


with open("qlora_mistral_7b.yml", "w") as f:
    f.write(config_text)


print("Dataset and QLoRA config created.")

# It will obtain Mistral 7B (~13 GB) and begin fine-tuning with QLoRA.
# When you encounter OOM (Out Of Reminiscence) errors, scale back max_seq_length or LoRA rank.


!axolotl --config /content material/qlora_mistral_7b.yml

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer


# Load the bottom Mistral 7B mannequin
base_model_path = "mistralai/mistral-7b-v0.1"   #First set up entry utilizing your person account on HF then run this half
output_dir = "/content material/mistral-7b-qlora-output"


print("nLoading base mannequin and tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
    base_model_path,
    trust_remote_code=True
)
base_model = AutoModelForCausalLM.from_pretrained(
    base_model_path,
    device_map="auto",
    torch_dtype=torch.float16,
    trust_remote_code=True
)


print("nLoading QLoRA adapter...")
mannequin = PeftModel.from_pretrained(
    base_model,
    output_dir,
    device_map="auto",
    torch_dtype=torch.float16
)
mannequin.eval()


# Instance immediate
immediate = "What are the primary variations between classical and quantum computing?"
inputs = tokenizer(immediate, return_tensors="pt").to("cuda")


print("nGenerating response...")
with torch.no_grad():
    outputs = mannequin.generate(**inputs, max_new_tokens=128)


response = tokenizer.decode(outputs[0], skip_special_tokens=True)
print("n=== Mannequin Output ===")
print(response)