Transformers

from datasets import load_dataset

# Load dataset
dataset = load_dataset("Yelp/yelp_review_full")
# Tune only on a validation partition of training data, keeping test untouched.
split = dataset["train"].train_test_split(test_size=0.1, seed=42, stratify_by_column="label")
train_dataset = split["train"]
eval_dataset = split["test"]
test_dataset = dataset["test"]

# Tokenize
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

def tokenize_function(examples):
    return tokenizer(
        examples["text"],
        truncation=True,
        max_length=512
    )

train_dataset = train_dataset.map(tokenize_function, batched=True)
eval_dataset = eval_dataset.map(tokenize_function, batched=True)

from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained(
    "google-bert/bert-base-uncased",
    num_labels=5  # Number of classes
)

import numpy as np

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return {"accuracy": float(np.mean(predictions == labels))}

from transformers import TrainingArguments

training_args = TrainingArguments(
    output_dir="./results",
    eval_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=3,
    weight_decay=0.01,
    report_to="none",  # Opt in to external trackers explicitly.
    logging_steps=10,
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
)

from transformers import Trainer, DataCollatorWithPadding

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    compute_metrics=compute_metrics,
    processing_class=tokenizer,
    data_collator=DataCollatorWithPadding(tokenizer),
)

# Start training
trainer.train()

# Evaluate
results = trainer.evaluate()
print(results)

trainer.save_model("./fine_tuned_model")
tokenizer.save_pretrained("./fine_tuned_model")

# Optional external upload, only when intended:
# Set hub_model_id="username/my-finetuned-model" in TrainingArguments first.
# trainer.push_to_hub(commit_message="Upload fine-tuned model")

output_dir="./results"

num_train_epochs=3

per_device_train_batch_size=8

learning_rate=2e-5  # Common for BERT-style models
learning_rate=5e-5  # Common for smaller models

weight_decay=0.01

eval_strategy="epoch"  # Evaluate after each epoch
eval_strategy="steps"  # Evaluate every eval_steps

save_strategy="epoch"
save_strategy="steps"
save_steps=500

load_best_model_at_end=True
metric_for_best_model="accuracy"  # Metric to compare

gradient_accumulation_steps=4  # Effective batch = per-device batch * accumulation * data-parallel workers (except partial final groups)

fp16=True

bf16=True

gradient_checkpointing=True  # Slower but uses less memory

optim="adamw_torch"  # Explicit portable choice; current Torch>=2.8 default is adamw_torch_fused
optim="adamw_8bit"    # 8-bit Adam (requires bitsandbytes)
optim="adafactor"     # Memory-efficient alternative

lr_scheduler_type="linear"       # Linear decay
lr_scheduler_type="cosine"       # Cosine annealing
lr_scheduler_type="constant"     # No decay
lr_scheduler_type="constant_with_warmup"

warmup_steps=500
# Or
warmup_steps=0.1  # 10% of total steps

logging_steps=10

report_to="none"  # Local-only default used here
report_to=["tensorboard"]
report_to=["wandb"]
report_to=["tensorboard", "wandb"]

ddp_backend="nccl"  # For multi-GPU

deepspeed="ds_config.json"

from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    data_collator=data_collator,
    eval_dataset=eval_dataset,
    processing_class=tokenizer,
)

from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=True,
    mlm_probability=0.15
)

from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(
    tokenizer=tokenizer,
    model=model,
    padding=True
)

import torch
from transformers import Trainer

class WeightedTrainer(Trainer):
    def __init__(self, *args, class_weights, **kwargs):
        super().__init__(*args, **kwargs)
        self.class_weights = torch.as_tensor(class_weights, dtype=torch.float32)
        self.model_accepts_loss_kwargs = False

    def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
        labels = inputs["labels"]
        outputs = model(**{key: value for key, value in inputs.items() if key != "labels"})
        logits = outputs.logits
        loss = torch.nn.functional.cross_entropy(
            logits, labels, weight=self.class_weights.to(device=logits.device, dtype=logits.dtype)
        )
        return (loss, outputs) if return_outputs else loss

from transformers import TrainerCallback

class CustomCallback(TrainerCallback):
    def on_epoch_end(self, args, state, control, **kwargs):
        print(f"Epoch {state.epoch} completed")
        # Custom logic here
        return control

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    processing_class=tokenizer,
    data_collator=DataCollatorWithPadding(tokenizer),
    callbacks=[CustomCallback],
)

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["query", "value"],
    lora_dropout=0.05,
    bias="none",
    task_type="SEQ_CLS"
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # Shows reduced parameter count

# Train normally with Trainer
trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset,
                  eval_dataset=eval_dataset, processing_class=tokenizer,
                  data_collator=DataCollatorWithPadding(tokenizer))
trainer.train()

model.gradient_checkpointing_enable()

training_args = TrainingArguments(
    output_dir="./checkpointed-results",
    gradient_checkpointing=True,
    report_to="none",
)

training_args = TrainingArguments(
    output_dir="./mixed-precision-results",
    bf16=True,  # Only on a backend that supports it; use fp16=True as an alternative.
    report_to="none",
)

{
  "train_batch_size": "auto",
  "train_micro_batch_size_per_gpu": "auto",
  "gradient_accumulation_steps": "auto",
  "optimizer": {"type": "AdamW", "params": {"lr": "auto"}},
  "bf16": {"enabled": "auto"},
  "zero_optimization": {"stage": 2}
}

training_args = TrainingArguments(
    output_dir="./distributed-results",
    deepspeed="ds_config.json",
    report_to="none",
)

def model_init():
    return AutoModelForSequenceClassification.from_pretrained(
        "google-bert/bert-base-uncased",
        num_labels=5
    )

def optuna_hp_space(trial):
    return {
        "learning_rate": trial.suggest_float("learning_rate", 1e-5, 5e-5, log=True),
        "per_device_train_batch_size": trial.suggest_categorical("per_device_train_batch_size", [8, 16, 32]),
        "num_train_epochs": trial.suggest_int("num_train_epochs", 2, 5),
    }

trainer = Trainer(model_init=model_init, args=training_args, train_dataset=train_dataset,
                  eval_dataset=eval_dataset, compute_metrics=compute_metrics,
                  processing_class=tokenizer, data_collator=DataCollatorWithPadding(tokenizer))
best_trial = trainer.hyperparameter_search(
    direction="maximize",
    backend="optuna",
    hp_space=optuna_hp_space,
    n_trials=10,
)

tensorboard --logdir ./logs  # Match TENSORBOARD_LOGGING_DIR used by training

import wandb
wandb.init(project="my-project")

training_args = TrainingArguments(
    output_dir="./tracked-results",
    report_to=["wandb"],
)

trainer.train(resume_from_checkpoint="./results/checkpoint-1000")