Hyperpod Nccl

# On each compute node (add to lifecycle script for persistence). Source: NVIDIA nccl-tests.
cd /opt && git clone <nccl-tests-source> nccl-tests
cd /opt/nccl-tests
make MPI=1 MPI_HOME=/usr/local/mpi NCCL_HOME=/usr/local/nccl CUDA_HOME=/usr/local/cuda
# Binary: /opt/nccl-tests/build/all_reduce_perf

# Single-GPU test (quick sanity check):
/opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1

# All-GPU test (p4d: 8 GPUs, p5: 8 GPUs):
/opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 8

# Expected output column headers:
# size  count  type  redop  root  time  algbw  busbw  error  time  algbw  busbw

# With MPI (from head node):
mpirun -np <TOTAL_RANKS> \
  --hostfile /etc/hosts \
  -N <RANKS_PER_NODE> \
  -x FI_PROVIDER=efa \
  -x FI_EFA_USE_DEVICE_RDMA=1 \
  -x NCCL_SOCKET_IFNAME=^lo,docker,efa,veth \
  -x NCCL_DEBUG=WARN \
  /opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1

# With Slurm:
srun --nodes=4 --ntasks-per-node=8 \
  /opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1

# With kubectl (EKS, 2 nodes, 8 GPUs each):
# Deploy as a K8s Job with 2 pods, each requesting 8 GPUs.
# Use mpirun inside the container, or the Kubeflow MPI Operator.
kubectl exec -n <NS> <POD> -- mpirun -np 16 -N 8 \
  --hostfile /etc/hosts \
  -x FI_PROVIDER=efa -x FI_EFA_USE_DEVICE_RDMA=1 \
  /opt/nccl-tests/build/all_reduce_perf -b 8 -e 8G -f 2 -g 1

# Test each node pair individually to find the outlier:
# From node A → node B:
fi_ping -p efa -I 100 <NODE_B_IP>

# From node B → node A:
fi_ping -p efa -I 100 <NODE_A_IP>

# Automate across all pairs (run on head node):
for node in $(scontrol show hostnames $SLURM_JOB_NODELIST); do
    echo -n "Testing $node: "
    fi_ping -p efa -I 10 "$node" 2>/dev/null | tail -1 || echo "FAILED"
done

# Enable per-rank debug files:
export NCCL_DEBUG=INFO
export NCCL_DEBUG_FILE=/tmp/nccl_rank${RANK}.log

# After training (or timeout), check which rank was slow:
# Look for the last "AllReduce" timestamp before the timeout:
grep -h "AllReduce\|ring\|timeout" /tmp/nccl_rank*.log | sort -k1,1 | tail -30

# Compare timestamps across ranks — the one furthest behind is the straggler:
for f in /tmp/nccl_rank*.log; do
    echo -n "$f: last line timestamp = "
    tail -1 "$f" | awk '{print $1, $2}'
done

import datetime
import torch.distributed as dist

# nodes * 5 + 600 is a simple heuristic — tune against your actual step time:
nodes = int(os.environ.get("WORLD_SIZE", "1")) // 8   # GPUs per node
timeout_s = nodes * 5 + 600

dist.init_process_group(
    backend="nccl",
    timeout=datetime.timedelta(seconds=timeout_s),
)

export TORCH_NCCL_ASYNC_ERROR_HANDLING=1
export TORCH_NCCL_BLOCKING_WAIT=1   # for debugging; has a perf cost at scale

# Full EFA performance configuration:
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1    # GPU Direct RDMA
export NCCL_PROTO=Simple           # large-message protocol (valid: LL, LL128, Simple)
export NCCL_SOCKET_IFNAME=^lo,docker,efa,veth
# Collective timeout goes in training code: init_process_group(timeout=timedelta(seconds=1800))

# Optional tuning for very large jobs:
export FI_EFA_FORK_SAFE=1            # safe for multiprocessing
export FI_EFA_ENABLE_SHM_TRANSFER=1  # intra-node shared memory

# Do NOT set in production:
# NCCL_DEBUG=INFO  (verbose; runtime overhead — disable in production)
# CUDA_LAUNCH_BLOCKING=1  (disables GPU/CPU overlap, very slow)

# Identify the bad node's instance ID:
kubectl get node <NODE_NAME> -o jsonpath='{.spec.providerID}' | cut -d'/' -f5
# OR for Slurm — list-cluster-nodes does NOT return PrivateDnsHostname (only describe-cluster-node does).
# Two-step: list candidate IDs, then describe each one until DNS matches the Slurm name.
SLURM_NODE="<SLURM_NODE_NAME>"
for IID in $(aws sagemaker list-cluster-nodes --cluster-name <C> --region <R> \
               --query 'ClusterNodeSummaries[?InstanceStatus.Status==`Running`].InstanceId' --output text); do
  DNS=$(aws sagemaker describe-cluster-node --cluster-name <C> --region <R> --node-id "$IID" \
          --query 'NodeDetails.PrivateDnsHostname' --output text 2>/dev/null)
  case "$DNS" in "$SLURM_NODE."*) echo "$SLURM_NODE → $IID"; break ;; esac
done

# EKS — cordon prevents new pods; drain evicts existing pods (emptyDir data lost).
kubectl cordon <NODE_NAME>
kubectl drain <NODE_NAME> --ignore-daemonsets --delete-emptydir-data

# Slurm — on the controller via SSM; running jobs continue until they finish.
scontrol update nodename=<NODE> state=drain reason="low-bandwidth-$(date +%Y%m%d)"

aws sagemaker batch-replace-cluster-nodes \
  --cluster-name <C> --region <R> \
  --node-ids '["<INSTANCE_ID>"]'

# Monitor replacement completion (read-only):
watch -n 10 "aws sagemaker list-cluster-nodes --cluster-name <C> --region <R> \
  --query 'ClusterNodeSummaries[*].{ID:InstanceId,State:InstanceStatus.Status}' \
  --output table"