(My way of) Running Jobs on Alvis

module load Python/3.11.3-GCCcore-12.3.0
module load PyTorch/2.1.2-foss-2023a-CUDA-12.1.1

# NOTE: If you try to run the following command without pre-loading the above
# modules, Alvis will complain and tell that python3 is not even an available
# command.
python3 -m venv my-env-name

source path/to/my/environment/my-env-name/bin/activate
pip install --upgrade pip
pip install jupyter
pip install <what you need>
pip install bitsandbytes
pip install optuna
pip install h5py
pip install optuna
pip install rdkit
pip install pynvml tqdm jsonargparse nltk rouge_score evaluate
pip install pandas tensorboard tabulate scikit-learn
pip install datasets
pip install tokenizers
pip install accelerate
pip install huggingface
pip install bitsandbytes
pip install trl

#!/bin/bash
# Load necessary modules
module load Python/3.11.3-GCCcore-12.3.0
module load PyTorch/2.1.2-foss-2023a-CUDA-12.1.1

# Source the virtual environment
source path/to/my/environment/my-env-name/bin/activate

source setup_environment.sh

#!/bin/bash
#SBATCH --account=my-account-number
#SBATCH --partition=alvis
#SBATCH --gpus-per-node=T4:1
#SBATCH --job-name=my-awesom-job
#SBATCH --time=2:00:00

source setup_environment.sh

python my_python_script.py

deactivate

#!/bin/bash
#SBATCH --account=my-account-number
#SBATCH --partition=alvis
#SBATCH --gpus-per-node=T4:1
#SBATCH --job-name=my-awesom-job
#SBATCH --time=2:00:00

# Load Python modules
module load Python/3.11.3-GCCcore-12.3.0
module load PyTorch/2.1.2-foss-2023a-CUDA-12.1.1

# Activate environment
source path/to/my/environment/my-env-name/bin/activate

# Run my stuff
python my_python_script.py

# Deactivate the environment
deactivate

source setup_environment.sh
jupyter notebook --no-browser

Bootstrap: localimage
From: /apps/containers/Conda/miniconda-22.11.1.sif
# NOTE: The line above can be modified to use a different base image if needed,
#       for example: "From <my/local/folders/container/file>.sif"

%post
    # Put all your installation commands here, all the following commands will
    # be executed when building the container. None of them are required, it's
    # just an example of how to install packages and libraries.
    apt-get -y update
    apt-get -y install git-lfs
    /opt/conda/bin/conda install -y pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
    /opt/conda/bin/conda install -y -c huggingface transformers tokenizers datasets
    /opt/conda/bin/conda install -y -c conda-forge accelerate pandas evaluate tensorboard tabulate scikit-learn
    pip install pynvml tqdm jsonargparse nltk rouge_score  
    conda install -c conda-forge tabulate
    conda install -y -c conda-forge scikit-learn
    conda install -y -c anaconda scikit-learn
    pip install rdkit
    pip install -U "huggingface_hub[cli]"
    conda remove transformers -y
    pip install transformers
    conda remove datasets -y
    pip install datasets
    pip install evaluate
    pip install peft
    
    [...]

apptainer build <path/to/destination/container/file>.sif <path/to/recipe/file>.def

#!/bin/bash

# Check if Apptainer path is provided
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
    echo "Usage: $0 <path-to-apptainer-image> [path-to-work-directory]"
    exit 1
fi

# Path to the Apptainer image
IMAGE_PATH="$1"

# Optional path to the work directory
if [ -n "$2" ]; then
    WORK_DIR="$2"
    # Check if directory exists
    if [ ! -d "$WORK_DIR" ]; then
        echo "The specified directory does not exist: $WORK_DIR"
        exit 1
    fi
    # Change to the specified directory
    cd "$WORK_DIR"
fi

# Check if Apptainer is installed
if ! command -v apptainer &> /dev/null; then
    echo "Apptainer is not installed. Please install it to continue."
    exit 1
fi

# Run the Apptainer image with Jupyter Notebook
apptainer exec "$IMAGE_PATH" jupyter notebook --no-browser --ip=0.0.0.0 --port=8889 --allow-root

#!/usr/bin/env bash
#SBATCH -A NAISS2024-5-630 -p alvis
#SBATCH -N 1
#SBATCH -C NOGPU
#SBATCH --cpus-per-task=32
#SBATCH -t 0-2:00:00
#SBATCH -J "slurm-score-predictions"
#SBATCH -o slurm-score-predictions.log

cd $SLURM_SUBMIT_DIR
echo "Running score_predictions.py"

export PYTHONPATH=$PYTHONPATH:/my/local/dir/containing/code

apptainer exec ~/containers/container_file.sif python $SLURM_SUBMIT_DIR/scripts/score_predictions.py --num_proc=32 --skip_if_log_exists

#!/usr/bin/env bash
#SBATCH -N 1 --gpus-per-node=T4:1
#SBATCH -t 0-2:00:00
#SBATCH -J "slurm-predict-MyModel"
#SBATCH -o slurm-predict-MyModel.log  # Output log file

echo "Starting at `date`"
echo "Running on hosts: $SLURM_NODELIST"
echo "Running on $SLURM_NNODES nodes."
echo "Running $SLURM_NTASKS tasks."
echo "Job id is $SLURM_JOBID"
echo "Job submission directory is: $SLURM_SUBMIT_DIR"
cd $SLURM_SUBMIT_DIR

local_dir=my/local/dir/containing/code

# echo "-------------------------------------------------------------------------"
# apptainer exec ~/containers/container_file.sif accelerate env
# echo "-------------------------------------------------------------------------"

export PYTHONPATH=$PYTHONPATH:my/local/dir/containing/code/

apptainer exec ~/containers/container_file.sif python ${local_dir}/scripts/collect_llm_predictions.py \
    --model_name=MyModel \
    --batch_size=32 \
    --num_proc=16 \
    --eval_gen_strategies=true