Conda is an open source, cross-platform, language-agnostic package manager and environment management system. It is released under the Berkeley Software Distribution License by Continuum Analytics.

Conda allows users to easily install different versions of binary software packages and any required libraries appropriate for their computing platform. Also, it allows users to switch between package versions and download and install updates from a software repository.

Conda is written in the Python programming language, but can manage projects containing code written in other languages (e.g., R), including multi-language projects. Conda can install the Python programming language, while similar Python-based cross-platform package managers (such as wheel or pip) cannot.

A set of Conda environments has been defined on the HPC system.

Conda configuration on the HPC cluster (private area)

To see a list of all available environments run:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda info --envs

Enable the "base" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate base

Disable the "base" environment:

conda deactivate

Available versions of the "statistical-genetics" environment:

Enable the default version of the "statistical-genetics" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate statistical-genetics

Enable the 2020.03.14 version of the "statistical-genetics" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate statistical-genetics-2020.03.14

Packages in the "statistical-genetics" environment (any version):

conda list

Disable the "statistical-genetics" environment (any version):

conda deactivate

Now we will see how to run the scripts that you find on the R page.

Script slurm-statistical-genetics-vrt.sh:

slurm-statistical-genetics-vrt.sh

#!/bin/bash
#SBATCH --job-name=hello
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --mem=512M
#SBATCH --partition=vrt
#SBATCH --time=0-00:05:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate statistical-genetics
 
Rscript --no-save hello.R
 
conda deactivate

Script slurm-statistical-genetics-bdw.sh:

slurm-statistical-genetics-bdw.sh

#!/bin/bash
#SBATCH --job-name=loop
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --mem=512M
#SBATCH --partition=bdw
#SBATCH --time=0-00:05:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate statistical-genetics
 
Rscript --no-save loop.R ${SLURM_NTASKS_PER_NODE:-1}
 
conda deactivate

Main packages included in 2022 condaenvs:

conda install:
 - cudatoolkit
 - pip
 - mpi4py
 - h5py
 - keras
 - opencv
 - opencv-python-headless
 - tensorflow
 - torchvision
 - easydict
 - pytorch 
 - statsmodels
 - flax
 - transformers

pip install:
 - keras.datasets
 - sklearn

Available versions of the "machine-learning-cuda" environment:

Main packages included in 2021 condaenvs:

conda install:
 - cudatoolkit
 - pip
 - mpi4py
 - h5py
 - keras
 - opencv
 - opencv-python-headless
 - tensorflow
 - torchvision
 - easydict
 - pytorch 

pip install:
 - keras.datasets
 - sklearn

Available versions of the "machine-learning-cuda" environment:

Creation scripts:

/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-10.0
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-10.1
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-10.2
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-11.0
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-11.1
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-11.2.0
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-11.2.1
/hpc/share/tools/conda/bin/hpc-make-condaenv-machine-learning-cuda-11.2.2

Enable the default version of the "machine-learning-cuda-10.1" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate machine-learning-cuda-10.1

Packages in the "machine-learning-cuda-XX" environment (any version):

conda list

Disable the "machine-learning-cuda" environment (any version):

conda deactivate

Script slurm-machine-learning-cuda.sh:

slurm-machine-learning-cuda.sh

#!/bin/bash
#SBATCH --job-name=conda
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:tesla:1
#SBATCH --partition=gpu
#SBATCH --mem=16G
#SBATCH --time=0-00:30:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate machine-learning-cuda-10.1
 
#...
 
conda deactivate

Available versions of the "cryolo-cpu" environment:

Enable the default version of the "cryolo-cpu" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate cryolo-cpu

Enable the 2020.09.18 version of the "cryolo-cpu" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate cryolo-cpu-2020.09.18

Packages in the "cryolo-cpu" environment (any version):

conda list

Disable the "cryolo-cpu" environment (any version):

conda deactivate

Script slurm-cryolo-vrt.sh:

slurm-cryolo-vrt.sh

#!/bin/bash
#SBATCH --job-name=cryolo
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --mem=1G
#SBATCH --partition=vrt
#SBATCH --time=0-01:00:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate cryolo-cpu
 
cryolo_gui.py config config_cryolo.json 220 \
--filter LOWPASS \
--low_pass_cutoff 0.1 \
--num_cpu ${SLURM_NTASKS_PER_NODE:-1}
 
conda deactivate

Script slurm-cryolo-bdw.sh:

slurm-cryolo-bdw.sh

#!/bin/bash
#SBATCH --job-name=cryolo
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --mem=4G
#SBATCH --partition=bdw
#SBATCH --time=0-01:00:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate cryolo-cpu
 
cryolo_gui.py config config_cryolo.json 220 \
--filter LOWPASS \
--low_pass_cutoff 0.1 \
--num_cpu ${SLURM_NTASKS_PER_NODE:-1}
 
conda deactivate

Available versions of the "cryolo-gpu" environment:

Enable the default version of the "cryolo-gpu" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate cryolo-gpu

Enable the 2020.09.18 version of the "cryolo-gpu" environment:

module load miniconda3
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
conda activate cryolo-gpu-2020.09.18

Packages in the "cryolo-gpu" environment (any version):

conda list

Disable the "cryolo-gpu" environment (any version):

conda deactivate

Script slurm-cryolo-gpu.sh:

slurm-cryolo-gpu.sh

#!/bin/bash
#SBATCH --job-name=cryolo
#SBATCH --output=%x.o%j
#SBATCH --error=%x.e%j
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --mem=1G
#SBATCH --partition=gpu
#SBATCH --time=0-01:00:00
##SBATCH --account=<account>
 
module load miniconda3
 
source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 
conda activate cryolo-gpu
 
cryolo_gui.py config config_cryolo.json 220 \
--filter LOWPASS \
--low_pass_cutoff 0.1 \
--num_cpu ${SLURM_NTASKS_PER_NODE:-1}
 
conda deactivate