Strumenti Utente

Strumenti Sito

Traccia: • Progetto di migrazione da PBS Pro a SLURM • 2019 - Utilizzo partizione KNL • CONDA user defined environments
calcoloscientifico:cluster:softwareapplicativo:conda_user

Differenze

Queste sono le differenze tra la revisione selezionata e la versione attuale della pagina.

Link a questa pagina di confronto

Entrambe le parti precedenti la revisioneRevisione precedente
Prossima revisione		Revisione precedente
calcoloscientifico:cluster:softwareapplicativo:conda_user [18/09/2025 16:20] – [Working with conda] fabio.spatarocalcoloscientifico:cluster:softwareapplicativo:conda_user [25/02/2026 15:08] (versione attuale) – fabio.spataro
Linea 123: Linea 123:
 It is also possible to install python packages using the "pip" command. It is also possible to install python packages using the "pip" command.
  
 +==== Example with conda and pip ====
 +
 +General template that can be used to create personal or group environments:
 +
 +<code bash hpc-make-condaenv-networkx>
 +#!/bin/bash
 +
 +set -e
 +
 +PYTHON_VERSION='3.12'
 +CONDAENV_NAME='networkx'
 +CONDAENV_RELEASE='py3.12'
 +CONDAENV_DEFAULT='yes'
 +NETWORKX_VERSION='3.6.1'
 +
 +help() {
 +    cat <<EOF
 +Synopsis:
 +  $(basename $0) [--dry-run]
 +  $(basename $0) [--pip-dry-run]
 +  $(basename $0) --help
 +
 +Usage:
 +  $(basename $0)
 +  $(basename $0) --dry-run
 +  $(basename $0) --pip-dry-run
 +
 +Example:
 +  $(basename $0)
 +  $(basename $0) --dry-run
 +  $(basename $0) --pip-dry-run
 +
 +Options:
 +  -d, --dry-run                 Only display what would have been done by conda
 +      --pip-dry-run             Only display what would have been done by pip
 +  -h, --help                    Display a help message and quit
 +EOF
 +}
 +
 +die() {
 +    if [ $# -ge 1 ]; then
 +        echo "$(basename "$0"): $@" >&2
 +    fi
 +    exit 1
 +}
 +
 +# Command line options
 +opt=$(getopt \
 +    --options hd \
 +    --longoptions help,pip-dry-run,dry-run \
 +    --name $(basename $0) \
 +    -- \
 +    "$@")
 +
 +retval=$?
 +
 +if [ $retval -ne 0 ]; then
 +    exit $retval
 +fi
 +
 +eval set -- "$opt"
 +
 +while true; do
 +    case "$1" in
 +        -h|--help)
 +            shift
 +            help
 +            exit;;
 +
 +        -d|--dry-run)
 +            shift
 +            opt_dry_run=true;;
 +
 +        --pip-dry-run)
 +            shift
 +            opt_pip_dry_run=true;;
 +
 +        --)
 +            shift
 +            break;;
 +
 +        *)
 +            die 'options parsing error'
 +            exit 1;;
 +    esac
 +done
 +
 +# CONDA dry run
 +if [ -n "$opt_dry_run" ]; then
 +    unset PYTHON_VERSION
 +fi
 +
 +# PIP dry run
 +if [ -n "$opt_pip_dry_run" ]; then
 +    unset NETWORKX_VERSION
 +fi
 +
 +# CONDA environment name
 +if [ -n "$NETWORKX_VERSION" ]; then
 +    CONDAENV_VERSION="$NETWORKX_VERSION"
 +else
 +    if [ -n "$PYTHON_VERSION" ]; then
 +        PY=${PYTHON_VERSION:+$(echo $PYTHON_VERSION | sed -re 's/([0-9]+)\.([0-9]+).*/py\1.\2/')}
 +    fi
 +    CONDAENV_VERSION=${PY:+${PY}-}$(date '+%Y.%m.%d')
 +fi
 +if [ -n "$CONDAENV_RELEASE" ]; then
 +    CONDAENV_VERSION="${CONDAENV_VERSION}-${CONDAENV_RELEASE}"
 +fi
 +CONDAENV="${CONDAENV_NAME}-${CONDAENV_VERSION}"
 +
 +module load miniconda3
 +
 +source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 +
 +if [ $UID = 0 ]; then
 +    CONDAENV_PREFIX="/hpc/share/tools/miniconda3/envs/$CONDAENV"
 +    CONDAENV_SYMLNK="/hpc/share/tools/miniconda3/envs/$CONDAENV_NAME"
 +else
 +    if [ -z "$CONDA_ENVS_PATH" ]; then
 +        echo 'Error: the CONDA_ENVS_PATH envarionment variable is not set' 1>&2
 +        exit 1
 +    fi
 +    if [ -z "$CONDA_PKGS_DIRS" ]; then
 +        echo 'Error: the CONDA_PKGS_DIRS envarionment variable is not set' 1>&2
 +        exit 1
 +    fi
 +    echo "CONDA_ENVS_PATH: $CONDA_ENVS_PATH"
 +    echo "CONDA_PKGS_DIRS: $CONDA_PKGS_DIRS"
 +    CONDAENV_PREFIX="$CONDA_ENVS_PATH/$CONDAENV"
 +    CONDAENV_SYMLNK="$CONDA_ENVS_PATH/$CONDAENV_NAME"
 +fi
 +
 +echo "CONDAENV_PREFIX: $CONDAENV_PREFIX"
 +if echo "$CONDAENV_DEFAULT" | egrep -q '^[yY][Ee][Ss]$'; then
 +    echo "CONDAENV_SYMLNK: $CONDAENV_SYMLNK"
 +fi
 +echo
 +
 +if [ -z "$opt_dry_run" ]; then
 +    if [ -d "$CONDAENV_PREFIX" ]; then
 +        echo -n "Should we remove the already existing condaenv '$CONDAENV_NAME' (yes/no)? "
 +
 +        read ans
 +
 +        case "$ans" in
 +            [yY]|[yY][Ee][Ss])
 +                conda env remove --name "$CONDAENV"
 +                ;;
 +            [nN]|[nN][oO])
 +                ;;
 +            *)
 +                echo
 +                exec "$0"
 +                exit
 +                ;;
 +        esac
 +    fi
 +
 +    if [ -d "$CONDAENV_PREFIX" ]; then
 +        echo -n "Should we proceed with the modification of the condaenv '$CONDAENV_NAME' (yes/no)? "
 +    else
 +        echo -n "Should we proceed with the creation of the condaenv '$CONDAENV_NAME' (yes/no)? "
 +    fi
 +
 +    read ans
 +
 +    case "$ans" in
 +        [yY]|[yY][Ee][Ss])
 +            ;;
 +        [nN]|[nN][oO])
 +            exit 1
 +            ;;
 +        *)
 +            echo
 +            exec "$0"
 +            exit
 +            ;;
 +    esac
 +fi
 +
 +conda info
 +
 +if [ ! -d "$CONDAENV_PREFIX" ]; then
 +    # CONDA create
 +    conda create \
 +        --yes \
 +        ${opt_dry_run:+--dry-run} \
 +        --prefix "$CONDAENV_PREFIX" \
 +        --override-channels \
 +        --channel conda-forge \
 +        python${PYTHON_VERSION:+=$PYTHON_VERSION} \
 +        pip \
 +        matplotlib
 +fi
 +
 +if [ -n "$opt_dry_run" ]; then
 +    conda deactivate
 +    exit
 +fi
 +
 +CONDADEF_PIP="$(which pip)"
 +
 +conda activate "$CONDAENV"
 +
 +CONDAENV_PIP="$(which pip)"
 +
 +# PIP check
 +if [ "$CONDADEF_PIP" = "$CONDAENV_PIP" ]; then
 +    echo "Error: condaenv pip is conda default pip: '$CONDAENV_PIP'" 1>&2
 +    exit 1
 +fi
 +
 +# PIP install or upgrade pip
 +pip install --upgrade ${opt_pip_dry_run:+--dry-run} pip
 +
 +# PIP install or upgrade
 +pip install --upgrade ${opt_pip_dry_run:+--dry-run} \
 +    networkx${NETWORKX_VERSION:+==$NETWORKX_VERSION}
 +
 +conda list || true
 +
 +conda deactivate
 +
 +# PIP dry run
 +if [ -n "$opt_pip_dry_run" ]; then
 +    exit
 +fi
 +
 +# CONDA environment symbolic link
 +if echo "$CONDAENV_DEFAULT" | egrep -q '^[yY][Ee][Ss]$'; then
 +    rm -f "$CONDAENV_SYMLNK"
 +    ln -s "$CONDAENV_PREFIX" "$CONDAENV_SYMLNK"
 +    symlinks -cs $(dirname "$CONDAENV_PREFIX")
 +    ls -lard "$CONDAENV_SYMLNK"*
 +fi
 +</code>
 +
 +This script is located in
 +
 +<code>
 +/hpc/share/tools/conda/bin/hpc-make-condaenv-networkx
 +</code>
  
 ==== Creating a user environment ==== ==== Creating a user environment ====
Linea 150: Linea 393:
 </code> </code>
  
-Typical work session with conda:+Typical work session with conda (CPU):
  
 <code> <code>
-[<USER>@ui01 ~]$ srun --nodes=1 --ntasks-per-node=2 --partition=cpu --qos=cpu --pty bash+[<USER>@ui01 ~]$ srun --nodes=1 --ntasks-per-node=2 --partition=cpu --qos=cpu --mem=8G --pty bash
  
 [<USER>@wn80 ~]$ module load miniconda3 [<USER>@wn80 ~]$ module load miniconda3
Linea 174: Linea 417:
 Python 3.9.18 Python 3.9.18
  
-(base) [<USER>@wn80 ~]$ conda create [...] +(base) [<USER>@wn80 ~]$ conda create --name my_conda_env [...] 
-(base) [<USER>@wn80 ~]$ conda activiate [...]+(base) [<USER>@wn80 ~]$ conda activate my_conda_env 
 + 
 +(my_conda_env) [<USER>@wn80 ~]$ [...] 
 +(my_conda_env) [<USER>@wn80 ~]$ conda deactivate
  
 (base) [<USER>@wn80 ~]$ conda deactivate (base) [<USER>@wn80 ~]$ conda deactivate
  
 [<USER>@wn80 ~]$ exit [<USER>@wn80 ~]$ exit
 +
 +[<USER>@ui01 ~]$
 +</code>
 +
 +Typical work session with conda (GPU):
 +
 +<code>
 +[<USER>@ui01 ~]$ srun --nodes=1 --ntasks-per-node=2 --partition=gpu --qos=gpu --gres=gpu:p100:1 --mem=8G --pty bash
 +
 +[<USER>@wn41 ~]$ module load miniconda3
 +
 +[...]
 +
 +[<USER>@wn41 ~]$ exit
  
 [<USER>@ui01 ~]$ [<USER>@ui01 ~]$
calcoloscientifico/cluster/softwareapplicativo/conda_user.1758205209.txt.gz · Ultima modifica: da fabio.spataro
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki