Strumenti Utente

Strumenti Sito

Traccia: • CONDA user defined environments • Progetto di migrazione da PBS Pro a SLURM
calcoloscientifico:cluster:softwareapplicativo:conda_user

Differenze

Queste sono le differenze tra la revisione selezionata e la versione attuale della pagina.

Link a questa pagina di confronto

Entrambe le parti precedenti la revisioneRevisione precedente
Prossima revisione		Revisione precedente
calcoloscientifico:cluster:softwareapplicativo:conda_user [18/03/2021 11:34] – fabio.spatarocalcoloscientifico:cluster:softwareapplicativo:conda_user [25/02/2026 15:08] (versione attuale) – fabio.spataro
Linea 18: Linea 18:
 CONDAENV_NAME='template' CONDAENV_NAME='template'
 CONDAENV_VERSION=$(date '+%Y.%m.%d') CONDAENV_VERSION=$(date '+%Y.%m.%d')
-PYTHON_VERSION='3.6'+PYTHON_VERSION='3.9'
  
 CONDAENV="${CONDAENV_NAME}-${CONDAENV_VERSION}" CONDAENV="${CONDAENV_NAME}-${CONDAENV_VERSION}"
Linea 38: Linea 38:
     CONDAENV_PREFIX="$CONDA_ENVS_PATH/$CONDAENV"     CONDAENV_PREFIX="$CONDA_ENVS_PATH/$CONDAENV"
     CONDAENV_SYMLNK="$CONDA_ENVS_PATH/$CONDAENV_NAME"     CONDAENV_SYMLNK="$CONDA_ENVS_PATH/$CONDAENV_NAME"
 +fi
 +
 +if [ $UID = 0 ]; then
 +    conda --version
 +    conda update conda
 +    conda --version
 +    echo
 fi fi
  
Linea 43: Linea 50:
 echo "CONDAENV_SYMLNK: $CONDAENV_SYMLNK" echo "CONDAENV_SYMLNK: $CONDAENV_SYMLNK"
 echo echo
-echo -n "Should we proceed with the creation of the condaenv '$CONDAENV_NAME' (yes/no)? "+ 
 +if [ -d "$CONDAENV_PREFIX" ]; then 
 +    echo -n "Should we remove the already existing condaenv '$CONDAENV_NAME' (yes/no)? " 
 + 
 +    read ans 
 + 
 +    case "$ans" in 
 +        [yY]|[yY][Ee][Ss]) 
 +        conda env remove --name "$CONDAENV" 
 +        ;; 
 +        [nN]|[nN][oO]) 
 +        ;; 
 +        *) 
 +        echo 
 +        exec "$0" 
 +        exit 
 +        ;; 
 +    esac 
 +fi 
 + 
 +if [ -d "$CONDAENV_PREFIX" ]; then 
 +    echo -n "Should we proceed with the modification of the condaenv '$CONDAENV_NAME' (yes/no)? " 
 +else 
 +    echo -n "Should we proceed with the creation of the condaenv '$CONDAENV_NAME' (yes/no)? " 
 +fi
  
 read ans read ans
Linea 62: Linea 93:
 conda info conda info
  
-if [ $UID = 0 ]; then +if [ ! -d "$CONDAENV_PREFIX" ]; then 
-    conda --version +    conda create --yes --prefix "$CONDAENV_PREFIX" python${PYTHON_VERSION:+=$PYTHON_VERSION}
-    conda update conda +
-    conda --version+
 fi fi
- 
-if [ -d "$CONDAENV_PREFIX" ]; then 
-    conda env remove --name "$CONDAENV" 
-fi 
- 
-conda create --yes --name "$CONDAENV" python=$PYTHON_VERSION 
  
 conda activate "$CONDAENV" conda activate "$CONDAENV"
  
 conda install --yes --name "$CONDAENV" \ conda install --yes --name "$CONDAENV" \
---channel conda-forge \ +    --channel conda-forge \ 
-7zip+    xz
  
 conda list || true conda list || true
Linea 100: Linea 123:
 It is also possible to install python packages using the "pip" command. It is also possible to install python packages using the "pip" command.
  
 +==== Example with conda and pip ====
 +
 +General template that can be used to create personal or group environments:
 +
 +<code bash hpc-make-condaenv-networkx>
 +#!/bin/bash
 +
 +set -e
 +
 +PYTHON_VERSION='3.12'
 +CONDAENV_NAME='networkx'
 +CONDAENV_RELEASE='py3.12'
 +CONDAENV_DEFAULT='yes'
 +NETWORKX_VERSION='3.6.1'
 +
 +help() {
 +    cat <<EOF
 +Synopsis:
 +  $(basename $0) [--dry-run]
 +  $(basename $0) [--pip-dry-run]
 +  $(basename $0) --help
 +
 +Usage:
 +  $(basename $0)
 +  $(basename $0) --dry-run
 +  $(basename $0) --pip-dry-run
 +
 +Example:
 +  $(basename $0)
 +  $(basename $0) --dry-run
 +  $(basename $0) --pip-dry-run
 +
 +Options:
 +  -d, --dry-run                 Only display what would have been done by conda
 +      --pip-dry-run             Only display what would have been done by pip
 +  -h, --help                    Display a help message and quit
 +EOF
 +}
 +
 +die() {
 +    if [ $# -ge 1 ]; then
 +        echo "$(basename "$0"): $@" >&2
 +    fi
 +    exit 1
 +}
 +
 +# Command line options
 +opt=$(getopt \
 +    --options hd \
 +    --longoptions help,pip-dry-run,dry-run \
 +    --name $(basename $0) \
 +    -- \
 +    "$@")
 +
 +retval=$?
 +
 +if [ $retval -ne 0 ]; then
 +    exit $retval
 +fi
 +
 +eval set -- "$opt"
 +
 +while true; do
 +    case "$1" in
 +        -h|--help)
 +            shift
 +            help
 +            exit;;
 +
 +        -d|--dry-run)
 +            shift
 +            opt_dry_run=true;;
 +
 +        --pip-dry-run)
 +            shift
 +            opt_pip_dry_run=true;;
 +
 +        --)
 +            shift
 +            break;;
 +
 +        *)
 +            die 'options parsing error'
 +            exit 1;;
 +    esac
 +done
 +
 +# CONDA dry run
 +if [ -n "$opt_dry_run" ]; then
 +    unset PYTHON_VERSION
 +fi
 +
 +# PIP dry run
 +if [ -n "$opt_pip_dry_run" ]; then
 +    unset NETWORKX_VERSION
 +fi
 +
 +# CONDA environment name
 +if [ -n "$NETWORKX_VERSION" ]; then
 +    CONDAENV_VERSION="$NETWORKX_VERSION"
 +else
 +    if [ -n "$PYTHON_VERSION" ]; then
 +        PY=${PYTHON_VERSION:+$(echo $PYTHON_VERSION | sed -re 's/([0-9]+)\.([0-9]+).*/py\1.\2/')}
 +    fi
 +    CONDAENV_VERSION=${PY:+${PY}-}$(date '+%Y.%m.%d')
 +fi
 +if [ -n "$CONDAENV_RELEASE" ]; then
 +    CONDAENV_VERSION="${CONDAENV_VERSION}-${CONDAENV_RELEASE}"
 +fi
 +CONDAENV="${CONDAENV_NAME}-${CONDAENV_VERSION}"
 +
 +module load miniconda3
 +
 +source "$CONDA_PREFIX/etc/profile.d/conda.sh"
 +
 +if [ $UID = 0 ]; then
 +    CONDAENV_PREFIX="/hpc/share/tools/miniconda3/envs/$CONDAENV"
 +    CONDAENV_SYMLNK="/hpc/share/tools/miniconda3/envs/$CONDAENV_NAME"
 +else
 +    if [ -z "$CONDA_ENVS_PATH" ]; then
 +        echo 'Error: the CONDA_ENVS_PATH envarionment variable is not set' 1>&2
 +        exit 1
 +    fi
 +    if [ -z "$CONDA_PKGS_DIRS" ]; then
 +        echo 'Error: the CONDA_PKGS_DIRS envarionment variable is not set' 1>&2
 +        exit 1
 +    fi
 +    echo "CONDA_ENVS_PATH: $CONDA_ENVS_PATH"
 +    echo "CONDA_PKGS_DIRS: $CONDA_PKGS_DIRS"
 +    CONDAENV_PREFIX="$CONDA_ENVS_PATH/$CONDAENV"
 +    CONDAENV_SYMLNK="$CONDA_ENVS_PATH/$CONDAENV_NAME"
 +fi
 +
 +echo "CONDAENV_PREFIX: $CONDAENV_PREFIX"
 +if echo "$CONDAENV_DEFAULT" | egrep -q '^[yY][Ee][Ss]$'; then
 +    echo "CONDAENV_SYMLNK: $CONDAENV_SYMLNK"
 +fi
 +echo
 +
 +if [ -z "$opt_dry_run" ]; then
 +    if [ -d "$CONDAENV_PREFIX" ]; then
 +        echo -n "Should we remove the already existing condaenv '$CONDAENV_NAME' (yes/no)? "
 +
 +        read ans
 +
 +        case "$ans" in
 +            [yY]|[yY][Ee][Ss])
 +                conda env remove --name "$CONDAENV"
 +                ;;
 +            [nN]|[nN][oO])
 +                ;;
 +            *)
 +                echo
 +                exec "$0"
 +                exit
 +                ;;
 +        esac
 +    fi
 +
 +    if [ -d "$CONDAENV_PREFIX" ]; then
 +        echo -n "Should we proceed with the modification of the condaenv '$CONDAENV_NAME' (yes/no)? "
 +    else
 +        echo -n "Should we proceed with the creation of the condaenv '$CONDAENV_NAME' (yes/no)? "
 +    fi
 +
 +    read ans
 +
 +    case "$ans" in
 +        [yY]|[yY][Ee][Ss])
 +            ;;
 +        [nN]|[nN][oO])
 +            exit 1
 +            ;;
 +        *)
 +            echo
 +            exec "$0"
 +            exit
 +            ;;
 +    esac
 +fi
 +
 +conda info
 +
 +if [ ! -d "$CONDAENV_PREFIX" ]; then
 +    # CONDA create
 +    conda create \
 +        --yes \
 +        ${opt_dry_run:+--dry-run} \
 +        --prefix "$CONDAENV_PREFIX" \
 +        --override-channels \
 +        --channel conda-forge \
 +        python${PYTHON_VERSION:+=$PYTHON_VERSION} \
 +        pip \
 +        matplotlib
 +fi
 +
 +if [ -n "$opt_dry_run" ]; then
 +    conda deactivate
 +    exit
 +fi
 +
 +CONDADEF_PIP="$(which pip)"
 +
 +conda activate "$CONDAENV"
 +
 +CONDAENV_PIP="$(which pip)"
 +
 +# PIP check
 +if [ "$CONDADEF_PIP" = "$CONDAENV_PIP" ]; then
 +    echo "Error: condaenv pip is conda default pip: '$CONDAENV_PIP'" 1>&2
 +    exit 1
 +fi
 +
 +# PIP install or upgrade pip
 +pip install --upgrade ${opt_pip_dry_run:+--dry-run} pip
 +
 +# PIP install or upgrade
 +pip install --upgrade ${opt_pip_dry_run:+--dry-run} \
 +    networkx${NETWORKX_VERSION:+==$NETWORKX_VERSION}
 +
 +conda list || true
 +
 +conda deactivate
 +
 +# PIP dry run
 +if [ -n "$opt_pip_dry_run" ]; then
 +    exit
 +fi
 +
 +# CONDA environment symbolic link
 +if echo "$CONDAENV_DEFAULT" | egrep -q '^[yY][Ee][Ss]$'; then
 +    rm -f "$CONDAENV_SYMLNK"
 +    ln -s "$CONDAENV_PREFIX" "$CONDAENV_SYMLNK"
 +    symlinks -cs $(dirname "$CONDAENV_PREFIX")
 +    ls -lard "$CONDAENV_SYMLNK"*
 +fi
 +</code>
 +
 +This script is located in
 +
 +<code>
 +/hpc/share/tools/conda/bin/hpc-make-condaenv-networkx
 +</code>
  
 ==== Creating a user environment ==== ==== Creating a user environment ====
Linea 116: Linea 382:
  
 ==== Working with conda ==== ==== Working with conda ====
 +
 +With conda, you can create, export, list, remove, and update that have different versions of Python and/or packages installed in them. Switching or moving between environments is called activating the environment.
  
 === Interactive work session === === Interactive work session ===
 +
 +If you want to work at group level change group otherwise skip this step:
  
 <code> <code>
-[<USER>@ui01 ~]$ module load miniconda3+newgrp <GROUP> 
 +</code>
  
-[<USER>@ui01 ~]$ source "$CONDA_PREFIX/etc/profile.d/conda.sh"+Typical work session with conda (CPU):
  
-[<USER>@ui01 ~]$ conda --help+<code> 
 +[<USER>@ui01 ~]$ srun --nodes=1 --ntasks-per-node=2 --partition=cpu --qos=cpu --mem=8G --pty bash
  
-[<USER>@ui01 ~]$ conda --version +[<USER>@wn80 ~]$ module load miniconda3
-conda 4.9.2+
  
-[<USER>@ui01 ~]$ conda info+[<USER>@wn80 ~]$ source "$CONDA_PREFIX/etc/profile.d/conda.sh"
  
-[<USER>@ui01 ~]$ conda activate+[<USER>@wn80 ~]$ conda --help
  
-(base) [<USER>@ui01 ~]$ which python+[<USER>@wn80 ~]$ conda --version 
 +conda 25.7.0 
 + 
 +[<USER>@wn80 ~]$ conda info 
 + 
 +[<USER>@wn80 ~]$ conda activate 
 + 
 +(base) [<USER>@wn80 ~]$ which python
 /hpc/share/tools/miniconda3/bin/python /hpc/share/tools/miniconda3/bin/python
  
-(base) [<USER>@ui01 ~]$ python -V +(base) [<USER>@wn80 ~]$ python -V 
-Python 3.7.4+Python 3.9.18 
 + 
 +(base) [<USER>@wn80 ~]$ conda create --name my_conda_env [...] 
 +(base) [<USER>@wn80 ~]$ conda activate my_conda_env 
 + 
 +(my_conda_env) [<USER>@wn80 ~]$ [...] 
 +(my_conda_env) [<USER>@wn80 ~]$ conda deactivate 
 + 
 +(base) [<USER>@wn80 ~]$ conda deactivate 
 + 
 +[<USER>@wn80 ~]$ exit 
 + 
 +[<USER>@ui01 ~]$ 
 +</code> 
 + 
 +Typical work session with conda (GPU): 
 + 
 +<code> 
 +[<USER>@ui01 ~]$ srun --nodes=1 --ntasks-per-node=2 --partition=gpu --qos=gpu --gres=gpu:p100:1 --mem=8G --pty bash 
 + 
 +[<USER>@wn41 ~]$ module load miniconda3 
 + 
 +[...]
  
-(base) [<USER>@ui01 ~]$ conda deactivate+[<USER>@wn41 ~]$ exit
  
 [<USER>@ui01 ~]$ [<USER>@ui01 ~]$
Linea 152: Linea 452:
 === Alternate channels === === Alternate channels ===
  
-To search for alternate channels that may provide the conda package you're looking for, navigate to [[https://anaconda.org]] and use the search bar at the top of the page.+To search for alternate channels that may provide the conda package you're looking for, navigate to [[https://anaconda.org|anaconda.org]] and use the search bar at the top of the page.
  
 To add a new channel: To add a new channel:
Linea 160: Linea 460:
 </code> </code>
  
-=== Environment ===+=== Remove unused packages and caches ===
  
-With conda, you can create, export, list, remove, and update [[https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments|environments]] that have different versions of Python and/or packages installed in them. Switching or moving between environments is called activating the environment.+To remove unused packages and caches 
 + 
 +<code> 
 +conda clean --all 
 +</code>
  
 === Environments list === === Environments list ===
calcoloscientifico/cluster/softwareapplicativo/conda_user.1616063648.txt.gz · Ultima modifica: da fabio.spataro
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki