lumi-tools/23.02 (lumi-tools-23.02.eb)
This software is archived in the LUMI-SoftwareStack GitHub repository as easybuild/easyconfigs/__archive__/l/lumi-tools/lumi-tools-23.02.eb. The corresponding module would be lumi-tools/23.02.
# Contributed by Kurt Lust for the LUMI project
easyblock = 'Bundle'
local_allocations_version = '20230210'
local_allocations_commit = 'c293356'
name = 'lumi-tools'
version = '23.02'
homepage = 'https://lumi-supercomputer.github.io/LUMI-EasyBuild-docs/l/lumi-tools/'
whatis = [
'Description: Provides commands to check quota and allocations on LUMI.'
]
description = """
This module provides several commands to check the state of your account:
* lumi-workspaces: to print an overview of the quota and allocations for all your projects
* lumi-quota: to check your quota
* lumi-allocations: to check your remaining allocations
* lumi-check-quota: to print a warning when you are over your quota
The check of the allocations is currently done based on pre-stored data. That
data is refreshed periodically, but the data can be out-of-date, especially
if the scripts that build up the cache fail. Currently the tool is not
able to show when the data was collected, so the results may be wrong without
warning.
The checks done by check-quota are also based on cached data that in principle
is refreshed every hour.
Note that cleaning up files may not have an immediate effect on the quota reported
by any of these tools. The lumi-quota and lumi-workspaces tools talk directly to
Lustre but there may be some delay also in the results returned by Lustre.
The lumi-check-quota tool uses cached data that is refresed once an hour and even just
after a refresh the result may not be fully accurate due to the possible delays
in Lustre reporting.
"""
usage = """
To quickly print an overview of quota and allocations, simply run
lumi-workspaces
The lumi-quota command comes in three different forms:
* lumi-quota : Shows your workspaces
* lumi-quota -v : Detailed quota information
* lumi-quota -p prj : Show quota of project prj
To check all your remaining allocations, simply run
lumi-allocations
To check only the allocations for a specific project , run
lumi-alllocations -p project_465000000
(replacing the last argument with the project for which you want the result).
lumi-allocations will also print the data at which the data and time at which the data
was last gathered so you can know if the synchronisation might have failed.
To only get the most important warnings about quota use and billing units for
storage, run
lumi-check-quota
"""
software_license_urls = [
'https://github.com/Lumi-supercomputer/lumi-allocations/blob/main/LICENSE',
]
toolchain = SYSTEM
local_quota_script = """#!/bin/bash
# Exit on errors, exit when accessing undefined variables
set -o errexit
set -o nounset
#set -o xtrace
#----------------------------------------------------------------------
# Global variable declarations
#----------------------------------------------------------------------
# Set needed variables
# Lustre project id mappings offsets
declare -A offsets=( ["users"]="1000000000" \
["projappl"]="2000000000" \
["scratch"]="3000000000" \
["flash"]="3000000000" \
)
# User specific values
username=$(whoami)
uid=$(id -u ${username})
homedir="/users/${username}"
# Active groups with scratch folder
active_groups=()
# Output formatting strings
desc_format="%-40s %8s/%-6s %8s/%s"
long_format="%-40s %8s %5s %5s %5s %5s %5s"
# Formatting used, default is desc, option -v sets this to long
formatting=desc
# Are we printing just a single project info (option -p)?
single_project=false
#----------------------------------------------------------------------
# Function declarations
#----------------------------------------------------------------------
# Usage
function usage() {
echo "This help script is used to manage your workspaces"
echo " $(basename $0) : Shows your workspaces"
echo " $(basename $0) -v : Detailed quota information"
echo " $(basename $0) -p prj : Show quota of project prj"
exit 0
}
# Quota query function
# Arguments: 1. Lustre project id
# 2. Directory
function quotaquery() {
local quota=$(lfs quota -q -p ${1} ${2} 2> /dev/null)
if [[ "${quota[@]}" == *"errors happened"* ]]; then
echo "${2}: Errors while reading quota (permission denied?)"
exit 1
fi
case "${formatting}" in
desc)
IFS=' '; local values=($quota); unset IFS
printf "${desc_format}" \
${2} \
$(numfmt --to si --from iec ${values[1]%\*}K) \
$(numfmt --to si --from iec ${values[2]%\*}K) \
$(numfmt --to si ${values[5]%\*}) \
$(numfmt --to si ${values[6]%\*})
;;
long)
IFS=' '; local values=($quota); unset IFS
printf "${long_format}" \
${values[0]} \
$(numfmt --to si --from iec ${values[1]%\*}K) \
$(numfmt --to si --from iec ${values[2]%\*}K) \
$(numfmt --to si --from iec ${values[3]%\*}K) \
$(numfmt --to si ${values[5]%\*}) \
$(numfmt --to si ${values[6]%\*}) \
$(numfmt --to si ${values[7]%\*})
;;
esac
}
# Print separator line
function print_line() {
case "${formatting}" in
desc)
echo "----------------------------------------------------------------------"
;;
long)
echo "-------------------------------------------------------------------------------"
esac
}
# Print project quota
# Arguments: 1. disk area (scratch or projappl), these have to be also valid
# keys in offsets array
# 2. project
function print_quota() {
local gid=$(getent group ${2} | cut -d : -f3)
if [[ -d "/${1}/${2}" ]]; then
echo "$(quotaquery $(( ${gid} + ${offsets[${1}]} )) /${1}/${2})"
fi
}
function print_quota2() {
local gid=$(getent group ${3} | cut -d : -f3)
if [[ -d "${2}/${1}/${3}" ]]; then
echo "$(quotaquery $(( ${gid} + ${offsets[${1}]} )) ${2}/${1}/${3})"
fi
}
# Print all active projects
# Arguments: 1... list of active projects
function print_projects() {
for grp in "${@}"; do
print_line
echo "Project: ${grp}"
echo
print_quota projappl ${grp}
print_quota scratch ${grp}
print_quota2 flash /pfs/lustref1 ${grp}
done
}
# Print home directory quota
function print_home() {
print_line
echo "Personal home folder"
echo
echo "$(quotaquery $(( ${uid} + ${offsets[users]} )) ${homedir} ${1})"
}
#----------------------------------------------------------------------
# Main script execution
#----------------------------------------------------------------------
# Fill in the active project info
# Each FMI project has a symlink in /scratch folder
my_groups=$(groups)
# DEBUG projects my_groups="ilvonens p_installation_spack project_2001659 project_2001981 project_2003573"
# Process arguments
while getopts "p:hv" arg; do
case "$arg" in
h*)
usage
;;
p)
my_groups=(${OPTARG})
single_project=true
;;
v)
formatting="long"
;;
esac
done
shift $((OPTIND-1))
# Filter active projects, they should have /scratch folder
for g in ${my_groups}; do
if [[ -d "/scratch/${g}" ]]; then
active_groups+=(${g})
fi
done
# Check that the -p argument is actually an active project
if [[ "${single_project:-false}" == "true" && ! -d "/scratch/${my_groups[0]}" ]]; then
echo "${my_groups[@]} is not an active project with quota"
exit 1
fi
# Print the description line
case "${formatting}" in
desc)
echo ;
echo "Disk area Capacity(used/max) Files(used/max)"
;;
long)
echo ;
printf "${long_format}" "Filesystem" "Used" "Quota" "Limit" "Files" "Quota" "Limit"
echo ;
;;
esac
if [[ "${single_project:-false}" == "false" ]]; then
print_home desc
fi
print_projects "${active_groups[@]}"
print_line
"""
local_workspaces_script = """#!/bin/bash
# Exit on errors, exit when accessing undefined variables
set -o errexit
set -o nounset
#set -o xtrace
# Usage
function usage() {
printf "This help script returns quota and allocation information about your workspaces\n"
printf "\nIt takes no further arguments in its current implementation.\n\n"
exit 0
}
while getopts "h" arg; do
case "$arg" in
h*)
usage
;;
esac
done
echo -e "\nQuota for your projects:"
lumi-quota
echo -e "\nStatus of your allocations:\n"
lumi-allocations
printf "\n"
"""
local_check_quota_script = """#!/bin/bash --norc
if [ -f /var/lib/user_info/users/$USER/$USER.json ]
then
/usr/bin/jq -j -r 'if .home_quota.block_quota_used >= .home_quota.block_quota_soft then "\u001b[31mWARNING: home directory space quota exceeded\u001b[0m\n" else "" end, if .home_quota.inode_quota_used >= .home_quota.inode_quota_soft then "\u001b[31mWARNING: home directory file count quota exceeded\u001b[0m\n" else "" end' < /var/lib/user_info/users/$USER/$USER.json
fi
for group in $(/usr/bin/groups)
do
if [ -f /var/lib/project_info/users/$group/$group.json ]
then
/usr/bin/jq 'if .billing.storage_hours.alloc == 0 then "1" else "" end' < /var/lib/project_info/users/$group/$group.json | grep -q 1 && continue
/usr/bin/jq -j -r 'if .storage_quotas.directories.scratch.block_quota_used >= .storage_quotas.directories.scratch.block_quota_soft then "\u001b[31mWARNING: "+.name+" scratch directory space quota exceeded\u001b[0m\n" else "" end, if .storage_quotas.directories.scratch.inode_quota_used >= .storage_quotas.directories.scratch.inode_quota_soft then "\u001b[31mWARNING: "+.name+" scratch directory file count quota exceeded\u001b[0m\n" else "" end' < /var/lib/project_info/users/$group/$group.json
/usr/bin/jq -j -r 'if .billing.storage_hours.remaining < 0 then "\u001b[31mWARNING: "+.name+" is out of storage hours\u001b[0m\n" else "" end' < /var/lib/project_info/users/$group/$group.json
fi
done
"""
components = [
('lumi-allocations', local_allocations_version, {
'easyblock': 'Tarball',
'sources': [{
'filename': '%(name)s-%(version)s.tar.gz',
'git_config': {
'url': 'https://github.com/Lumi-supercomputer',
'repo_name': '%(name)s',
'commit': local_allocations_commit,
'keep_git_dir': False,
},
}],
'install_type': 'merge',
'start_dir': '%(namelower)s'
}),
]
postinstallcmds = [
'cd %(installdir)s/bin ; cat >lumi-quota <<EOF\n' + local_quota_script.replace('$', '\$') + '\nEOF\n',
'cd %(installdir)s/bin ; cat >lumi-workspaces <<EOF\n' + local_workspaces_script.replace('$', '\$') + '\nEOF\n',
'cd %(installdir)s/bin ; cat >lumi-check-quota <<EOF\n' + local_check_quota_script.replace('$', '\$') + '\nEOF\n',
'cd %(installdir)s/bin ; chmod a+rx lumi-quota lumi-workspaces lumi-check-quota',
'mkdir -p %(installdir)s/share/licenses/lumi-allocations',
'cd %(installdir)s ; mv LICENSE share/licenses/lumi-allocations',
]
sanity_check_paths = {
'files': ['bin/lumi-quota', 'bin/lumi-allocations'],
'dirs': []
}
sanity_check_commands = [
'lumi-workspaces -h',
'lumi-quota -h',
'lumi-allocations -h'
]
modluafooter = """
add_property("lmod","sticky")
"""
moduleclass = 'tools'