diff options
author | 2016-12-18 14:08:30 +0000 | |
---|---|---|
committer | 2016-12-18 14:08:30 +0000 | |
commit | 1eca098ae348b789976fb0db0ddcaca80f841853 (patch) | |
tree | 9ca22b3625dd486f727a6b05faf92e2bf0d30a1e | |
parent | 2016-12-18 13:42:14 UTC (diff) | |
parent | scons-utils.eclass: Switch to get_nproc from multiprocessing.eclass (diff) | |
download | gentoo-1eca098ae348b789976fb0db0ddcaca80f841853.tar.gz gentoo-1eca098ae348b789976fb0db0ddcaca80f841853.tar.bz2 gentoo-1eca098ae348b789976fb0db0ddcaca80f841853.zip |
Merge updates from master
-rw-r--r-- | eclass/multiprocessing.eclass | 53 | ||||
-rw-r--r-- | eclass/scons-utils.eclass | 34 | ||||
-rwxr-xr-x | eclass/tests/multiprocessing_makeopts_jobs.sh | 8 | ||||
-rwxr-xr-x | eclass/tests/multiprocessing_makeopts_loadavg.sh | 8 |
4 files changed, 63 insertions, 40 deletions
diff --git a/eclass/multiprocessing.eclass b/eclass/multiprocessing.eclass index 06e004aa1669..70ca475a8c72 100644 --- a/eclass/multiprocessing.eclass +++ b/eclass/multiprocessing.eclass @@ -53,27 +53,60 @@ bashpid() { sh -c 'echo ${PPID}' } +# @FUNCTION: get_nproc +# @USAGE: [${fallback:-1}] +# @DESCRIPTION: +# Attempt to figure out the number of processing units available. +# If the value can not be determined, prints the provided fallback +# instead. If no fallback is provided, defaults to 1. +get_nproc() { + local nproc + + # GNU + if type -P nproc &>/dev/null; then + nproc=$(nproc) + fi + + # BSD + if [[ -z ${nproc} ]] && type -P sysctl &>/dev/null; then + nproc=$(sysctl -n hw.ncpu 2>/dev/null) + fi + + # fallback to python2.6+ + # note: this may fail (raise NotImplementedError) + if [[ -z ${nproc} ]] && type -P python &>/dev/null; then + nproc=$(python -c 'import multiprocessing; print(multiprocessing.cpu_count());' 2>/dev/null) + fi + + if [[ -n ${nproc} ]]; then + echo "${nproc}" + else + echo "${1:-1}" + fi +} + # @FUNCTION: makeopts_jobs -# @USAGE: [${MAKEOPTS}] +# @USAGE: [${MAKEOPTS}] [${inf:-999}] # @DESCRIPTION: # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number # specified therein. Useful for running non-make tools in parallel too. # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the # number as bash normalizes it to [0, 255]. If the flags haven't specified a # -j flag, then "1" is shown as that is the default `make` uses. Since there's -# no way to represent infinity, we return 999 if the user has -j without a number. +# no way to represent infinity, we return ${inf} (defaults to 999) if the user +# has -j without a number. makeopts_jobs() { [[ $# -eq 0 ]] && set -- ${MAKEOPTS} # This assumes the first .* will be more greedy than the second .* # since POSIX doesn't specify a non-greedy match (i.e. ".*?"). local jobs=$(echo " $* " | sed -r -n \ - -e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \ - -e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p') + -e 's:.*[[:space:]](-[a-z]*j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \ + -e "s:.*[[:space:]](-[a-z]*j|--jobs)[[:space:]].*:${2:-999}:p") echo ${jobs:-1} } # @FUNCTION: makeopts_loadavg -# @USAGE: [${MAKEOPTS}] +# @USAGE: [${MAKEOPTS}] [${inf:-999}] # @DESCRIPTION: # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the value set # for load-average. For make and ninja based builds this will mean new jobs are @@ -81,15 +114,17 @@ makeopts_jobs() { # get excessive due to I/O and not just due to CPU load. # Be aware that the returned number might be a floating-point number. Test # whether your software supports that. +# If no limit is specified or --load-average is used without a number, ${inf} +# (defaults to 999) is returned. makeopts_loadavg() { [[ $# -eq 0 ]] && set -- ${MAKEOPTS} # This assumes the first .* will be more greedy than the second .* # since POSIX doesn't specify a non-greedy match (i.e. ".*?"). local lavg=$(echo " $* " | sed -r -n \ - -e 's:.*[[:space:]](-l|--(load-average|max-load)[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+).*:\3:p' \ - -e 's:.*[[:space:]](-l|--(load-average|max-load))[[:space:]].*:999:p') - # Default to 999 since the default is to not use a load limit. - echo ${lavg:-999} + -e 's:.*[[:space:]](-[a-z]*l|--(load-average|max-load)[=[:space:]])[[:space:]]*([0-9]+|[0-9]+\.[0-9]+).*:\3:p' \ + -e "s:.*[[:space:]](-[a-z]*l|--(load-average|max-load))[[:space:]].*:${2:-999}:p") + # Default to ${inf} since the default is to not use a load limit. + echo ${lavg:-${2:-999}} } # @FUNCTION: multijob_init diff --git a/eclass/scons-utils.eclass b/eclass/scons-utils.eclass index 0493ec1dec5a..d504b659df2a 100644 --- a/eclass/scons-utils.eclass +++ b/eclass/scons-utils.eclass @@ -98,6 +98,8 @@ case ${EAPI:-0} in *) die "EAPI ${EAPI} unsupported." esac +inherit multiprocessing + # -- ebuild variables setup -- if [[ -n ${SCONS_MIN_VERSION} ]]; then @@ -149,32 +151,6 @@ escons() { return ${ret} } -# @FUNCTION: _scons_get_default_jobs -# @INTERNAL -# @DESCRIPTION: -# Output the default number of jobs, used if -j is used without -# argument. Tries to figure out the number of logical CPUs, falling -# back to hardcoded constant. -_scons_get_default_jobs() { - local nproc - - if type -P nproc &>/dev/null; then - # GNU - nproc=$(nproc) - elif type -P python &>/dev/null; then - # fallback to python2.6+ - # note: this may fail (raise NotImplementedError) - nproc=$(python -c 'import multiprocessing; print(multiprocessing.cpu_count());' 2>/dev/null) - fi - - if [[ ${nproc} ]]; then - echo $(( nproc + 1 )) - else - # random default - echo 5 - fi -} - # @FUNCTION: _scons_clean_makeopts # @INTERNAL # @USAGE: [makeflags] [...] @@ -217,8 +193,8 @@ _scons_clean_makeopts() { new_makeopts+=( ${1} ${2} ) shift else - # no value means no limit, let's pass a random int - new_makeopts+=( ${1}=$(_scons_get_default_jobs) ) + # no value means no limit, let's pass a default instead + new_makeopts+=( ${1}=$(( $(get_nproc) + 1 )) ) fi ;; # strip other long options @@ -241,7 +217,7 @@ _scons_clean_makeopts() { new_optstr+="j ${2}" shift else - new_optstr+="j $(_scons_get_default_jobs)" + new_optstr+="j $(( $(get_nproc) + 1 ))" fi ;; # otherwise, everything after -j is treated as an arg diff --git a/eclass/tests/multiprocessing_makeopts_jobs.sh b/eclass/tests/multiprocessing_makeopts_jobs.sh index 017d491156a0..ef477277ab3b 100755 --- a/eclass/tests/multiprocessing_makeopts_jobs.sh +++ b/eclass/tests/multiprocessing_makeopts_jobs.sh @@ -9,7 +9,7 @@ inherit multiprocessing test-makeopts_jobs() { local exp=$1; shift - tbegin "makeopts_jobs($*) == ${exp}" + tbegin "makeopts_jobs($1${2+; inf=${2}}) == ${exp}" local act=$(makeopts_jobs "$@") [[ ${act} == "${exp}" ]] tend $? "Got back: ${act}" @@ -31,9 +31,15 @@ tests=( 7 "-l3 --jobs 7 -w" 4 "-j1 -j 2 --jobs 3 --jobs=4" 8 " -j 8 " + 999 "-kj" + 4 "-kj4" + 5 "-kj 5" ) for (( i = 0; i < ${#tests[@]}; i += 2 )) ; do test-makeopts_jobs "${tests[i]}" "${tests[i+1]}" done +# test custom inf value +test-makeopts_jobs 645 "-j" 645 + texit diff --git a/eclass/tests/multiprocessing_makeopts_loadavg.sh b/eclass/tests/multiprocessing_makeopts_loadavg.sh index 12f9d01f9fcd..6b976beb1aef 100755 --- a/eclass/tests/multiprocessing_makeopts_loadavg.sh +++ b/eclass/tests/multiprocessing_makeopts_loadavg.sh @@ -9,7 +9,7 @@ inherit multiprocessing test-makeopts_loadavg() { local exp=$1; shift - tbegin "makeopts_loadavg($*) == ${exp}" + tbegin "makeopts_loadavg($1${2+; inf=${2}}) == ${exp}" local act=$(makeopts_loadavg "$@") [[ ${act} == "${exp}" ]] tend $? "Got back: ${act}" @@ -28,9 +28,15 @@ tests=( 4 "-j1 -j 2 --load-average 3 --load-average=4" 3 " --max-load=3 -x" 8 " -l 8 " + 999 "-kl" + 4 "-kl4" + 5 "-kl 5" ) for (( i = 0; i < ${#tests[@]}; i += 2 )) ; do test-makeopts_loadavg "${tests[i]}" "${tests[i+1]}" done +# test custom inf value +test-makeopts_loadavg 645 "-l" 645 + texit |