diff --git a/recon_surf/fs_time b/recon_surf/fs_time index 62b3d712..072f91ee 100755 --- a/recon_surf/fs_time +++ b/recon_surf/fs_time @@ -1,27 +1,31 @@ #!/bin/bash # fs_time -VERSION='$Id: fs_time,v 1.0 2024/03/08 15:12:00 kueglerd Exp $' +VERSION="\$Id: fs_time, v1.1 2024/11/13 15:04:00 kueglerd Exp \$" outfile="" key="@#@FSTIME " +flags=() cmd=() verbose=0 +print_uptime=$FSTIME_LOAD -if [[ -z "$FSTIME_LOAD" ]] +if [[ -z "$print_uptime" ]] then # Turn on by default - export FSTIME_LOAD=1 + print_uptime=1 fi function usage() { cat << EOF - fs_time [options] command args options: - -o outputfile : save resource info into outputfile - -k key - -l : report on load averages as from uptime + -o outputfile : save resource info into outputfile instead + -a : append to outputfile (see time) + -k key : key to find timing info in output + -l|--load : report on load averages as from uptime (default: \$FSTIME_LOAD or on) + --no-load : do not report on load averages + --debug : make output more verbose EOF } @@ -44,31 +48,29 @@ Default fs_time Output (see also the manual page for /usr/bin/time): 2. Time stamp at the onset of execution 3. Command name 4. N Number of arguments passed to command -5. e Elapsed real time in seconds . This is the total - amount of time from when the command started to when it ended regardless - of how much of the CPU it got. +5. e Elapsed real time in seconds. This is the total amount of time from when the + command started to when it ended regardless of how much of the CPU it got. 6. S Total number of CPU-seconds that the process spent in kernel mode. 7. U Total number of CPU-seconds that the process spent in user mode. 8. P Percentage of the CPU that this job got, computed as (U+S)/e. 9. M Maximum resident set size of the process during its lifetime, in Kbytes. 10. F Number of major page faults that occurred while the process was running. These are faults where the page has to be read in from disk. -11. R Number of minor, or recoverable, page faults. These are - faults for pages that are not valid but which have not yet been - claimed by other virtual pages. Thus the data in the page is - still valid but the system tables must be updated. -12. W Number of times the process was swapped out of main memory. -13. c Number of times the process was context-switched involuntarily - (because the time slice expired). -14. w Number of waits: times that the program was context-switched voluntarily, - for instance while waiting for an I/O operation to complete. +11. R Number of minor, or recoverable, page faults. These are faults for pages that + are not valid but which have not yet been claimed by other virtual pages. Thus + the data in the page is still valid but the system tables must be updated. +12. W Number of times the process was swapped out of main memory. +13. c Number of times the process was context-switched involuntarily (because the + time slice expired). +14. w Number of waits: times that the program was context-switched voluntarily, for + instance while waiting for an I/O operation to complete. 15. I Number of file system inputs by the process. 16. O Number of file system outputs by the process. 17. L L1 L5 L15 : load averages at 1, 5, and 15 min (with setenv FSTIME_LOAD 1) Example: -fs_time -o resource.dat mri_convert orig.mgz myfile.mgz +fs_time mri_convert orig.mgz myfile.mgz mri_convert orig.mgz myfile.mgz reading from orig.mgz... TR=2730.00, TE=3.44, TI=1000.00, flip angle=7.00 @@ -83,7 +85,7 @@ produces the information about resources. It also creates a file resource.dat with the resource information. In this case, the above is interpreted as: -@#@FSTIME : key for easy searching/grepping +@#@FSTIME : key for easy searching/grepping mri_convert : command that was run 2016:01:21:18:27:08 : time stamp at the onset of execution year:month:day:hour:min:sec N 2 : mri_convert was run with 2 arguments @@ -112,73 +114,50 @@ EOF function arg1err() { # param 1 : flag - echo "ERROR: flag $1 requires one argument" + echo "ERROR: Flag $1 requires one argument!" exit 1 } -inputargs=("$@") -any_help=$(echo "$@" | grep -e -help) -if [[ -n "$any_help" ]] -then - usage - help - exit 0 -fi -any_version=$(echo "$@" | grep -e -version) -if [[ -n "$any_version" ]] -then - echo "$VERSION" - exit 0 -fi - # sourcing FreeSurfer should not be needed at this point, # source $FREESURFER_HOME/sources.sh -cmdline=("$@") while [[ $# != 0 ]] do - flag=$1 + flag="$1" shift case $flag in - -o) - if [[ "$#" -lt 1 ]] ; then arg1err "$flag"; fi - outfile=$1 - shift - ;; - -k) - if [[ "$#" -lt 1 ]] ; then arg1err "$flag" ; fi - key=$1 - shift - ;; - -l|-load) - export FSTIME_LOAD=1 - ;; - -no-load) - export FSTIME_LOAD=0 - ;; - -debug) - verbose=1 - ;; - *) - # must be at the start of the command to run - # put item back into the list - cmd=("$flag" "$@") - break - ;; + -o|-k) if [[ "$#" -lt 1 ]] ; then arg1err "$flag"; fi ;; esac + case "$flag" in + -help|--help) usage ; help ; exit 0 ;; + -version|--version) echo "$VERSION" ; exit 0 ;; + -o) outfile="$1" ; shift ;; + -k) key="$1" ; shift ;; + -a) flags+=("$flag") ;; + -l|-load|--load) print_uptime=1 ;; + -no-load|--no-load) print_uptime=0 ;; + -debug|--debug) verbose=1 ;; + # we must now be at the start of the command to run, so put all items back into the list + *) cmd=("$flag" "$@") ; break ;; + esac done if [[ "$verbose" == 1 ]] then echo "Parameters to fs_time:" - if [[ -n "$outfile" ]] ; then echo "-o $outfile" ; fi - if [[ "$key" != "@#@FSTIME " ]] ; then echo "-k $key" ; fi - echo "FSTIME_LOAD=$FSTIME_LOAD" + if [[ -n "$outfile" ]] ; then echo " -o $outfile" ; fi + if [[ "$key" != "@#@FSTIME " ]] ; then echo " -k $key" ; fi + if [[ "${#flags[@]}" -gt 0 ]] ; then echo " ${flags[*]}" ; fi + if [[ "$print_uptime" != "$FSTIME_LOAD" ]] && [[ -n "$FSTIME_LOAD" ]] ; then + if [[ "$print_uptime" == 1 ]] ; then echo " --load" ; else echo " --noload" ; fi + else + echo " FSTIME_LOAD=$FSTIME_LOAD" ; + fi echo "command:" - echo "${cmd[*]}" + echo " ${cmd[*]}" echo "" fi @@ -186,13 +165,13 @@ fi if [[ "${#cmd[@]}" == 0 ]] then usage - echo "ERROR: no command passed to execute" + echo "ERROR: No command passed to execute!" exit 1 fi if [[ ! -e /usr/bin/time ]] then - echo "ERROR: cannot find /usr/bin/time" + echo "ERROR: Cannot find /usr/bin/time!" exit 1 fi @@ -217,36 +196,43 @@ fi nargs=$((${#cmd[@]} - 1 - npyargs)) -function make_string () +function make_uptime () +{ + # shellcheck disable=SC2207 + uptime_data_array=($(uptime | sed 's/,/ /g')) + echo "L ${uptime_data_array[-3]} ${uptime_data_array[-2]} ${uptime_data_array[-1]}" +} + +function make_fmt () { # param 1 : key # param 2 : command # param 3 : num args - dt=$(date '+%Y:%m:%d:%H:%M:%S') - uptime_data_array=($(uptime | sed 's/,/ /g')) - echo "$1 $dt $2 N $3 ${uptime_data_array[-3]} ${uptime_data_array[-2]} ${uptime_data_array[-1]}" - export upt="L ${uptime_data_array[-3]} ${uptime_data_array[-2]} ${uptime_data_array[-1]}" + # param 4 : additional data + add=$4 + if [[ "${#add}" -gt 0 ]] && [[ "${add:0:1}" != " " ]] ; then add=" $add" ; fi + echo "$1 $(date '+%Y:%m:%d:%H:%M:%S') $2 N $3$add" } -if [[ "$FSTIME_LOAD" == 1 ]] +if [[ "$print_uptime" == 1 ]] then - make_string "@#@FSLOADPRE" "$command" "$nargs" + uptime="$(make_uptime)" + make_fmt "@#@FSLOADPRE" "$command" "$nargs" "$uptime" else - upt="" + uptime="" fi -dt=$(date '+%Y:%m:%d:%H:%M:%S') -fmt="$key $dt $command N $nargs e %e S %S U %U P %P M %M F %F R %R W %W c %c w %w I %I O %O $upt" +fmt="$(make_fmt "$key" "$command" "$nargs" "e %e S %S U %U P %P M %M F %F R %R W %W c %c w %w I %I O %O$uptime")" -timecmd=("/usr/bin/time") -if [[ -n "$outfile" ]] ; then timecmd=("${timecmd[@]}" -o "$outfile"); fi +timecmd=("/usr/bin/time" "${flags[@]}") +if [[ -n "$outfile" ]] ; then timecmd+=(-o "$outfile"); fi +if [[ "$verbose" == 1 ]] ; then echo "${timecmd[@]}" -f "'$fmt'" "${cmd[@]}" ; fi "${timecmd[@]}" -f "$fmt" "${cmd[@]}" st=$? -if [[ -n "$outfile" ]] ; then cat $outfile; fi -if [[ "$FSTIME_LOAD" == 1 ]] +if [[ "$print_uptime" == 1 ]] then - make_string "@#@FSLOADPOST" "$command" "$nargs" + make_fmt "@#@FSLOADPOST" "$command" "$nargs" fi exit $st diff --git a/recon_surf/functions.sh b/recon_surf/functions.sh index 902769e6..30a6ed9b 100644 --- a/recon_surf/functions.sh +++ b/recon_surf/functions.sh @@ -7,7 +7,7 @@ export binpath # fs_time command from fs60, fs72 fails in parallel mode, use local one # also check for failure (e.g. on mac it fails, so we cannot use it there) -if FSTIME_LOAD=0 "${binpath}fs_time" echo testing &> /dev/null ; then timecmd="${binpath}fs_time" +if "${binpath}fs_time" --no-load echo testing &> /dev/null ; then timecmd="${binpath}fs_time" else timecmd="" ; echo "INFO: Testing fs_time was not successful, not reporting per-command runtimes." fi export timecmd @@ -48,6 +48,36 @@ function check_create_subjects_dir_properties() fi } +function time_it() +{ + # parameters + # $1 : timing file + # $* : cmd (command to run) + + # wraps cmd with fs_time, but fs_time's timing information (the output of fs_time) is stored to the + # timing file ($1) instead of stdout. If cmd is using fs_time itself, the FSLOAD environment variable + # is read and the appropriate --load/--no-load argument is passed to fs_time inside cmd. + local TF="$1" + shift + local cmd=("$@") + if [[ -n "$timecmd" ]] + then + timecmd_pos=-1 + for (( i=0; i<${#cmd[@]}; i++)) ; do if [ "${cmd[i]}" == "$timecmd" ]; then timecmd_pos=$i ; break ; fi ; done + if [ "$timecmd_pos" -gt -1 ] ; then + if [[ "$FSLOAD" == 1 ]] ; then a="--load" ; else a="--no-load" ; fi + cmd=("${cmd[@]:0:$timecmd_pos}" "$a" "${cmd[@]:$timecmd_pos}") + fi + # timecmd is non-empty here, so time/fs_time does not fail + printf -v key "%s\n-> " "$(echo_quoted "${cmd[@]}")" + "${binpath}fs_time" -k "$key" --no-load -o "$TF" -a "${cmd[@]}" + else + echo "WARNING: Using time_it, but time seems to fail. Not timing..." + "${cmd[@]}" + fi + if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit "${PIPESTATUS[0]}" ; fi +} + function RunIt() { # parameters @@ -63,7 +93,7 @@ function RunIt() run_it_cmdf "$LF" "$CMDF" $cmd else run_it "$LF" $cmd - if [ "${PIPESTATUS[0]}" -ne 0 ] ; then exit 1 ; fi + if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit 1 ; fi fi } @@ -76,7 +106,7 @@ function run_it() shift echo_quoted "$@" | tee -a "$LF" $timecmd "$@" 2>&1 | tee -a "$LF" - if [ "${PIPESTATUS[0]}" -ne 0 ] ; then exit 1 ; fi + if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit 1 ; fi } function run_it_cmdf() @@ -89,28 +119,31 @@ function run_it_cmdf() local CMDF=$2 shift shift + local cmd cmd="$(echo_quoted "$@" | tee -a "$LF")" printf -v tmp %q "$cmd" echo "echo $tmp" | tee -a "$CMDF" echo "$timecmd $cmd" | tee -a "$CMDF" - echo "if [ \${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi" >> "$CMDF" + echo "if [[ \${PIPESTATUS[0]} != 0 ]] ; then exit 1 ; fi" >> "$CMDF" } function RunBatchJobs() { -# parameters -# $1 : LF -# $2 ... : CMDFS - local LOG_FILE=$1 + # parameters + # $1 : LF + # $2 ... : CMDFS # launch jobs found in command files (shift past first logfile arg). # job output goes to a logfile named after the command file, which # later gets appended to LOG_FILE + local LOG_FILE=$1 + echo echo "RunBatchJobs: Logfile: $LOG_FILE" local PIDS=() local LOGS=() + local CMDFS=() shift local JOB local LOG @@ -118,36 +151,43 @@ function RunBatchJobs() echo "RunBatchJobs: CMDF: $cmdf" chmod u+x "$cmdf" JOB="$cmdf" - LOG=$cmdf.log - echo "" >& "$LOG" - echo " $JOB" >> "$LOG" - echo "" >> "$LOG" + LOG="$cmdf.log" + printf "\n %s\n\n" "$JOB" > "$LOG" exec "$JOB" >> "$LOG" 2>&1 & - PIDS=("${PIDS[@]}" "$!") - LOGS=("${LOGS[@]}" "$LOG") - + PIDS+=("$!") + CMDFS+=("$JOB") + LOGS+=("$LOG") done + # wait till all processes have finished - local PIDS_STATUS=() - for pid in "${PIDS[@]}"; do - echo "Waiting for PID $pid of (${PIDS[*]}) to complete..." - wait "$pid" - PIDS_STATUS=("${PIDS_STATUS[@]}" "$?") - done - # now append their logs to the main log file - for log in "${LOGS[@]}" + local unsuccessful=() + for i in $(seq "${#PIDS}") do - tee -a "$LOG_FILE" < "$log" - rm -f "$log" - done - echo "PIDs (${PIDS[*]}) completed and logs appended." - # and check for failures - for pid_status in "${PIDS_STATUS[@]}" - do - if [ "$pid_status" != "0" ] ; then - exit 1 + echo "Waiting for PID ${PIDS[i-1]} of (${PIDS[*]}) to complete..." + wait "${PIDS[i-1]}" + status="$?" + # now append their logs to the main log file + tee -a "$LOG_FILE" < "${LOGS[i-1]}" + rm -f "${LOGS[i-1]}" + if [[ "$status" != "0" ]] + then + unsuccessful+=($((i - 1))) + { + echo "ERROR: The script ${CMDFS[i-1]} (PID: ${PIDS[i-1]}) did not complete successfully!" + echo "========================================" + echo "" + } | tee -a "$LOG_FILE" fi done + # and check for failures + if [[ "${#unsuccessful}" == 0 ]] + then + echo "PIDs (${PIDS[*]}) completed successfully! Their logs have been appended." | tee -a "$LOG_FILE" + else + echo "PIDs (${unsuccessful[*]}) of (${PIDS[*}]}) have NOT completed successfully! All logs appended." | \ + tee -a "$LOG_FILE" + exit 1 + fi } function check_allow_root() @@ -200,22 +240,22 @@ function softlink_or_copy() { echo "echo $(echo_quoted "${ln_cmd[@]}")" echo "$timecmd $(echo_quoted "${ln_cmd[@]}")" - echo "if [ \${PIPESTATUS[0]} -ne 0 ]" + echo "if [[ \${PIPESTATUS[0]} != 0 ]]" echo "then" echo " echo $(echo_quoted "${cp_cmd[@]}")" echo " $timecmd $(echo_quoted "${cp_cmd[@]}")" - echo " if [ \${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi" + echo " if [[ \${PIPESTATUS[0]} != 0 ]] ; then exit 1 ; fi" echo "fi" } | tee -a "$CMDF" else { echo_quoted "${ln_cmd[@]}" $timecmd "${ln_cmd[@]}" 2>&1 - if [ "${PIPESTATUS[0]}" -ne 0 ] + if [[ "${PIPESTATUS[0]}" != 0 ]] then echo_quoted "${cp_cmd[@]}" $timecmd "${cp_cmd[@]}" 2>&1 - if [ "${PIPESTATUS[0]}" -ne 0 ] ; then exit 1 ; fi + if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit 1 ; fi fi } | tee -a "$LF" if [[ "${PIPESTATUS[0]}" != 0 ]]; then exit 1; fi # forward subshell exit to main script diff --git a/run_fastsurfer.sh b/run_fastsurfer.sh index 8c9c6e16..a711cca5 100755 --- a/run_fastsurfer.sh +++ b/run_fastsurfer.sh @@ -337,6 +337,16 @@ Reuter M, Schmansky NJ, Rosas HD, Fischl B. Within-subject template estimation https://doi.org/10.1016/j.neuroimage.2012.02.084 EOF + +# Environment variables (for advanced users / developers only): +# SUBJECTS_DIR (path to SUBJECTS_DIR, no default, either SUBJECTS_DIR or --sd must be set) +# FASTSURFER_HOME (path to FastSurfer installation, default: script location) +# FASTSURFER_EXECTIMELOG (path to execution time log file relative to $SUBJECTS_DIR/$SID, default: scripts/exectime.log) +# FREESURFER_HOME (path to FreeSurfer installation, must be set for surface pipeline) +# FS_LICENSE (path to FreeSurfer license file, overwritten by --fs_license, must be found for surface +# pipeline, default: search in FREESURFER_HOME, unless...) +# DO_NOT_SEARCH_FS_LICENSE_IN_FREESURFER_HOME +# (developer: if "true", deactivate search for the FreeSurfer license file in FREESURFER_HOME) } # PRINT USAGE if called without params @@ -618,6 +628,7 @@ if [[ -z "$conformed_name" ]] ; then conformed_name="$subject_dir/mri/orig.mgz"; if [[ -z "$conformed_name_t2" ]] ; then conformed_name_t2="$subject_dir/mri/T2orig.mgz" ; fi if [[ -z "$norm_name" ]] ; then norm_name="$subject_dir/mri/orig_nu.mgz" ; fi if [[ -z "$norm_name_t2" ]] ; then norm_name_t2="$subject_dir/mri/T2_nu.mgz" ; fi +if [[ -z "$exec_time_log" ]] ; then exec_time_log="$subject_dir/${FASTSURFER_EXECTIMELOG:-scripts/exectime.log}" ; fi if [[ -z "$seg_log" ]] ; then seg_log="$subject_dir/scripts/deep-seg.log" ; fi if [[ -z "$build_log" ]] ; then build_log="$subject_dir/scripts/build.log" ; fi # T2 image is only used in segmentation pipeline (but registration is done even if hypvinn is off) @@ -871,7 +882,10 @@ set +eo > /dev/null ########################################## START ######################################################## mkdir -p "$(dirname "$seg_log")" +mkdir -p "$(dirname "$exec_time_log")" + +wrap=("time_it" "$exec_time_log") if [[ -f "$seg_log" ]]; then log_existed="true" ; else log_existed="false" ; fi @@ -937,21 +951,25 @@ asegdkt_segfile_manedit=$(add_file_suffix "$asegdkt_segfile" "manedit") if [[ "$run_seg_pipeline" == "1" ]] then + + echo "SEGMENTATION PIPELINE" >> "$exec_time_log" + echo "=====================" >> "$exec_time_log" + # "============= Running FastSurferCNN (Creating Segmentation aparc.DKTatlas.aseg.mgz) ===============" # use FastSurferCNN to create cortical parcellation + anatomical segmentation into 95 classes. if [[ "$run_asegdkt_module" == "1" ]] then - cmd=($python "$fastsurfercnndir/run_prediction.py" --t1 "$t1" - --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" - --brainmask_name "$mask_name" --aseg_name "$aseg_segfile" --sid "$subject" - --seg_log "$seg_log" --vox_size "$vox_size" --batch_size "$batch_size" - --viewagg_device "$viewagg" --device "$device" --threads "$threads_seg") + echo "MODULE: FastSurferVINN aseg+DKT segmentation" >> "$exec_time_log" + cmd=($python "$fastsurfercnndir/run_prediction.py" --t1 "$t1" --sid "$subject" --asegdkt_segfile "$asegdkt_segfile" + --conformed_name "$conformed_name" --brainmask_name "$mask_name" --seg_log "$seg_log" --vox_size "$vox_size" + --aseg_name "$aseg_segfile" --batch_size "$batch_size" --viewagg_device "$viewagg" --device "$device" + --threads "$threads_seg") # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it if [[ "$sd" == "${asegdkt_segfile:0:${#sd}}" ]] ; then cmd+=(--sd "$sd") ; fi if [[ "$native_image" != "false" ]] ; then cmd+=(--orientation native --image_size fov) ; fi echo_quoted "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" exit_code="${PIPESTATUS[0]}" if [[ "${exit_code}" == 2 ]] then @@ -990,36 +1008,17 @@ then fi fi fi - if [[ -n "$t2" ]] - then - { - echo "INFO: Copying T2 file to ${copy_name_T2}..." - cmd=("nib-convert" "$t2" "$copy_name_T2") - echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 - # do not terminate if this fails - - echo "INFO: Robust scaling (partial conforming) of T2 image..." - cmd=($python "${fastsurfercnndir}/data_loader/conform.py" --no_strict_lia - --no_iso_vox --no_img_size -i "$t2" -o "$conformed_name_t2") - echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 - exit_code=$? - echo "Done." - exit $exit_code # this will only terminate the subshell - } | tee -a "$seg_log" - if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: Robust scaling of T2 failed!" | tee -a "$seg_log" ; exit 1 ; fi - fi if [[ "$run_biasfield" == "1" ]] then + echo "MODULE: Biasfield correction" >> "$exec_time_log" { # this will always run, since norm_name is set to subject_dir/mri/orig_nu.mgz, if it is not passed/empty - cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$conformed_name" - --rescale "$norm_name" --aseg "$aseg_segfile" --threads "$threads_seg") + cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$conformed_name" --rescale "$norm_name" + --aseg "$aseg_segfile" --threads "$threads_seg") echo "INFO: Running N4 bias-field correction..." echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 + "${wrap[@]}" "${cmd[@]}" 2>&1 exit $? # this will only terminate the subshell } | tee -a "$seg_log" if [[ "${PIPESTATUS[0]}" != 0 ]] @@ -1039,7 +1038,7 @@ then echo "INFO: Running talairach registration..." echo_quoted "${cmd[@]}" } | tee -a "$seg_log" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" if [[ "${PIPESTATUS[0]}" != 0 ]] then echo "ERROR: Talairach registration failed!" | tee -a "$seg_log" @@ -1070,7 +1069,7 @@ then fi { echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 + "${wrap[@]}" "${cmd[@]}" 2>&1 exit $? # this will only terminate the subshell } | tee -a "$seg_log" if [[ "${PIPESTATUS[0]}" != 0 ]] @@ -1092,7 +1091,7 @@ then ) { echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 + "${wrap[@]}" "${cmd[@]}" 2>&1 exit $? # this will only terminate the subshell } | tee -a "$seg_log" if [[ "${PIPESTATUS[0]}" != 0 ]] @@ -1105,16 +1104,34 @@ then if [[ -n "$t2" ]] then + echo "MODULE: T2 preprocessing" >> "$exec_time_log" + { + echo "INFO: Copying T2 file to ${copy_name_T2}..." + cmd=("nib-convert" "$t2" "$copy_name_T2") + echo_quoted "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" 2>&1 + # do not terminate if this fails + + echo "INFO: Robust scaling (partial conforming) of T2 image..." + cmd=($python "${fastsurfercnndir}/data_loader/conform.py" --no_strict_lia --no_iso_vox --no_img_size + -i "$t2" -o "$conformed_name_t2") + echo_quoted "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" 2>&1 + exit_code=$? + echo "Done." + exit $exit_code # this will only terminate the subshell + } | tee -a "$seg_log" + if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: Robust scaling of T2 failed!" | tee -a "$seg_log" ; exit 1 ; fi if [[ "$run_biasfield" == "1" ]] then # ... we have a t2 image, bias field-correct it (save robustly scaled uchar) - cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$copy_name_T2" - --out "$norm_name_t2" --threads "$threads_seg" --uchar) + cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$copy_name_T2" --out "$norm_name_t2" + --threads "$threads_seg" --uchar) { echo "INFO: Running N4 bias-field correction of the t2..." echo_quoted "${cmd[@]}" } | tee -a "$seg_log" - "${cmd[@]}" 2>&1 | tee -a "$seg_log" + "${wrap[@]}" "${cmd[@]}" 2>&1 | tee -a "$seg_log" if [[ "${PIPESTATUS[0]}" != 0 ]] then echo "ERROR: T2 Biasfield correction failed!" | tee -a "$seg_log" @@ -1131,7 +1148,7 @@ then echo " passed T2 image is properly scaled and typed. T2 needs to be uchar and" echo " robustly scaled (see FastSurferCNN/utils/data_loader/conform.py)!" } | tee -a "$seg_log" - "${cmd[@]}" 2>&1 | tee -a "$seg_log" + "${wrap[@]}" "${cmd[@]}" 2>&1 | tee -a "$seg_log" fi fi @@ -1139,6 +1156,7 @@ then then # ============================= CC SEGMENTATION ============================================ + echo "MODULE: FastSurfer-CC Corpus Callosum processing" >> "$exec_time_log" # generate file names of for the analysis asegdkt_withcc_segfile="$(add_file_suffix "$asegdkt_segfile" "withCC")" asegdkt_withcc_vinn_statsfile="$(add_file_suffix "$asegdkt_vinn_statsfile" "withCC")" @@ -1152,7 +1170,7 @@ then # if we are trying to create the thickness image in a headless setting, wrap call in xvfb-run { echo_quoted "${cmd[@]}" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: FastSurferCC corpus callosum analysis failed!" ; exit 1 ; fi if [[ "$edits" == 1 ]] && [[ -f "$callosum_seg_manedit" ]] ; then callosum_seg="$callosum_seg_manedit" ; fi @@ -1160,7 +1178,7 @@ then cmd=($python "$CorpusCallosumDir/paint_cc_into_pred.py" -in_cc "$callosum_seg" -in_pred "$asegdkt_segfile" "-out" "$asegdkt_withcc_segfile" "-aseg" "$aseg_auto_segfile") echo_quoted "${cmd[@]}" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: asegdkt cc inpainting failed!" ; exit 1 ; fi if [[ "$run_biasfield" == 1 ]] @@ -1189,7 +1207,7 @@ then rhCerebralWhiteMatter lhCerebralWhiteMatter CerebralWhiteMatter ) echo_quoted "${cmd[@]}" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: asegdkt statsfile ($asegdkt_withcc_segfile) generation failed!" ; exit 1 # this will only terminate the subshell @@ -1211,7 +1229,7 @@ then measures --import "all" --file "$asegdkt_withcc_vinn_statsfile" ) echo_quoted "${cmd[@]}" - "${cmd[@]}" 2>&1 + "${wrap[@]}" "${cmd[@]}" 2>&1 if [[ "${PIPESTATUS[0]}" != 0 ]] ; then echo "ERROR: aseg statsfile ($aseg_auto_segfile) failed!" ; exit 1 ; fi } | tee -a "$seg_log" if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit 1; fi # forward subshell exit to main script @@ -1221,6 +1239,7 @@ then if [[ "$run_cereb_module" == "1" ]] then + echo "MODULE: CerebNet cerebellum segmentation" >> "$exec_time_log" if [[ "$run_biasfield" == "1" ]] then cereb_flags+=(--norm_name "$norm_name" --cereb_statsfile "$cereb_statsfile") @@ -1231,16 +1250,14 @@ then } | tee -a "$seg_log" fi - cmd=($python "$cerebnetdir/run_prediction.py" --t1 "$t1" - --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" - --cereb_segfile "$cereb_segfile" --seg_log "$seg_log" --async_io - --batch_size "$batch_size" --viewagg_device "$viewagg" --device "$device" - --threads "$threads_seg" "${cereb_flags[@]}") + cmd=($python "$cerebnetdir/run_prediction.py" --t1 "$t1" --asegdkt_segfile "$asegdkt_segfile" --seg_log "$seg_log" + --conformed_name "$conformed_name" --cereb_segfile "$cereb_segfile" --async_io --batch_size "$batch_size" + --viewagg_device "$viewagg" --device "$device" --threads "$threads_seg" "${cereb_flags[@]}") # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it if [[ "$sd" == "${cereb_segfile:0:${#sd}}" ]] ; then cmd=("${cmd[@]}" --sd "$sd"); fi if [[ "$native_image" != "false" ]] ; then cmd+=(--orientation native --image_size fov --vox_size none) ; fi echo_quoted "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" # no tee, directly logging to $seg_log + "${wrap[@]}" "${cmd[@]}" # no tee, directly logging to $seg_log if [[ "${PIPESTATUS[0]}" != 0 ]] then echo "ERROR: Cerebellum Segmentation failed!" | tee -a "$seg_log" @@ -1250,25 +1267,25 @@ then if [[ "$run_hypvinn_module" == "1" ]] then - # currently, the order of the T2 preprocessing only is registration to T1w - cmd=($python "$hypvinndir/run_prediction.py" --sd "${sd}" --sid "${subject}" - "${hypvinn_flags[@]}" --reg_mode "$hypvinn_regmode" --threads "$threads_seg" --async_io - --batch_size "$batch_size" --seg_log "$seg_log" --device "$device" - --viewagg_device "$viewagg" --t1) + echo "MODULE: HypVINN hypothalamus segmentation" >> "$exec_time_log" + # currently, the order of the T2 preprocessing only is registration to T1w + cmd=($python "$hypvinndir/run_prediction.py" --sd "${sd}" --sid "${subject}" --reg_mode "$hypvinn_regmode" + "${hypvinn_flags[@]}" --threads "$threads_seg" --async_io --batch_size "$batch_size" --seg_log "$seg_log" + --device "$device" --viewagg_device "$viewagg" --t1) if [[ "$run_biasfield" == "1" ]] then cmd+=("$norm_name") - if [[ -n "$t2" ]] ; then cmd+=(--t2 "$norm_name_t2"); fi + if [[ -n "$t2" ]] ; then cmd+=(--t2 "$norm_name_t2") ; fi else { echo "WARNING: We strongly recommend to *not* exclude the biasfield (--no_biasfield)" echo " with the hypothal module!" } | tee -a "$seg_log" cmd+=("$t1") - if [[ -n "$t2" ]] ; then cmd+=(--t2 "$t2"); fi + if [[ -n "$t2" ]] ; then cmd+=(--t2 "$t2") ; fi fi echo_quoted "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" + "${wrap[@]}" "${cmd[@]}" # no tee, directly logging to $seg_log if [[ "${PIPESTATUS[0]}" != 0 ]] then echo "ERROR: Hypothalamus Segmentation failed!" | tee -a "$seg_log" @@ -1288,6 +1305,10 @@ fi if [[ "$run_surf_pipeline" == "1" ]] then + + echo "SURFACE RECONSTRUCTION PIPELINE" >> "$exec_time_log" + echo "===============================" >> "$exec_time_log" + if [[ "$threads_surf" == "max" ]]; then threads_surf="$(nproc)" ; fi if [[ "$threads_surf" == "0" ]]; then threads_surf=1 ; fi # ============= Running recon-surf (surfaces, thickness etc.) =============== @@ -1297,8 +1318,13 @@ then cmd=("./recon-surf.sh" --sid "$subject" --sd "$sd" --t1 "$conformed_name" --mask_name "$mask_name" --asegdkt_segfile "$asegdkt_segfile" --threads "$threads_surf" --py "$python" "${surf_flags[@]}") echo_quoted "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" != 0 ]] ; then exit 1 ; fi + "${wrap[@]}" "${cmd[@]}" # no tee, this gets logged to recon-surf.log from inside recon-surf.sh + if [[ "${PIPESTATUS[0]}" != 0 ]] + then + echo "ERROR: Surface reconstruction failed! See recon-surf log: $subject_dir/scripts/recon-surf.log" | \ + tee -a "$seg_log" + exit 1 + fi popd > /dev/null || return fi