From f92be170479107d9dd336cd79d5d3983b9e38b82 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 7 Jan 2026 11:27:08 -0600 Subject: [PATCH 01/74] PFMs pipeline, modified from Andrea's code, tested using Mac25Rhesus_v5 (and Mac25Cyno. --- PFM/RunPFMpipelineFullRun.sh | 275 ++++++++++++++++++++++++++++++++++ PFM/scripts/ImportPFMNotes.m | 72 +++++++++ PFM/scripts/ImportPFMNotes.sh | 122 +++++++++++++++ PFM/scripts/PFMNotes.m | 99 ++++++++++++ PFM/scripts/PFMNotes.sh | 122 +++++++++++++++ PFM/scripts/PFMNotesGroup.m | 125 ++++++++++++++++ PFM/scripts/PFMNotesGroup.sh | 114 ++++++++++++++ 7 files changed, 929 insertions(+) create mode 100755 PFM/RunPFMpipelineFullRun.sh create mode 100755 PFM/scripts/ImportPFMNotes.m create mode 100755 PFM/scripts/ImportPFMNotes.sh create mode 100755 PFM/scripts/PFMNotes.m create mode 100755 PFM/scripts/PFMNotes.sh create mode 100755 PFM/scripts/PFMNotesGroup.m create mode 100755 PFM/scripts/PFMNotesGroup.sh diff --git a/PFM/RunPFMpipelineFullRun.sh b/PFM/RunPFMpipelineFullRun.sh new file mode 100755 index 000000000..71381bfda --- /dev/null +++ b/PFM/RunPFMpipelineFullRun.sh @@ -0,0 +1,275 @@ +#!/bin/bash + +set -eu + +# This is an example script to run the full PFM postprocessing pipeline +# Steps involved: +# 1. ImportPFMNotes - Import PFM results and create time courses, spectra, and maps +# 2. RSNRegression - Run RSN regression on PFM data for dual regression +# 3. PFMNotesGroup - Generate group-level statistics and averages +# +# Please make sure that PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are done properly +# matching the input arguments before running this PFM pipeline + +# Global default values +DEFAULT_STUDY_FOLDER="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" +DEFAULT_SUBJECT_LIST="$(find ${DEFAULT_STUDY_FOLDER} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" +# fourRunSubjects=$(ls $DEFAULT_STUDY_FOLDER/A*/MNINonLinear/Results/BOLD_REST_4_PA -d | awk -F'/' '{print $(NF-3)}' | tr '\n' ' ') +# DEFAULT_SUBJECT_LIST=$(echo "${DEFAULT_SUBJECT_LIST}" | tr ' ' '\n' | grep -vxFf <(echo "${fourRunSubjects}" | tr ' ' '\n') | tr '\n' ' ') # remove four run subjects + + +DEFAULT_ENVIRONMENT_SCRIPT="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" +DEFAULT_GROUP_NAME="Mac25Rhesus_v5" # the group average name, which must be specified the same in MakeAverageDataset before running this tICA script +DEFAULT_REG_NAME="" # the registration string corresponding to the input files, which must be specified the same in MSMAll pipeline before running this tICA script +DEFAULT_MATLAB_MODE=1 # MatlabMode +DEFAULT_RUN_LOCAL=0 +DEFAULT_QUEUE="matlabparallelhigh.q" + +get_options() { + local scriptName=$(basename "$0") + local arguments=("$@") + + # initialize global variables + StudyFolder="${DEFAULT_STUDY_FOLDER}" + Subjlist="${DEFAULT_SUBJECT_LIST}" + EnvironmentScript="${DEFAULT_ENVIRONMENT_SCRIPT}" + GroupAverageName="${DEFAULT_GROUP_NAME}" + RegName="${DEFAULT_REG_NAME}" + MatlabMode="${DEFAULT_MATLAB_MODE}" + RunLocal="${DEFAULT_RUN_LOCAL}" + QUEUE="${DEFAULT_QUEUE}" + + # parse arguments + local index argument + + for ((index = 0; index < ${#arguments[@]}; ++index)) + do + argument="${arguments[index]}" + + case "$argument" in + --StudyFolder=*) + StudyFolder="${argument#*=}" + ;; + --Subject=*) + Subjlist="${argument#*=}" + ;; + --EnvironmentScript=*) + EnvironmentScript="${argument#*=}" + ;; + --GroupAverageName=*) + GroupAverageName="${argument#*=}" + ;; + --RegName=*) + RegName="${argument#*=}" + ;; + --MatlabMode=*) + MatlabMode="${argument#*=}" + ;; + *) + echo "ERROR: Unrecognized Option: ${argument}" + exit 1 + ;; + esac + done + + # check required parameters + if [[ "$StudyFolder" == "" ]] + then + echo "ERROR: StudyFolder not specified" + exit 1 + fi + + if [[ "$Subjlist" == "" ]] + then + echo "ERROR: Subjlist not specified" + exit 1 + fi + + if [[ "$EnvironmentScript" == "" ]] + then + echo "ERROR: EnvironmentScript not specified" + exit 1 + fi + + if [[ "$GroupAverageName" == "" ]] + then + echo "ERROR: GroupAverageName not specified" + exit 1 + fi + + # if [[ "$RegName" == "" ]] + # then + # echo "ERROR: RegName not specified" + # exit 1 + # fi + + if [[ "$MatlabMode" == "" ]] + then + echo "ERROR: MatlabMode not specified" + exit 1 + fi + + # report options + echo "-- ${scriptName}: Specified Command-Line Options: -- Start --" + echo " StudyFolder: ${StudyFolder}" + echo " Subjlist: ${Subjlist}" + echo " EnvironmentScript: ${EnvironmentScript}" + echo " GroupAverageName: ${GroupAverageName}" + echo " RegName: ${RegName}" + echo " MatlabMode: ${MatlabMode}" + echo "-- ${scriptName}: Specified Command-Line Options: -- End --" +} + + + +# +# Function Description +# Main processing of this script +# +# Gets user specified command line options and runs PFM postprocessing pipeline +# (please make sure the PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are finished before running this script) +# + +main() { + + # get command line options + get_options "$@" + + # set up pipeline environment variables and software + source "${EnvironmentScript}" + + if ((RunLocal)) || [[ "$QUEUE" == "" ]]; then + echo "running locally" + queuing_command=("$HCPPIPEDIR"/global/scripts/captureoutput.sh) + else + echo "queueing with fsl_sub to $QUEUE" + queuing_command=("$FSLDIR/bin/fsl_sub" -q "$QUEUE") + fi + + # general settings + # set the start step beginning from RunPROFUMO which is by default the first step + # StartStep="RunPROFUMO" + # StopStep="RunPROFUMO" + # StartStep="ImportPFMNotes" + # StopStep="ImportPFMNotes" + # StartStep="ImportPFMNotes" + # StartStep="RSNRegression" + # StopStep="RSNRegression" + # StartStep="RunPROFUMO" + # StopStep="RSNRegression" + + StartStep="RunPROFUMO" + StopStep="PFMNotesGroup" + + # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' + parLimit=-1 + + + # general inputs + fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" + # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" + + OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" + # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs + ConcatName="BOLD_REST_CONCAT" + # set the output spectra size for individual projection, RunsXNumTimePoints + subjectExpectedTimepoints="8508" + # set temporal highpass full-width (2*sigma) used in preprocessing + HighPass="pd2" + + + # PFM settings for REST data + PFMdim="46" # or 92 depending on your PROFUMO results + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim} + # Reference image for PROFUMO + RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" + + # set the file name component representing the preprocessing already done + fMRIProcSTRING="hp${HighPass}_clean" + + # set the mesh resolution, like '32' for 32k_fs_LR + LowResMesh="10" + + # RSN regression settings + RSNMethod="dual" # dual or single (default: dual) + LowDims="6" + FixLegacyBiasString="NO" + ScaleFactor="0.01" + + # CIFTI settings + CIFTIVertices="27559" # of rows in cifti + CIFTIVolume="66480" + + # Volume template file + VolumeTemplateCIFTI="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_WF8/tICA_d46/sICA_VolMaps_46.dscalar.nii" + + # PROFUMO settings + ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" + ProfumoConfig="${PFMFolder}/dataLocations.json" + TR="1.0" + ProfumoThreads="14" + DOFCorrection="0.5" + CovModel="Subject" + # RefImage will be auto-set based on data type below + + # build Profumo data location json + mkdir -p $PFMFolder + echo '{' > $ProfumoConfig + for Subject in $(echo $Subjlist | tr "@" "\n"); do + echo -e "\t\"$Subject\": {" >> $ProfumoConfig + for fMRIName in $(echo $fMRINames | tr "@" "\n"); do + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${fMRIProcSTRING}.dtseries.nii" + if [[ -e $runFile ]]; then + echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig + fi + done + perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma + echo -e "\t}," >> $ProfumoConfig + done + perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma + echo "}" >> $ProfumoConfig + + # PFM pipeline execution + echo "Starting PFM postprocessing pipeline..." + echo "Data type: ${OutputfMRIName}" + echo "PFM dimension: ${PFMdim}" + + "${queuing_command[@]}" "$HCPPIPEDIR"/PFM/PFMPipeline.sh \ + --study-folder="$StudyFolder" \ + --subject-list="$Subjlist" \ + --fmri-names="$fMRINames" \ + --output-fmri-name="$OutputfMRIName" \ + --proc-string="$fMRIProcSTRING" \ + --group-average-name="$GroupAverageName" \ + --pfm-dimension="$PFMdim" \ + --pfm-folder="$PFMFolder" \ + --surf-reg-name="$RegName" \ + --concat-name="$ConcatName" \ + --low-res-mesh="$LowResMesh" \ + --runs-timepoints="$subjectExpectedTimepoints" \ + --cifti-vertices="$CIFTIVertices" \ + --cifti-volume="$CIFTIVolume" \ + --rsn-method="$RSNMethod" \ + --low-dims="$LowDims" \ + --fix-legacy-bias="$FixLegacyBiasString" \ + --scale-factor="$ScaleFactor" \ + --starting-step="$StartStep" \ + --stop-after-step="$StopStep" \ + --parallel-limit="$parLimit" \ + --matlab-run-mode="$MatlabMode" \ + --profumo-config="$ProfumoConfig" \ + --profumo-singularity="$ProfumoSingularity" \ + --profumo-tr="$TR" \ + --profumo-threads="$ProfumoThreads" \ + --profumo-dof-correction="$DOFCorrection" \ + --profumo-cov-model="$CovModel" \ + --ref-image="$RefImage" \ + --volume-template-file="$VolumeTemplateCIFTI" + + echo "PFM pipeline submitted successfully!" +} + +# +# Invoke the main function to get things started +# +main "$@" \ No newline at end of file diff --git a/PFM/scripts/ImportPFMNotes.m b/PFM/scripts/ImportPFMNotes.m new file mode 100755 index 000000000..716c28546 --- /dev/null +++ b/PFM/scripts/ImportPFMNotes.m @@ -0,0 +1,72 @@ +function ImportPFMNotes(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) + + Subjlist = strsplit(SubjListRaw, '@'); + fMRINames = strsplit(fMRIListRaw, '@'); + TR = str2double(TR); + wbcommand = 'wb_command'; + + for s = 1:length(Subjlist) + s + subfMRINames = {}; + if ~strcmp(ConcatName, '') + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii']) + c = 1; + for r = 1:length(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + subfMRINames{c} = fMRINames{r}; + c = c + 1; + end + end + end + else + c = 1; + for r = 1:length(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + subfMRINames{c} = fMRINames{r}; + c = c + 1; + end + end + end + + if length(subfMRINames) ~= 0 + origTCS = []; + TCS = []; + for r = 1:length(subfMRINames) + runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + + origTCS = [origTCS ; runTCS]; + TCS = [TCS ; runTCS .* repmat(runAmp', length(runTCS), 1)]; + end + + % sICATCS = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputfMRIName OutputSTRING RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + % sICASpectra = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputfMRIName OutputSTRING RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + + PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); + % PFMTCSorig.diminfo{1,2} = sICATCS.diminfo{1,2}; + ts.Nnodes = size(origTCS, 2); + ts.Nsubjects = 1; + ts.ts = origTCS; + ts.NtimepointsPerSubject = size(origTCS, 1); + PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + % PFMSpectraorig.diminfo{1,2} = sICASpectra.diminfo{1,2}; + + PFMTCS = cifti_struct_create_sdseries(TCS'); + % PFMTCS.diminfo{1,2} = sICATCS.diminfo{1,2}; + ts.Nnodes = size(TCS, 2); + ts.Nsubjects = 1; + ts.ts = TCS; + ts.NtimepointsPerSubject = size(TCS, 1); + PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + % PFMSpectra.diminfo{1,2} = sICASpectra.diminfo{1,2}; + + ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + + ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + + unix(['cp ' PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii ' StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); + end + end +end \ No newline at end of file diff --git a/PFM/scripts/ImportPFMNotes.sh b/PFM/scripts/ImportPFMNotes.sh new file mode 100755 index 000000000..9f925eb97 --- /dev/null +++ b/PFM/scripts/ImportPFMNotes.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -eu + +pipedirguessed=0 +if [[ "${HCPPIPEDIR:-}" == "" ]] +then + pipedirguessed=1 + export HCPPIPEDIR="$(dirname -- "$0")/../.." +fi + +source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" +g_matlab_default_mode=1 + +#this function gets called by opts_ParseArguments when --help is specified +function usage() +{ + #header text + echo " +$log_ToolName: Import PFM notes and create time courses, spectra, and maps + +Usage: $log_ToolName PARAMETER... + +PARAMETERs are [ ] = optional; < > = user supplied value +" + #automatic argument descriptions + opts_ShowArguments + + #do not use exit, the parsing code takes care of it +} + +#arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] +opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" +opts_AddMandatory '--subject-list' 'SubjListRaw' '100206@100307...' 'list of subject IDs separated by @s' +opts_AddMandatory '--fmri-names' 'fMRIListRaw' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' 'list of fmri run names separated by @s' +opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing" +opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" +opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string identifier" +opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" +opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string" +opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" +opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" +opts_AddMandatory '--concat-name' 'ConcatName' 'string' "concatenated fMRI name if using multi-run data" +opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' + +opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode +0 = compiled MATLAB +1 = interpreted MATLAB +2 = Octave" "$g_matlab_default_mode" + +opts_ParseArguments "$@" + +if ((pipedirguessed)) +then + log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" +fi + +#display the parsed/default values +opts_ShowValues + +RegString="" +if [[ "$RegName" != "" ]] +then + RegString="_$RegName" +fi + +case "$MatlabMode" in + (0) + if [[ "${MATLAB_COMPILER_RUNTIME:-}" == "" ]] + then + log_Err_Abort "to use compiled matlab, you must set and export the variable MATLAB_COMPILER_RUNTIME" + fi + ;; + (1) + matlab_interpreter=(matlab -nodisplay -nosplash) + ;; + (2) + matlab_interpreter=(octave-cli -q --no-window-system) + ;; + (*) + log_Err_Abort "unrecognized matlab mode '$MatlabMode', use 0, 1, or 2" + ;; +esac + +IFS='@' read -a SubjList <<<"$SubjListRaw" +IFS='@' read -a fMRIList <<<"$fMRIListRaw" + +#shortcut in case the folder gets renamed +this_script_dir=$(dirname "$0") + +#matlab function arguments converted to strings +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$OutputPrefix" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") + +case "$MatlabMode" in + (0) + matlab_cmd=("$this_script_dir/Compiled_ImportPFMNotes/run_ImportPFMNotes.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + log_Msg "running compiled matlab command: ${matlab_cmd[*]}" + "${matlab_cmd[@]}" + ;; + (1 | 2) + #reformat argument array so matlab sees them as strings + matlab_args="" + for thisarg in "${matlab_argarray[@]}" + do + if [[ "$matlab_args" != "" ]] + then + matlab_args+=", " + fi + matlab_args+="'$thisarg'" + done + matlabcode=" + addpath('$HCPPIPEDIR/global/matlab'); + addpath('$this_script_dir'); + addpath('$HCPCIFTIRWDIR'); + ImportPFMNotes($matlab_args);" + + log_Msg "running matlab code: $matlabcode" + "${matlab_interpreter[@]}" <<<"$matlabcode" + echo + ;; +esac \ No newline at end of file diff --git a/PFM/scripts/PFMNotes.m b/PFM/scripts/PFMNotes.m new file mode 100755 index 000000000..10f840be8 --- /dev/null +++ b/PFM/scripts/PFMNotes.m @@ -0,0 +1,99 @@ +function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRIName, PFMdimStr, OutputPrefix, RegString, LowResMesh, RunsXNumTimePointsStr, CIFTIVerticesStr, CIFTIVolumeStr, PFMFolder) + + % Parse string inputs + Subjlist = strsplit(SubjListRaw, '@'); + PFMdim = str2double(PFMdimStr); + RunsXNumTimePoints = str2double(RunsXNumTimePointsStr); + CIFTI = str2double(CIFTIVerticesStr); + CIFTIVol = str2double(CIFTIVolumeStr); + + wbcommand = 'wb_command'; + + c = 1; + SubjFolderlist = {}; + StudyFolderNumber = []; + for i = 1:length(Subjlist) + SubjFolderlist{c} = [StudyFolder '/' Subjlist{i}]; + StudyFolderNumber = [StudyFolderNumber 1]; + c = c + 1; + end + + for i = 1:length(Subjlist) + if ~isfile([StudyFolder '/' Subjlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii']) + Subjlist{i} + end + end + + s1 = 0; + m1 = 0; + TCSMask = zeros(PFMdim, RunsXNumTimePoints, length(SubjFolderlist), 'single'); + TCSAll = zeros(PFMdim, RunsXNumTimePoints, length(SubjFolderlist), 'single'); + SpectraOne = []; + PFMMapsOne = []; + PFMVolMapsOne = []; + + for i = 1:length(SubjFolderlist) + Subjlist{i} + if exist([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii']) + PFMMapsSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + PFMVolMapsSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + TCSSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + SpectraSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + TCSAll(1:size(TCSSub.cdata,1), 1:size(TCSSub.cdata,2), i) = TCSSub.cdata; + + PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = NaN; + PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; + + RANK(i) = rank(TCSSub.cdata); + COND(i) = cond(TCSSub.cdata); + + if StudyFolderNumber(i) == 1 + if length(TCSSub.cdata) == RunsXNumTimePoints + TCSMask(:, :, i) = repmat(1, PFMdim, RunsXNumTimePoints, 1); + if isempty(SpectraOne) + SpectraOne = SpectraSub; + SpectraOne.cdata = SpectraSub.cdata * 0; + end + SpectraOne.cdata = SpectraOne.cdata + SpectraSub.cdata; + s1 = s1 + 1; + end + if isempty(PFMMapsOne) + PFMMapsOne = PFMMapsSub; + PFMMapsOne.cdata = PFMMapsSub.cdata * 0; + PFMVolMapsOne = PFMVolMapsSub; + PFMVolMapsOne.cdata = PFMVolMapsSub.cdata * 0; + end + PFMMapsOne.cdata = PFMMapsOne.cdata + PFMMapsSub.cdata; + PFMVolMapsOne.cdata = PFMVolMapsOne.cdata + PFMVolMapsSub.cdata; + m1 = m1 + 1; + end + end + end + + TCSMaskConcat = TCSSub; + TCSMaskConcat.cdata = squeeze(reshape(TCSMask, PFMdim, RunsXNumTimePoints * length(SubjFolderlist))); + TCSFullConcat = TCSSub; + TCSFullConcat.cdata = squeeze(reshape(TCSAll, PFMdim, RunsXNumTimePoints * length(SubjFolderlist))); + ciftisavereset(TCSMaskConcat, [PFMFolder '/PFM_TCSMASK_' num2str(PFMdim) '.sdseries.nii'], wbcommand); + ciftisavereset(TCSFullConcat, [PFMFolder '/PFM_TCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); + + PFMTSTDs = std(TCSFullConcat.cdata, [], 2); + PFMPercentVariances = (((PFMTSTDs .^ 2) / sum(PFMTSTDs .^ 2)) * 100); + dlmwrite([PFMFolder '/PFM_stats_' num2str(PFMdim) '.wb_annsub.csv'], [round(PFMPercentVariances, 2)], ','); + + TCSAVGOne = TCSSub; + TCSAVGOne.cdata = sum(TCSAll .* single(TCSMask == 1), 3) / s1; + TCSABSAVGOne = TCSSub; + TCSABSAVGOne.cdata = sum(abs(TCSAll .* single(TCSMask == 1)), 3) / s1; + + SpectraOne.cdata = SpectraOne.cdata / s1; + PFMMapsOne.cdata = PFMMapsOne.cdata / m1; + PFMVolMapsOne.cdata = PFMVolMapsOne.cdata / m1; + + ciftisavereset(TCSAVGOne, [PFMFolder '/PFM_AVGTCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); + ciftisavereset(TCSABSAVGOne, [PFMFolder '/PFM_ABSAVGTCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); + + ciftisavereset(SpectraOne, [PFMFolder '/PFM_Spectra_' num2str(PFMdim) '.sdseries.nii'], wbcommand); + ciftisavereset(PFMMapsOne, [PFMFolder '/PFM_Maps_' num2str(PFMdim) '.dscalar.nii'], wbcommand); + ciftisavereset(PFMVolMapsOne, [PFMFolder '/PFM_VolMaps_' num2str(PFMdim) '.dscalar.nii'], wbcommand); +end \ No newline at end of file diff --git a/PFM/scripts/PFMNotes.sh b/PFM/scripts/PFMNotes.sh new file mode 100755 index 000000000..31e541f3b --- /dev/null +++ b/PFM/scripts/PFMNotes.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -eu + +pipedirguessed=0 +if [[ "${HCPPIPEDIR:-}" == "" ]] +then + pipedirguessed=1 + export HCPPIPEDIR="$(dirname -- "$0")/../.." +fi + +source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" +g_matlab_default_mode=1 + +#this function gets called by opts_ParseArguments when --help is specified +function usage() +{ + #header text + echo " +$log_ToolName: Generate group-level PFM notes and statistics + +Usage: $log_ToolName PARAMETER... + +PARAMETERs are [ ] = optional; < > = user supplied value +" + #automatic argument descriptions + opts_ShowArguments + + #do not use exit, the parsing code takes care of it +} + +#arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] +opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" +opts_AddMandatory '--subject-list' 'SubjListRaw' '100206@100307...' 'list of subject IDs separated by @s' +opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' +opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" +opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality" +opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" +opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string" +opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" +opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' 'integer' "total timepoints across runs" +opts_AddMandatory '--cifti-vertices' 'CIFTIVertices' 'integer' "number of CIFTI vertices" +opts_AddMandatory '--cifti-volume' 'CIFTIVolume' 'integer' "number of CIFTI volume vertices" +opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" + + +opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode +0 = compiled MATLAB +1 = interpreted MATLAB +2 = Octave" "$g_matlab_default_mode" + +opts_ParseArguments "$@" + +if ((pipedirguessed)) +then + log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" +fi + +#display the parsed/default values +opts_ShowValues + +RegString="" +if [[ "$RegName" != "" ]] +then + RegString="_$RegName" +fi + +case "$MatlabMode" in + (0) + if [[ "${MATLAB_COMPILER_RUNTIME:-}" == "" ]] + then + log_Err_Abort "to use compiled matlab, you must set and export the variable MATLAB_COMPILER_RUNTIME" + fi + ;; + (1) + matlab_interpreter=(matlab -nodisplay -nosplash) + ;; + (2) + matlab_interpreter=(octave-cli -q --no-window-system) + ;; + (*) + log_Err_Abort "unrecognized matlab mode '$MatlabMode', use 0, 1, or 2" + ;; +esac + +#shortcut in case the folder gets renamed +this_script_dir=$(dirname "$0") + +#matlab function arguments converted to strings +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$GroupAverageName" "$OutputfMRIName" "$PFMdim" "$OutputPrefix" "$RegString" "$LowResMesh" "$RunsXNumTimePoints" "$CIFTIVertices" "$CIFTIVolume" "$PFMFolder") + +case "$MatlabMode" in + (0) + matlab_cmd=("$this_script_dir/Compiled_PFMNotesGroup/run_PFMNotesGroup.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + log_Msg "running compiled matlab command: ${matlab_cmd[*]}" + "${matlab_cmd[@]}" + ;; + (1 | 2) + #reformat argument array so matlab sees them as strings + matlab_args="" + for thisarg in "${matlab_argarray[@]}" + do + if [[ "$matlab_args" != "" ]] + then + matlab_args+=", " + fi + matlab_args+="'$thisarg'" + done + matlabcode=" + addpath('$HCPPIPEDIR/global/matlab'); + addpath('$this_script_dir'); + addpath('$HCPCIFTIRWDIR'); + PFMNotesGroup($matlab_args);" + + log_Msg "running matlab code: $matlabcode" + "${matlab_interpreter[@]}" <<<"$matlabcode" + echo + ;; +esac + +log_Msg "PFMNotesGroup step completed successfully" \ No newline at end of file diff --git a/PFM/scripts/PFMNotesGroup.m b/PFM/scripts/PFMNotesGroup.m new file mode 100755 index 000000000..68e499069 --- /dev/null +++ b/PFM/scripts/PFMNotesGroup.m @@ -0,0 +1,125 @@ +function PFMNotesGroup(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) + + %% example parameters for 1071 3T MSMAll Rest + % StudyFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final'; + % SubjlistRaw = "100206@100307"; + % PFMdim = 92; + % OutputPrefix = ['rfMRI_REST_d92_WF5_S1200_MSMAll3T1071_PFMs_tclean']; + % RegName = '_MSMAll'; %e.g. _MSMAll + % LowResMesh = '32'; + % RunsXNumTimePoints = 4800; + % PFMFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll3T1071/MNINonLinear/Results/rfMRI_REST/PFMs_chpc'; + % + % CIFTIVertices = 91282; % no longer used + % CIFTIVol = 283226 ;% no longer used + + wbcommand = 'wb_command'; + + %% parse arguments + subjList = regexp(SubjlistRaw,'@','split'); + nS = numel(subjList); + RunsXNumTimePoints = str2double(RunsXNumTimePoints); + PFMdim = str2double(PFMdim); + if nargin < 8;error('All arguments are required.');end % failsafe + + %% preallocate + [TCSMask, TCSAll] = deal(zeros(PFMdim,RunsXNumTimePoints, nS,'single')); + [spectra,PFMmaps, PFMvolMaps] = deal([]); + %PFMMapsAll = zeros(CIFTIVertices,PFMdim, nS,'single'); + %PFMVolMapsAll = zeros(CIFTIVol,PFMdim, nS,'single'); + + %% loop over subjects + for iS = 1:nS + subj = subjList{iS}; + subjDir = [StudyFolder '/' subj '/MNINonLinear/fsaverage_LR' LowResMesh 'k']; + fprintf('processing %s ... \n', subj); + + % load subject data + PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'],wbcommand); + PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'],wbcommand); + TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'],wbcommand); + SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'],wbcommand); + + + % clean up NaNs and Infs in PFM VolMaps + infMask = isinf(PFMVolMapsSub.cdata); + nanMask = isnan(PFMVolMapsSub.cdata); + if any(infMask, 'all') + warning('Found Infs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = 0; + end + if any(nanMask, 'all') + warning('Found NaNs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; + end + + % store subject TCS in TCSALL array + TCSAll(1:size(TCSSub.cdata,1),1:size(TCSSub.cdata,2),iS) = TCSSub.cdata; + + % Perform running sum of spectra and PFM maps + if size(TCSSub.cdata,2) == RunsXNumTimePoints + TCSMask(:,:,iS) = repmat(1,PFMdim,RunsXNumTimePoints,1); + if isempty(spectra) + spectra = SpectraSub; + spectra.cdata = SpectraSub.cdata*0; + end + spectra.cdata = spectra.cdata + SpectraSub.cdata; + end + if isempty(PFMmaps) + PFMmaps = PFMMapsSub; + PFMmaps.cdata = PFMMapsSub.cdata*0; + PFMvolMaps = PFMVolMapsSub; + PFMvolMaps.cdata = PFMVolMapsSub.cdata*0; + end + PFMmaps.cdata = PFMmaps.cdata + PFMMapsSub.cdata; + PFMvolMaps.cdata = PFMvolMaps.cdata + PFMVolMapsSub.cdata; + end % for iS = 1:nS + + %% package and save outputs + if ~exist(PFMFolder,'dir');mkdir(PFMFolder);end + dimStr = num2str(PFMdim); + % timeseries + TCSMaskConcat = TCSSub; + TCSMaskConcat.cdata = squeeze(reshape(TCSMask,PFMdim,RunsXNumTimePoints* nS)); + TCSFullConcat = TCSSub; + TCSFullConcat.cdata = squeeze(reshape(TCSAll,PFMdim,RunsXNumTimePoints* nS)); + ciftisavereset(TCSMaskConcat,[PFMFolder '/PFM_TCSMASK_' dimStr '.sdseries.nii'],wbcommand); + ciftisavereset(TCSFullConcat,[PFMFolder '/PFM_TCS_' dimStr '.sdseries.nii'],wbcommand); + + TCSAVG = TCSSub; + TCSAVG.cdata = sum(TCSAll.*TCSMask,3)/nS; + TCSABSAVG = TCSSub; + TCSABSAVG.cdata = sum(abs(TCSAll.*TCSMask),3)/nS; + ciftisavereset(TCSAVG,[PFMFolder '/PFM_AVGTCS_' dimStr '.sdseries.nii'],wbcommand); + ciftisavereset(TCSABSAVG,[PFMFolder '/PFM_ABSAVGTCS_' dimStr '.sdseries.nii'],wbcommand); + + + % PFM stats + PFMTSTDs = std(TCSFullConcat.cdata,[],2); + PFMPercentVariances = (((PFMTSTDs.^2)/sum(PFMTSTDs.^2))*100); + dlmwrite([PFMFolder '/PFM_stats_' dimStr '.wb_annsub.csv'],[(1:PFMdim)' round(PFMPercentVariances,2)],','); + + % spectra + spectra.cdata = spectra.cdata/nS; + ciftisavereset(spectra,[PFMFolder '/PFM_Spectra_' dimStr '.sdseries.nii'],wbcommand); + + + % PFM maps + PFMmaps.cdata = PFMmaps.cdata/nS; + PFMvolMaps.cdata = PFMvolMaps.cdata/nS; + + %TRIM = repmat(squeeze(trimmean(PFMMapsAll,10,3)),1,1, nS); + %MAD = repmat(squeeze(mad(PFMMapsAll,1,3)*1.4826),1,1, nS); + %MASK = (PFMMapsAll>TRIM-MAD*2).*(PFMMapsAllTRIM-MAD*2).*(PFMVolMapsAll = user supplied value +" + #automatic argument descriptions + opts_ShowArguments + + #do not use exit, the parsing code takes care of it +} + +#arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] +opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains all subjects" +opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of subject IDs separated by @s" +opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" +opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" +opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" +opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" +opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' 'integer' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" +opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" +opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode +0 = compiled MATLAB +1 = interpreted MATLAB +2 = Octave" "$g_matlab_default_mode" + +opts_ParseArguments "$@" + +if ((pipedirguessed)) +then + log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" +fi + +#display the parsed/default values +opts_ShowValues + +RegString="" +if [[ "$RegName" != "" ]] +then + RegString="_$RegName" +fi + +case "$MatlabMode" in + (0) + if [[ "${MATLAB_COMPILER_RUNTIME:-}" == "" ]] + then + log_Err_Abort "to use compiled matlab, you must set and export the variable MATLAB_COMPILER_RUNTIME" + fi + ;; + (1) + matlab_interpreter=(matlab -nodisplay -nosplash) + ;; + (2) + matlab_interpreter=(octave-cli -q --no-window-system) + ;; + (*) + log_Err_Abort "unrecognized matlab mode '$MatlabMode', use 0, 1, or 2" + ;; +esac + +#shortcut in case the folder gets renamed +this_script_dir=$(dirname "$0") + +#matlab function arguments converted to strings +matlab_argarray=("$StudyFolder" "$SubjlistRaw" "$PFMdim" "$OutputPrefix" "$RegString" "$LowResMesh" "$RunsXNumTimePoints" "$PFMFolder") + +case "$MatlabMode" in + (0) + matlab_cmd=("$this_script_dir/Compiled_PFMNotesGroup/run_PFMNotesGroup.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + log_Msg "running compiled matlab command: ${matlab_cmd[*]}" + "${matlab_cmd[@]}" + ;; + (1 | 2) + #reformat argument array so matlab sees them as strings + matlab_args="" + for thisarg in "${matlab_argarray[@]}" + do + if [[ "$matlab_args" != "" ]] + then + matlab_args+=", " + fi + matlab_args+="'$thisarg'" + done + matlabcode=" + addpath('$HCPPIPEDIR/global/matlab'); + addpath('$this_script_dir'); + addpath('$HCPCIFTIRWDIR'); + PFMNotesGroup($matlab_args);" + + log_Msg "running matlab code: $matlabcode" + "${matlab_interpreter[@]}" <<<"$matlabcode" + echo + ;; +esac \ No newline at end of file From dc7db84d4e4771322a5afd91ab1f49de51506a50 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 7 Jan 2026 14:13:21 -0600 Subject: [PATCH 02/74] forgot the actual pipeline script --- PFM/PFMPipeline.sh | 307 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100755 PFM/PFMPipeline.sh diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh new file mode 100755 index 000000000..2e08668a5 --- /dev/null +++ b/PFM/PFMPipeline.sh @@ -0,0 +1,307 @@ +#!/bin/bash +set -eu + +pipedirguessed=0 +if [[ "${HCPPIPEDIR:-}" == "" ]] +then + pipedirguessed=1 + #fix this if the script is more than one level below HCPPIPEDIR + export HCPPIPEDIR="$(dirname -- "$0")/.." +fi + +source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/parallel.shlib" "$@" + +g_matlab_default_mode=1 + +#add steps to this array and in the switch cases below +pipelineSteps=(RunPROFUMO ImportPFMNotes RSNRegression PFMNotesGroup) +defaultStart="${pipelineSteps[0]}" +defaultStopAfter="${pipelineSteps[${#pipelineSteps[@]} - 1]}" +stepsText="$(IFS=$'\n'; echo "${pipelineSteps[*]}")" + +#description to use in usage - syntax of parameters is now explained automatically +opts_SetScriptDescription "implements complete PFM pipeline with four main steps: Run PROFUMO, Import PFM Notes, RSN Regression, and PFM Notes Group processing" + +#mandatory parameters +opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains all subjects" +opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of subject IDs separated by @s" +opts_AddMandatory '--fmri-names' 'fMRINames' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' "list of fmri run names separated by @s" +opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" +opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing already done, e.g. '_Atlas_MSMAll_hp2000_clean_rclean_tclean'" +opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' +opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" +opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder containing Results.ppp" +opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" +opts_AddMandatory '--profumo-config' 'ProfumoConfig' 'path' "path to PROFUMO JSON configuration file" +opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' +opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO postprocessing" +opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" '4800' +opts_AddMandatory '--cifti-vertices' 'CIFTIVertices' "number of CIFTI vertices" '91282' +opts_AddMandatory '--cifti-volume' 'CIFTIVolume' "number of CIFTI volume vertices" '283226' +opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' +opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' + +#PROFUMO specific parameters +opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threads for PROFUMO" '25' +opts_AddOptional '--profumo-dof-correction' 'DOFCorrection' 'float' "DOF correction for PROFUMO" '0.5' +opts_AddOptional '--profumo-cov-model' 'CovModel' 'string' "covariance model for PROFUMO" 'Subject' +opts_AddOptional '--profumo-singularity' 'ProfumoSingularity' 'path' "path to PROFUMO singularity container" + +#optional parameters +opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' + +#RSN regression specific parameters +opts_AddOptional '--rsn-method' 'RSNMethod' 'weighted or dual' "RSN regression method: weighted (WR) or dual (DR) - default: dual" 'dual' +opts_AddOptional '--low-dims' 'LowDims' 'string' "low dimensionalities for RSN regression" '7@8@9@10@11@12@13@14@15@16@17@18@19@20@21' +opts_AddOptional '--low-dims-template-file' 'LowDimTemplate' 'path' "low dimensionality template name for RSN regression" '' +opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' +opts_AddOptional '--scale-factor' 'ScaleFactor' 'float' 'scale factor for RSN regression' '0.01' + +#general settings +opts_AddOptional '--starting-step' 'startStep' 'step' "what step to start processing at, one of: +$stepsText" "$defaultStart" +opts_AddOptional '--stop-after-step' 'stopAfterStep' 'step' "what step to stop processing after, same valid values as --starting-step" "$defaultStopAfter" +opts_AddOptional '--parallel-limit' 'parLimit' 'integer' "set how many subjects to do in parallel during RSN regression, defaults to all detected physical cores" '-1' +opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode +0 = compiled MATLAB +1 = interpreted MATLAB +2 = Octave" "$g_matlab_default_mode" + +opts_ParseArguments "$@" + +if ((pipedirguessed)) +then + log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" +fi + +#display the parsed/default values +opts_ShowValues + +#processing code goes here +IFS='@' read -a Subjlist <<<"$SubjlistRaw" +IFS='@' read -a fMRINamesArray <<<"$fMRINames" + +FixLegacyBiasBool=$(opts_StringToBool "$FixLegacyBias") + +if ! [[ "$parLimit" == "-1" || "$parLimit" =~ [1-9][0-9]* ]] +then + log_Err_Abort "--parallel-limit must be a positive integer or -1, provided value: '$parLimit'" +fi + +function stepNameToInd() +{ + for ((i = 0; i < ${#pipelineSteps[@]}; ++i)) + do + if [[ "$1" == "${pipelineSteps[i]}" ]] + then + echo "$i" + return + fi + done + log_Err_Abort "unrecognized step name: '$1'" +} + +startInd=$(stepNameToInd "$startStep") +stopAfterInd=$(stepNameToInd "$stopAfterStep") + +if ((startInd > stopAfterInd)) +then + log_Err_Abort "starting step '$startStep' must not be after the stopping step '$stopAfterStep'" +fi + +RegString="" +if [[ "$RegName" != "" ]] +then + RegString="_$RegName" +fi + +# Auto-generate output strings +OutputSTRING="_d${PFMdim}_${GroupAverageName}_WR" +OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_PFMs_tclean" + +# Volume template file path +# VolumeTemplateFile="${StudyFolder}/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.2.dscalar.nii" + +for ((stepInd = startInd; stepInd <= stopAfterInd; ++stepInd)) +do + stepName="${pipelineSteps[stepInd]}" + case "$stepName" in + (RunPROFUMO) + log_Msg "Running PROFUMO analysis step" + + # Validate required PROFUMO parameters + if [[ "$ProfumoConfig" == "" ]] + then + log_Err_Abort "PROFUMO config file must be specified with --profumo-config" + fi + if [[ "$ProfumoSingularity" == "" ]] + then + log_Err_Abort "PROFUMO singularity container must be specified with --profumo-singularity" + fi + if [[ "$RefImage" == "" ]] + then + log_Err_Abort "Reference image must be specified with --ref-image" + fi + + # Set up PROFUMO paths + PFM_PATH="${PFMFolder}/Analysis.pfm" + RESULTS_PATH="${PFMFolder}/Results.ppp" + REAL_REF_IMAGE=$(readlink -f "${RefImage}") + + # Calculate low rank data parameter + LowRankData=$((PFMdim * 5)) + + # Create output directory + mkdir -p "${PFMFolder}" + + # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" + echo apptainer exec --bind $(dirname "${StudyFolder}") \ + --env PROFUMODIR=/opt/profumo \ + "${ProfumoSingularity}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + "${PFMdim}" "${PFM_PATH}" \ + --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ + --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" + apptainer exec --bind $(dirname "${StudyFolder}") \ + --env PROFUMODIR=/opt/profumo \ + "${ProfumoSingularity}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + "${PFMdim}" "${PFM_PATH}" \ + --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ + --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" + + log_Msg "Running PROFUMO postprocessing" + echo apptainer exec --bind $(dirname "${StudyFolder}") \ + --env PROFUMODIR=/opt/profumo \ + "${ProfumoSingularity}" \ + /opt/fsl/fslpython/envs/profumo/bin/python3 /opt/profumo/Python/postprocess_results.py \ + --web-report \ + "${PFM_PATH}" \ + "${RESULTS_PATH}" \ + "${REAL_REF_IMAGE}" + apptainer exec --bind $(dirname "${StudyFolder}") \ + --env PROFUMODIR=/opt/profumo \ + "${ProfumoSingularity}" \ + /opt/fsl/fslpython/envs/profumo/bin/python3 /opt/profumo/Python/postprocess_results.py \ + --web-report \ + "${PFM_PATH}" \ + "${RESULTS_PATH}" \ + "${REAL_REF_IMAGE}" + ;; + (ImportPFMNotes) + log_Msg "Running ImportPFMNotes step" + "$HCPPIPEDIR"/PFM/scripts/ImportPFMNotes.sh \ + --study-folder="$StudyFolder" \ + --subject-list="$SubjlistRaw" \ + --fmri-names="$fMRINames" \ + --concat-name="$ConcatName" \ + --proc-string="$fMRIProcSTRING" \ + --output-fmri-name="$OutputfMRIName" \ + --output-string="$OutputSTRING" \ + --output-prefix="$OutputPrefix" \ + --surf-reg-name="$RegName" \ + --low-res-mesh="$LowResMesh" \ + --profumo-tr="$TR" \ + --pfm-folder="$PFMFolder" \ + --matlab-run-mode="$MatlabMode" + ;; + (RSNRegression) + log_Msg "Running RSNRegression step" + + # Set up template paths + # LowDimTemplate="${StudyFolder}/${GroupAverageName}/MNINonLinear/Results/${OutputfMRIName}/sICA/melodic_oIC_${PFMdim}.dscalar.nii" + + for Subject in "${Subjlist[@]}" + do + # Build list of existing fMRI files for this subject (same logic as your example) + fMRINamesForSub="" + for fMRIName in "${fMRINamesArray[@]}" + do + if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}${fMRIProcSTRING}.dtseries.nii" ]] + then + if [[ "$fMRINamesForSub" != "" ]] + then + fMRINamesForSub="${fMRINamesForSub}@${fMRIName}" + else + fMRINamesForSub="${fMRIName}" + fi + fi + done + + if [[ "$fMRINamesForSub" == "" ]] + then + log_Warn "No valid fMRI runs found for subject $Subject, skipping" + continue + fi + + # Set maps and timeseries based on method (following your example exactly) + if [[ "$RSNMethod" == "single" ]] + then + # TimeSeries="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_${RegName}_ts.${LowResMesh}k_fs_LR.sdseries.nii" + TimeSeries="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_ts.${LowResMesh}k_fs_LR.sdseries.nii" + elif [[ "$RSNMethod" == "dual" ]] + then + # GroupMaps="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_${RegName}_origmaps.${LowResMesh}k_fs_LR.dscalar.nii" + # GroupMaps="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_origmaps.${LowResMesh}k_fs_LR.dscalar.nii" + GroupMaps="${PFMFolder}/Results.ppp/Maps/Group.dscalar.nii" + TimeSeries="" + fi + + # Build RSN regression command (exactly like your example) + rsn_cmd=("$HCPPIPEDIR"/global/scripts/RSNregression.sh + --study-folder="$StudyFolder" + --subject="$Subject" + --subject-timeseries="$ConcatName" # "$fMRINamesForSub" + --surf-reg-name="$RegName" + --low-res="$LowResMesh" + --proc-string="_$fMRIProcSTRING" + --method="$RSNMethod" + --low-ica-dims="$LowDims" + --low-ica-template-name="$LowDimTemplate" + --output-string="$OutputPrefix" + --output-spectra="$RunsXNumTimePoints" + --volume-template-cifti="$VolumeTemplateFile" + --output-z=1 + --fix-legacy-bias="$FixLegacyBias" + --scale-factor="$ScaleFactor" + ) + + # Add conditional arguments based on method + if [[ "$RSNMethod" == "dual" ]] + then + rsn_cmd+=(--group-maps="$GroupMaps") + fi + if [[ "$RSNMethod" == "single" ]] + then + rsn_cmd+=(--timeseries="$TimeSeries") + fi + + # Queue parallel job + par_addjob "${rsn_cmd[@]}" + done + + # Run the jobs + par_runjobs "$parLimit" + ;; + (PFMNotesGroup) + log_Msg "Running PFMNotesGroup step" + "$HCPPIPEDIR"/PFM/scripts/PFMNotesGroup.sh \ + --study-folder="$StudyFolder" \ + --subject-list="$SubjlistRaw" \ + --pfm-dimension="$PFMdim" \ + --output-prefix="$OutputPrefix" \ + --surf-reg-name="$RegName" \ + --low-res-mesh="$LowResMesh" \ + --runs-timepoints="$RunsXNumTimePoints" \ + --pfm-folder="$PFMFolder" \ + --matlab-run-mode="$MatlabMode" + ;; + (*) #NOTE: this case MUST be last + log_Err_Abort "internal error: unimplemented pipeline step '$stepName'" + ;; + esac + log_Msg "step $stepName complete" +done \ No newline at end of file From 304121411ab35beae6e1e8795b510729c9da3a90 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 7 Jan 2026 15:00:52 -0600 Subject: [PATCH 03/74] add random seed --- PFM/PFMPipeline.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 2e08668a5..ef993c88f 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -49,6 +49,7 @@ opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threa opts_AddOptional '--profumo-dof-correction' 'DOFCorrection' 'float' "DOF correction for PROFUMO" '0.5' opts_AddOptional '--profumo-cov-model' 'CovModel' 'string' "covariance model for PROFUMO" 'Subject' opts_AddOptional '--profumo-singularity' 'ProfumoSingularity' 'path' "path to PROFUMO singularity container" +opts_AddOptional '--profumo-random-seed' 'RandomSeed' 'integer' "random seed for PROFUMO" '123' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -171,7 +172,7 @@ do /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ - --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" + --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" log_Msg "Running PROFUMO postprocessing" echo apptainer exec --bind $(dirname "${StudyFolder}") \ From 7346fbe006defff0eb1293b8d1995da22cfb2888 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 09:57:06 -0600 Subject: [PATCH 04/74] moved example script from pipeline dir to Examples/Scripts --- Examples/Scripts/RunPFMpipelineFullRun.sh | 275 ++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100755 Examples/Scripts/RunPFMpipelineFullRun.sh diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh new file mode 100755 index 000000000..71381bfda --- /dev/null +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -0,0 +1,275 @@ +#!/bin/bash + +set -eu + +# This is an example script to run the full PFM postprocessing pipeline +# Steps involved: +# 1. ImportPFMNotes - Import PFM results and create time courses, spectra, and maps +# 2. RSNRegression - Run RSN regression on PFM data for dual regression +# 3. PFMNotesGroup - Generate group-level statistics and averages +# +# Please make sure that PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are done properly +# matching the input arguments before running this PFM pipeline + +# Global default values +DEFAULT_STUDY_FOLDER="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" +DEFAULT_SUBJECT_LIST="$(find ${DEFAULT_STUDY_FOLDER} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" +# fourRunSubjects=$(ls $DEFAULT_STUDY_FOLDER/A*/MNINonLinear/Results/BOLD_REST_4_PA -d | awk -F'/' '{print $(NF-3)}' | tr '\n' ' ') +# DEFAULT_SUBJECT_LIST=$(echo "${DEFAULT_SUBJECT_LIST}" | tr ' ' '\n' | grep -vxFf <(echo "${fourRunSubjects}" | tr ' ' '\n') | tr '\n' ' ') # remove four run subjects + + +DEFAULT_ENVIRONMENT_SCRIPT="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" +DEFAULT_GROUP_NAME="Mac25Rhesus_v5" # the group average name, which must be specified the same in MakeAverageDataset before running this tICA script +DEFAULT_REG_NAME="" # the registration string corresponding to the input files, which must be specified the same in MSMAll pipeline before running this tICA script +DEFAULT_MATLAB_MODE=1 # MatlabMode +DEFAULT_RUN_LOCAL=0 +DEFAULT_QUEUE="matlabparallelhigh.q" + +get_options() { + local scriptName=$(basename "$0") + local arguments=("$@") + + # initialize global variables + StudyFolder="${DEFAULT_STUDY_FOLDER}" + Subjlist="${DEFAULT_SUBJECT_LIST}" + EnvironmentScript="${DEFAULT_ENVIRONMENT_SCRIPT}" + GroupAverageName="${DEFAULT_GROUP_NAME}" + RegName="${DEFAULT_REG_NAME}" + MatlabMode="${DEFAULT_MATLAB_MODE}" + RunLocal="${DEFAULT_RUN_LOCAL}" + QUEUE="${DEFAULT_QUEUE}" + + # parse arguments + local index argument + + for ((index = 0; index < ${#arguments[@]}; ++index)) + do + argument="${arguments[index]}" + + case "$argument" in + --StudyFolder=*) + StudyFolder="${argument#*=}" + ;; + --Subject=*) + Subjlist="${argument#*=}" + ;; + --EnvironmentScript=*) + EnvironmentScript="${argument#*=}" + ;; + --GroupAverageName=*) + GroupAverageName="${argument#*=}" + ;; + --RegName=*) + RegName="${argument#*=}" + ;; + --MatlabMode=*) + MatlabMode="${argument#*=}" + ;; + *) + echo "ERROR: Unrecognized Option: ${argument}" + exit 1 + ;; + esac + done + + # check required parameters + if [[ "$StudyFolder" == "" ]] + then + echo "ERROR: StudyFolder not specified" + exit 1 + fi + + if [[ "$Subjlist" == "" ]] + then + echo "ERROR: Subjlist not specified" + exit 1 + fi + + if [[ "$EnvironmentScript" == "" ]] + then + echo "ERROR: EnvironmentScript not specified" + exit 1 + fi + + if [[ "$GroupAverageName" == "" ]] + then + echo "ERROR: GroupAverageName not specified" + exit 1 + fi + + # if [[ "$RegName" == "" ]] + # then + # echo "ERROR: RegName not specified" + # exit 1 + # fi + + if [[ "$MatlabMode" == "" ]] + then + echo "ERROR: MatlabMode not specified" + exit 1 + fi + + # report options + echo "-- ${scriptName}: Specified Command-Line Options: -- Start --" + echo " StudyFolder: ${StudyFolder}" + echo " Subjlist: ${Subjlist}" + echo " EnvironmentScript: ${EnvironmentScript}" + echo " GroupAverageName: ${GroupAverageName}" + echo " RegName: ${RegName}" + echo " MatlabMode: ${MatlabMode}" + echo "-- ${scriptName}: Specified Command-Line Options: -- End --" +} + + + +# +# Function Description +# Main processing of this script +# +# Gets user specified command line options and runs PFM postprocessing pipeline +# (please make sure the PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are finished before running this script) +# + +main() { + + # get command line options + get_options "$@" + + # set up pipeline environment variables and software + source "${EnvironmentScript}" + + if ((RunLocal)) || [[ "$QUEUE" == "" ]]; then + echo "running locally" + queuing_command=("$HCPPIPEDIR"/global/scripts/captureoutput.sh) + else + echo "queueing with fsl_sub to $QUEUE" + queuing_command=("$FSLDIR/bin/fsl_sub" -q "$QUEUE") + fi + + # general settings + # set the start step beginning from RunPROFUMO which is by default the first step + # StartStep="RunPROFUMO" + # StopStep="RunPROFUMO" + # StartStep="ImportPFMNotes" + # StopStep="ImportPFMNotes" + # StartStep="ImportPFMNotes" + # StartStep="RSNRegression" + # StopStep="RSNRegression" + # StartStep="RunPROFUMO" + # StopStep="RSNRegression" + + StartStep="RunPROFUMO" + StopStep="PFMNotesGroup" + + # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' + parLimit=-1 + + + # general inputs + fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" + # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" + + OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" + # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs + ConcatName="BOLD_REST_CONCAT" + # set the output spectra size for individual projection, RunsXNumTimePoints + subjectExpectedTimepoints="8508" + # set temporal highpass full-width (2*sigma) used in preprocessing + HighPass="pd2" + + + # PFM settings for REST data + PFMdim="46" # or 92 depending on your PROFUMO results + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim} + # Reference image for PROFUMO + RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" + + # set the file name component representing the preprocessing already done + fMRIProcSTRING="hp${HighPass}_clean" + + # set the mesh resolution, like '32' for 32k_fs_LR + LowResMesh="10" + + # RSN regression settings + RSNMethod="dual" # dual or single (default: dual) + LowDims="6" + FixLegacyBiasString="NO" + ScaleFactor="0.01" + + # CIFTI settings + CIFTIVertices="27559" # of rows in cifti + CIFTIVolume="66480" + + # Volume template file + VolumeTemplateCIFTI="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_WF8/tICA_d46/sICA_VolMaps_46.dscalar.nii" + + # PROFUMO settings + ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" + ProfumoConfig="${PFMFolder}/dataLocations.json" + TR="1.0" + ProfumoThreads="14" + DOFCorrection="0.5" + CovModel="Subject" + # RefImage will be auto-set based on data type below + + # build Profumo data location json + mkdir -p $PFMFolder + echo '{' > $ProfumoConfig + for Subject in $(echo $Subjlist | tr "@" "\n"); do + echo -e "\t\"$Subject\": {" >> $ProfumoConfig + for fMRIName in $(echo $fMRINames | tr "@" "\n"); do + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${fMRIProcSTRING}.dtseries.nii" + if [[ -e $runFile ]]; then + echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig + fi + done + perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma + echo -e "\t}," >> $ProfumoConfig + done + perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma + echo "}" >> $ProfumoConfig + + # PFM pipeline execution + echo "Starting PFM postprocessing pipeline..." + echo "Data type: ${OutputfMRIName}" + echo "PFM dimension: ${PFMdim}" + + "${queuing_command[@]}" "$HCPPIPEDIR"/PFM/PFMPipeline.sh \ + --study-folder="$StudyFolder" \ + --subject-list="$Subjlist" \ + --fmri-names="$fMRINames" \ + --output-fmri-name="$OutputfMRIName" \ + --proc-string="$fMRIProcSTRING" \ + --group-average-name="$GroupAverageName" \ + --pfm-dimension="$PFMdim" \ + --pfm-folder="$PFMFolder" \ + --surf-reg-name="$RegName" \ + --concat-name="$ConcatName" \ + --low-res-mesh="$LowResMesh" \ + --runs-timepoints="$subjectExpectedTimepoints" \ + --cifti-vertices="$CIFTIVertices" \ + --cifti-volume="$CIFTIVolume" \ + --rsn-method="$RSNMethod" \ + --low-dims="$LowDims" \ + --fix-legacy-bias="$FixLegacyBiasString" \ + --scale-factor="$ScaleFactor" \ + --starting-step="$StartStep" \ + --stop-after-step="$StopStep" \ + --parallel-limit="$parLimit" \ + --matlab-run-mode="$MatlabMode" \ + --profumo-config="$ProfumoConfig" \ + --profumo-singularity="$ProfumoSingularity" \ + --profumo-tr="$TR" \ + --profumo-threads="$ProfumoThreads" \ + --profumo-dof-correction="$DOFCorrection" \ + --profumo-cov-model="$CovModel" \ + --ref-image="$RefImage" \ + --volume-template-file="$VolumeTemplateCIFTI" + + echo "PFM pipeline submitted successfully!" +} + +# +# Invoke the main function to get things started +# +main "$@" \ No newline at end of file From 98ee4c2d3a1f71d801afbcfc4d8b1a0e24d1b8cc Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 10:25:49 -0600 Subject: [PATCH 05/74] Added random seed and warm start optional parameters --- Examples/Scripts/RunPFMpipelineFullRun.sh | 4 ++++ PFM/PFMPipeline.sh | 21 ++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 71381bfda..1303f065f 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -210,6 +210,8 @@ main() { ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" + nStarts="100" # number of multi-start iterations for PROFUMO + RandomSeed="2" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below # build Profumo data location json @@ -263,6 +265,8 @@ main() { --profumo-threads="$ProfumoThreads" \ --profumo-dof-correction="$DOFCorrection" \ --profumo-cov-model="$CovModel" \ + --profumo-multi-start-iterations="$nStarts"\ + --profumo-random-seed="$RandomSeed" \ --ref-image="$RefImage" \ --volume-template-file="$VolumeTemplateCIFTI" diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index ef993c88f..90b7b5df2 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -50,6 +50,8 @@ opts_AddOptional '--profumo-dof-correction' 'DOFCorrection' 'float' "DOF correct opts_AddOptional '--profumo-cov-model' 'CovModel' 'string' "covariance model for PROFUMO" 'Subject' opts_AddOptional '--profumo-singularity' 'ProfumoSingularity' 'path' "path to PROFUMO singularity container" opts_AddOptional '--profumo-random-seed' 'RandomSeed' 'integer' "random seed for PROFUMO" '123' +opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'integer' "number of iterations of group-level spatial decomposition before inferring full model" '5' +opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -119,9 +121,9 @@ then RegString="_$RegName" fi -# Auto-generate output strings -OutputSTRING="_d${PFMdim}_${GroupAverageName}_WR" -OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_PFMs_tclean" +# Auto-generate output strings (include random seed for uniqueness) +OutputSTRING="_d${PFMdim}_${GroupAverageName}_seed${RandomSeed}_WR" +OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${RandomSeed}_PFMs_tclean" # Volume template file path # VolumeTemplateFile="${StudyFolder}/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.2.dscalar.nii" @@ -158,6 +160,13 @@ do # Create output directory mkdir -p "${PFMFolder}" + # Build optional initialMaps argument + InitialMapsArg="" + if [[ -n "${InitialMaps}" && -f "${InitialMaps}" ]] + then + InitialMapsArg="--initialMaps ${InitialMaps}" + fi + # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ @@ -165,14 +174,16 @@ do /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ - --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" + --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" \ + --multiStartIterations "${MultiStartIterations}" ${InitialMapsArg} apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ - --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" + --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ + --multiStartIterations "${MultiStartIterations}" ${InitialMapsArg} log_Msg "Running PROFUMO postprocessing" echo apptainer exec --bind $(dirname "${StudyFolder}") \ From 359d752a1f721eab51a9b2fa76cc7975b496d951 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 10:27:42 -0600 Subject: [PATCH 06/74] remove script at old location --- PFM/RunPFMpipelineFullRun.sh | 275 ----------------------------------- 1 file changed, 275 deletions(-) delete mode 100755 PFM/RunPFMpipelineFullRun.sh diff --git a/PFM/RunPFMpipelineFullRun.sh b/PFM/RunPFMpipelineFullRun.sh deleted file mode 100755 index 71381bfda..000000000 --- a/PFM/RunPFMpipelineFullRun.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/bin/bash - -set -eu - -# This is an example script to run the full PFM postprocessing pipeline -# Steps involved: -# 1. ImportPFMNotes - Import PFM results and create time courses, spectra, and maps -# 2. RSNRegression - Run RSN regression on PFM data for dual regression -# 3. PFMNotesGroup - Generate group-level statistics and averages -# -# Please make sure that PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are done properly -# matching the input arguments before running this PFM pipeline - -# Global default values -DEFAULT_STUDY_FOLDER="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" -DEFAULT_SUBJECT_LIST="$(find ${DEFAULT_STUDY_FOLDER} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" -# fourRunSubjects=$(ls $DEFAULT_STUDY_FOLDER/A*/MNINonLinear/Results/BOLD_REST_4_PA -d | awk -F'/' '{print $(NF-3)}' | tr '\n' ' ') -# DEFAULT_SUBJECT_LIST=$(echo "${DEFAULT_SUBJECT_LIST}" | tr ' ' '\n' | grep -vxFf <(echo "${fourRunSubjects}" | tr ' ' '\n') | tr '\n' ' ') # remove four run subjects - - -DEFAULT_ENVIRONMENT_SCRIPT="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" -DEFAULT_GROUP_NAME="Mac25Rhesus_v5" # the group average name, which must be specified the same in MakeAverageDataset before running this tICA script -DEFAULT_REG_NAME="" # the registration string corresponding to the input files, which must be specified the same in MSMAll pipeline before running this tICA script -DEFAULT_MATLAB_MODE=1 # MatlabMode -DEFAULT_RUN_LOCAL=0 -DEFAULT_QUEUE="matlabparallelhigh.q" - -get_options() { - local scriptName=$(basename "$0") - local arguments=("$@") - - # initialize global variables - StudyFolder="${DEFAULT_STUDY_FOLDER}" - Subjlist="${DEFAULT_SUBJECT_LIST}" - EnvironmentScript="${DEFAULT_ENVIRONMENT_SCRIPT}" - GroupAverageName="${DEFAULT_GROUP_NAME}" - RegName="${DEFAULT_REG_NAME}" - MatlabMode="${DEFAULT_MATLAB_MODE}" - RunLocal="${DEFAULT_RUN_LOCAL}" - QUEUE="${DEFAULT_QUEUE}" - - # parse arguments - local index argument - - for ((index = 0; index < ${#arguments[@]}; ++index)) - do - argument="${arguments[index]}" - - case "$argument" in - --StudyFolder=*) - StudyFolder="${argument#*=}" - ;; - --Subject=*) - Subjlist="${argument#*=}" - ;; - --EnvironmentScript=*) - EnvironmentScript="${argument#*=}" - ;; - --GroupAverageName=*) - GroupAverageName="${argument#*=}" - ;; - --RegName=*) - RegName="${argument#*=}" - ;; - --MatlabMode=*) - MatlabMode="${argument#*=}" - ;; - *) - echo "ERROR: Unrecognized Option: ${argument}" - exit 1 - ;; - esac - done - - # check required parameters - if [[ "$StudyFolder" == "" ]] - then - echo "ERROR: StudyFolder not specified" - exit 1 - fi - - if [[ "$Subjlist" == "" ]] - then - echo "ERROR: Subjlist not specified" - exit 1 - fi - - if [[ "$EnvironmentScript" == "" ]] - then - echo "ERROR: EnvironmentScript not specified" - exit 1 - fi - - if [[ "$GroupAverageName" == "" ]] - then - echo "ERROR: GroupAverageName not specified" - exit 1 - fi - - # if [[ "$RegName" == "" ]] - # then - # echo "ERROR: RegName not specified" - # exit 1 - # fi - - if [[ "$MatlabMode" == "" ]] - then - echo "ERROR: MatlabMode not specified" - exit 1 - fi - - # report options - echo "-- ${scriptName}: Specified Command-Line Options: -- Start --" - echo " StudyFolder: ${StudyFolder}" - echo " Subjlist: ${Subjlist}" - echo " EnvironmentScript: ${EnvironmentScript}" - echo " GroupAverageName: ${GroupAverageName}" - echo " RegName: ${RegName}" - echo " MatlabMode: ${MatlabMode}" - echo "-- ${scriptName}: Specified Command-Line Options: -- End --" -} - - - -# -# Function Description -# Main processing of this script -# -# Gets user specified command line options and runs PFM postprocessing pipeline -# (please make sure the PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are finished before running this script) -# - -main() { - - # get command line options - get_options "$@" - - # set up pipeline environment variables and software - source "${EnvironmentScript}" - - if ((RunLocal)) || [[ "$QUEUE" == "" ]]; then - echo "running locally" - queuing_command=("$HCPPIPEDIR"/global/scripts/captureoutput.sh) - else - echo "queueing with fsl_sub to $QUEUE" - queuing_command=("$FSLDIR/bin/fsl_sub" -q "$QUEUE") - fi - - # general settings - # set the start step beginning from RunPROFUMO which is by default the first step - # StartStep="RunPROFUMO" - # StopStep="RunPROFUMO" - # StartStep="ImportPFMNotes" - # StopStep="ImportPFMNotes" - # StartStep="ImportPFMNotes" - # StartStep="RSNRegression" - # StopStep="RSNRegression" - # StartStep="RunPROFUMO" - # StopStep="RSNRegression" - - StartStep="RunPROFUMO" - StopStep="PFMNotesGroup" - - # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' - parLimit=-1 - - - # general inputs - fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" - # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" - - OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" - # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs - ConcatName="BOLD_REST_CONCAT" - # set the output spectra size for individual projection, RunsXNumTimePoints - subjectExpectedTimepoints="8508" - # set temporal highpass full-width (2*sigma) used in preprocessing - HighPass="pd2" - - - # PFM settings for REST data - PFMdim="46" # or 92 depending on your PROFUMO results - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim} - # Reference image for PROFUMO - RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" - - # set the file name component representing the preprocessing already done - fMRIProcSTRING="hp${HighPass}_clean" - - # set the mesh resolution, like '32' for 32k_fs_LR - LowResMesh="10" - - # RSN regression settings - RSNMethod="dual" # dual or single (default: dual) - LowDims="6" - FixLegacyBiasString="NO" - ScaleFactor="0.01" - - # CIFTI settings - CIFTIVertices="27559" # of rows in cifti - CIFTIVolume="66480" - - # Volume template file - VolumeTemplateCIFTI="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_WF8/tICA_d46/sICA_VolMaps_46.dscalar.nii" - - # PROFUMO settings - ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" - ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="1.0" - ProfumoThreads="14" - DOFCorrection="0.5" - CovModel="Subject" - # RefImage will be auto-set based on data type below - - # build Profumo data location json - mkdir -p $PFMFolder - echo '{' > $ProfumoConfig - for Subject in $(echo $Subjlist | tr "@" "\n"); do - echo -e "\t\"$Subject\": {" >> $ProfumoConfig - for fMRIName in $(echo $fMRINames | tr "@" "\n"); do - runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${fMRIProcSTRING}.dtseries.nii" - if [[ -e $runFile ]]; then - echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig - fi - done - perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma - echo -e "\t}," >> $ProfumoConfig - done - perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma - echo "}" >> $ProfumoConfig - - # PFM pipeline execution - echo "Starting PFM postprocessing pipeline..." - echo "Data type: ${OutputfMRIName}" - echo "PFM dimension: ${PFMdim}" - - "${queuing_command[@]}" "$HCPPIPEDIR"/PFM/PFMPipeline.sh \ - --study-folder="$StudyFolder" \ - --subject-list="$Subjlist" \ - --fmri-names="$fMRINames" \ - --output-fmri-name="$OutputfMRIName" \ - --proc-string="$fMRIProcSTRING" \ - --group-average-name="$GroupAverageName" \ - --pfm-dimension="$PFMdim" \ - --pfm-folder="$PFMFolder" \ - --surf-reg-name="$RegName" \ - --concat-name="$ConcatName" \ - --low-res-mesh="$LowResMesh" \ - --runs-timepoints="$subjectExpectedTimepoints" \ - --cifti-vertices="$CIFTIVertices" \ - --cifti-volume="$CIFTIVolume" \ - --rsn-method="$RSNMethod" \ - --low-dims="$LowDims" \ - --fix-legacy-bias="$FixLegacyBiasString" \ - --scale-factor="$ScaleFactor" \ - --starting-step="$StartStep" \ - --stop-after-step="$StopStep" \ - --parallel-limit="$parLimit" \ - --matlab-run-mode="$MatlabMode" \ - --profumo-config="$ProfumoConfig" \ - --profumo-singularity="$ProfumoSingularity" \ - --profumo-tr="$TR" \ - --profumo-threads="$ProfumoThreads" \ - --profumo-dof-correction="$DOFCorrection" \ - --profumo-cov-model="$CovModel" \ - --ref-image="$RefImage" \ - --volume-template-file="$VolumeTemplateCIFTI" - - echo "PFM pipeline submitted successfully!" -} - -# -# Invoke the main function to get things started -# -main "$@" \ No newline at end of file From 427e75ae4c79c3016917bef14bb68095fe91cca0 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 10:44:02 -0600 Subject: [PATCH 07/74] Update PFM/PFMPipeline.sh Co-authored-by: Tim Coalson --- PFM/PFMPipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 90b7b5df2..8111c6722 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -36,9 +36,9 @@ opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g. opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder containing Results.ppp" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" opts_AddMandatory '--profumo-config' 'ProfumoConfig' 'path' "path to PROFUMO JSON configuration file" -opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' +opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO postprocessing" -opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" '4800' +opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" opts_AddMandatory '--cifti-vertices' 'CIFTIVertices' "number of CIFTI vertices" '91282' opts_AddMandatory '--cifti-volume' 'CIFTIVolume' "number of CIFTI volume vertices" '283226' opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' From bcef6f5e915e2b00a509173792ad8262c3d400e3 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 10:54:05 -0600 Subject: [PATCH 08/74] removed unnecessary unix call and cifti magic numbers --- PFM/PFMPipeline.sh | 2 -- PFM/scripts/ImportPFMNotes.m | 2 +- PFM/scripts/PFMNotes.m | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 8111c6722..b8a110da0 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -39,8 +39,6 @@ opts_AddMandatory '--profumo-config' 'ProfumoConfig' 'path' "path to PROFUMO JSO opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO postprocessing" opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" -opts_AddMandatory '--cifti-vertices' 'CIFTIVertices' "number of CIFTI vertices" '91282' -opts_AddMandatory '--cifti-volume' 'CIFTIVolume' "number of CIFTI volume vertices" '283226' opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' diff --git a/PFM/scripts/ImportPFMNotes.m b/PFM/scripts/ImportPFMNotes.m index 716c28546..e683ef7d0 100755 --- a/PFM/scripts/ImportPFMNotes.m +++ b/PFM/scripts/ImportPFMNotes.m @@ -66,7 +66,7 @@ function ImportPFMNotes(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIP ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - unix(['cp ' PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii ' StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); + copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); end end end \ No newline at end of file diff --git a/PFM/scripts/PFMNotes.m b/PFM/scripts/PFMNotes.m index 10f840be8..98e558e82 100755 --- a/PFM/scripts/PFMNotes.m +++ b/PFM/scripts/PFMNotes.m @@ -4,9 +4,6 @@ function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRINam Subjlist = strsplit(SubjListRaw, '@'); PFMdim = str2double(PFMdimStr); RunsXNumTimePoints = str2double(RunsXNumTimePointsStr); - CIFTI = str2double(CIFTIVerticesStr); - CIFTIVol = str2double(CIFTIVolumeStr); - wbcommand = 'wb_command'; c = 1; From 23260a943c2ba29d882ccfd1c9716f64ec0f9aba Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 20 Jan 2026 14:24:14 -0600 Subject: [PATCH 09/74] fixed mandatory newopts syntax --- PFM/PFMPipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index b8a110da0..9c1294045 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -36,9 +36,9 @@ opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g. opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder containing Results.ppp" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" opts_AddMandatory '--profumo-config' 'ProfumoConfig' 'path' "path to PROFUMO JSON configuration file" -opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" +opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" "repetition time for PROFUMO analysis in seconds" opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO postprocessing" -opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" +opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs" "total timepoints across runs" opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' From e45ea9332f60909a5537167a8b09d0f0d8d152ca Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Fri, 30 Jan 2026 14:41:26 -0600 Subject: [PATCH 10/74] nStarts 1000 in example script --- Examples/Scripts/RunPFMpipelineFullRun.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 1303f065f..c1c0385e5 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -168,6 +168,7 @@ main() { # general inputs fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" + randSeed=2 # random seed for PROFUMO OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs @@ -179,8 +180,8 @@ main() { # PFM settings for REST data - PFMdim="46" # or 92 depending on your PROFUMO results - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim} + PFMdim="16" # set the PFM dimensionality + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim}_s${randSeed}_M1k # Reference image for PROFUMO RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" @@ -196,22 +197,18 @@ main() { FixLegacyBiasString="NO" ScaleFactor="0.01" - # CIFTI settings - CIFTIVertices="27559" # of rows in cifti - CIFTIVolume="66480" - # Volume template file - VolumeTemplateCIFTI="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_WF8/tICA_d46/sICA_VolMaps_46.dscalar.nii" + VolumeTemplateCIFTI="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus/Mac25Rhesus_v5/MNINonLinear/Results/Mac25Rhesus_v5_VolMaps_16_template.dscalar.nii" # PROFUMO settings ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="1.0" + TR="0.702" ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" - nStarts="100" # number of multi-start iterations for PROFUMO - RandomSeed="2" # random seed for PROFUMO reproducibility + nStarts="1000" # number of multi-start iterations for PROFUMO + RandomSeed="$randSeed" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below # build Profumo data location json @@ -249,8 +246,6 @@ main() { --concat-name="$ConcatName" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ - --cifti-vertices="$CIFTIVertices" \ - --cifti-volume="$CIFTIVolume" \ --rsn-method="$RSNMethod" \ --low-dims="$LowDims" \ --fix-legacy-bias="$FixLegacyBiasString" \ From 2868e127d38272d516b4ed23345fbacd95686b34 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 10:59:41 -0600 Subject: [PATCH 11/74] remove global DEFAULTS from batch script. Rename functions. --- Examples/Scripts/RunPFMpipelineFullRun.sh | 49 +++++++------------ PFM/PFMPipeline.sh | 16 +++--- PFM/scripts/{PFMNotesGroup.m => GroupPFMs.m} | 5 +- .../{PFMNotesGroup.sh => GroupPFMs.sh} | 4 +- .../{ImportPFMNotes.m => PostPROFUMO.m} | 17 +++---- .../{ImportPFMNotes.sh => PostPROFUMO.sh} | 4 +- PFM/scripts/{PFMNotes.m => RunPROFUMO.m} | 18 +++---- PFM/scripts/{PFMNotes.sh => RunPROFUMO.sh} | 6 +-- 8 files changed, 50 insertions(+), 69 deletions(-) rename PFM/scripts/{PFMNotesGroup.m => GroupPFMs.m} (96%) rename PFM/scripts/{PFMNotesGroup.sh => GroupPFMs.sh} (95%) rename PFM/scripts/{ImportPFMNotes.m => PostPROFUMO.m} (79%) rename PFM/scripts/{ImportPFMNotes.sh => PostPROFUMO.sh} (95%) rename PFM/scripts/{PFMNotes.m => RunPROFUMO.m} (88%) rename PFM/scripts/{PFMNotes.sh => RunPROFUMO.sh} (94%) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index c1c0385e5..543070edc 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -4,40 +4,27 @@ set -eu # This is an example script to run the full PFM postprocessing pipeline # Steps involved: -# 1. ImportPFMNotes - Import PFM results and create time courses, spectra, and maps -# 2. RSNRegression - Run RSN regression on PFM data for dual regression -# 3. PFMNotesGroup - Generate group-level statistics and averages +# 1. RunPROFUMO - Run PROFUMO analysis +# 2. PostPROFUMO - Create time courses, spectra, and maps from PFM results +# 3. RSNRegression - Run RSN regression on PFM data for dual regression +# 4. GroupPFMs - Generate group-level statistics and averages # # Please make sure that PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are done properly -# matching the input arguments before running this PFM pipeline - -# Global default values -DEFAULT_STUDY_FOLDER="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" -DEFAULT_SUBJECT_LIST="$(find ${DEFAULT_STUDY_FOLDER} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" -# fourRunSubjects=$(ls $DEFAULT_STUDY_FOLDER/A*/MNINonLinear/Results/BOLD_REST_4_PA -d | awk -F'/' '{print $(NF-3)}' | tr '\n' ' ') -# DEFAULT_SUBJECT_LIST=$(echo "${DEFAULT_SUBJECT_LIST}" | tr ' ' '\n' | grep -vxFf <(echo "${fourRunSubjects}" | tr ' ' '\n') | tr '\n' ' ') # remove four run subjects - - -DEFAULT_ENVIRONMENT_SCRIPT="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" -DEFAULT_GROUP_NAME="Mac25Rhesus_v5" # the group average name, which must be specified the same in MakeAverageDataset before running this tICA script -DEFAULT_REG_NAME="" # the registration string corresponding to the input files, which must be specified the same in MSMAll pipeline before running this tICA script -DEFAULT_MATLAB_MODE=1 # MatlabMode -DEFAULT_RUN_LOCAL=0 -DEFAULT_QUEUE="matlabparallelhigh.q" +# matching the input arguments before running this PFM pipeline get_options() { local scriptName=$(basename "$0") local arguments=("$@") - # initialize global variables - StudyFolder="${DEFAULT_STUDY_FOLDER}" - Subjlist="${DEFAULT_SUBJECT_LIST}" - EnvironmentScript="${DEFAULT_ENVIRONMENT_SCRIPT}" - GroupAverageName="${DEFAULT_GROUP_NAME}" - RegName="${DEFAULT_REG_NAME}" - MatlabMode="${DEFAULT_MATLAB_MODE}" - RunLocal="${DEFAULT_RUN_LOCAL}" - QUEUE="${DEFAULT_QUEUE}" + # initialize variables + StudyFolder="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" + Subjlist="$(find ${StudyFolder} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" + EnvironmentScript="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" + GroupAverageName="Mac25Rhesus_v5" + RegName="" + MatlabMode=1 + RunLocal=0 + QUEUE="matlabparallelhigh.q" # parse arguments local index argument @@ -150,16 +137,16 @@ main() { # set the start step beginning from RunPROFUMO which is by default the first step # StartStep="RunPROFUMO" # StopStep="RunPROFUMO" - # StartStep="ImportPFMNotes" - # StopStep="ImportPFMNotes" - # StartStep="ImportPFMNotes" + # StartStep="PostPROFUMO" + # StopStep="PostPROFUMO" + # StartStep="PostPROFUMO" # StartStep="RSNRegression" # StopStep="RSNRegression" # StartStep="RunPROFUMO" # StopStep="RSNRegression" StartStep="RunPROFUMO" - StopStep="PFMNotesGroup" + StopStep="GroupPFMs" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' parLimit=-1 diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 9c1294045..663f0c4f8 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -16,13 +16,13 @@ source "$HCPPIPEDIR/global/scripts/parallel.shlib" "$@" g_matlab_default_mode=1 -#add steps to this array and in the switch cases below -pipelineSteps=(RunPROFUMO ImportPFMNotes RSNRegression PFMNotesGroup) +# add steps to this array and in the switch cases below +pipelineSteps=(RunPROFUMO PostPROFUMO RSNRegression GroupPFMs) defaultStart="${pipelineSteps[0]}" defaultStopAfter="${pipelineSteps[${#pipelineSteps[@]} - 1]}" stepsText="$(IFS=$'\n'; echo "${pipelineSteps[*]}")" -#description to use in usage - syntax of parameters is now explained automatically +#description to use in usage opts_SetScriptDescription "implements complete PFM pipeline with four main steps: Run PROFUMO, Import PFM Notes, RSN Regression, and PFM Notes Group processing" #mandatory parameters @@ -202,8 +202,8 @@ do "${REAL_REF_IMAGE}" ;; (ImportPFMNotes) - log_Msg "Running ImportPFMNotes step" - "$HCPPIPEDIR"/PFM/scripts/ImportPFMNotes.sh \ + log_Msg "Running PostPROFUMO step" + "$HCPPIPEDIR"/PFM/scripts/PostPROFUMO.sh \ --study-folder="$StudyFolder" \ --subject-list="$SubjlistRaw" \ --fmri-names="$fMRINames" \ @@ -296,9 +296,9 @@ do # Run the jobs par_runjobs "$parLimit" ;; - (PFMNotesGroup) - log_Msg "Running PFMNotesGroup step" - "$HCPPIPEDIR"/PFM/scripts/PFMNotesGroup.sh \ + (GroupPFMs) + log_Msg "Running GroupPFMs step" + "$HCPPIPEDIR"/PFM/scripts/GroupPFMs.sh \ --study-folder="$StudyFolder" \ --subject-list="$SubjlistRaw" \ --pfm-dimension="$PFMdim" \ diff --git a/PFM/scripts/PFMNotesGroup.m b/PFM/scripts/GroupPFMs.m similarity index 96% rename from PFM/scripts/PFMNotesGroup.m rename to PFM/scripts/GroupPFMs.m index 68e499069..44b82c33f 100755 --- a/PFM/scripts/PFMNotesGroup.m +++ b/PFM/scripts/GroupPFMs.m @@ -1,4 +1,4 @@ -function PFMNotesGroup(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) +function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) %% example parameters for 1071 3T MSMAll Rest % StudyFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final'; @@ -9,9 +9,6 @@ function PFMNotesGroup(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, % LowResMesh = '32'; % RunsXNumTimePoints = 4800; % PFMFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll3T1071/MNINonLinear/Results/rfMRI_REST/PFMs_chpc'; - % - % CIFTIVertices = 91282; % no longer used - % CIFTIVol = 283226 ;% no longer used wbcommand = 'wb_command'; diff --git a/PFM/scripts/PFMNotesGroup.sh b/PFM/scripts/GroupPFMs.sh similarity index 95% rename from PFM/scripts/PFMNotesGroup.sh rename to PFM/scripts/GroupPFMs.sh index 2c82aa13b..1c943b597 100755 --- a/PFM/scripts/PFMNotesGroup.sh +++ b/PFM/scripts/GroupPFMs.sh @@ -86,7 +86,7 @@ matlab_argarray=("$StudyFolder" "$SubjlistRaw" "$PFMdim" "$OutputPrefix" "$RegSt case "$MatlabMode" in (0) - matlab_cmd=("$this_script_dir/Compiled_PFMNotesGroup/run_PFMNotesGroup.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + matlab_cmd=("$this_script_dir/Compiled_GroupPFMs/run_GroupPFMs.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") log_Msg "running compiled matlab command: ${matlab_cmd[*]}" "${matlab_cmd[@]}" ;; @@ -105,7 +105,7 @@ case "$MatlabMode" in addpath('$HCPPIPEDIR/global/matlab'); addpath('$this_script_dir'); addpath('$HCPCIFTIRWDIR'); - PFMNotesGroup($matlab_args);" + GroupPFMs($matlab_args);" log_Msg "running matlab code: $matlabcode" "${matlab_interpreter[@]}" <<<"$matlabcode" diff --git a/PFM/scripts/ImportPFMNotes.m b/PFM/scripts/PostPROFUMO.m similarity index 79% rename from PFM/scripts/ImportPFMNotes.m rename to PFM/scripts/PostPROFUMO.m index e683ef7d0..5b996d430 100755 --- a/PFM/scripts/ImportPFMNotes.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,17 +1,17 @@ -function ImportPFMNotes(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) Subjlist = strsplit(SubjListRaw, '@'); fMRINames = strsplit(fMRIListRaw, '@'); TR = str2double(TR); wbcommand = 'wb_command'; - for s = 1:length(Subjlist) + for s = 1:numel(Subjlist) s subfMRINames = {}; if ~strcmp(ConcatName, '') if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii']) c = 1; - for r = 1:length(fMRINames) + for r = 1:numel(fMRINames) if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) subfMRINames{c} = fMRINames{r}; c = c + 1; @@ -20,7 +20,7 @@ function ImportPFMNotes(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIP end else c = 1; - for r = 1:length(fMRINames) + for r = 1:numel(fMRINames) if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) subfMRINames{c} = fMRINames{r}; c = c + 1; @@ -28,20 +28,17 @@ function ImportPFMNotes(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIP end end - if length(subfMRINames) ~= 0 + if numel(subfMRINames) ~= 0 origTCS = []; TCS = []; - for r = 1:length(subfMRINames) + for r = 1:numel(subfMRINames) runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', length(runTCS), 1)]; + TCS = [TCS ; runTCS .* repmat(runAmp', numel(runTCS), 1)]; end - % sICATCS = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputfMRIName OutputSTRING RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - % sICASpectra = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputfMRIName OutputSTRING RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); % PFMTCSorig.diminfo{1,2} = sICATCS.diminfo{1,2}; ts.Nnodes = size(origTCS, 2); diff --git a/PFM/scripts/ImportPFMNotes.sh b/PFM/scripts/PostPROFUMO.sh similarity index 95% rename from PFM/scripts/ImportPFMNotes.sh rename to PFM/scripts/PostPROFUMO.sh index 9f925eb97..66e7aab55 100755 --- a/PFM/scripts/ImportPFMNotes.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -94,7 +94,7 @@ matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fM case "$MatlabMode" in (0) - matlab_cmd=("$this_script_dir/Compiled_ImportPFMNotes/run_ImportPFMNotes.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + matlab_cmd=("$this_script_dir/Compiled_PostPROFUMO/run_PostPROFUMO.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") log_Msg "running compiled matlab command: ${matlab_cmd[*]}" "${matlab_cmd[@]}" ;; @@ -113,7 +113,7 @@ case "$MatlabMode" in addpath('$HCPPIPEDIR/global/matlab'); addpath('$this_script_dir'); addpath('$HCPCIFTIRWDIR'); - ImportPFMNotes($matlab_args);" + PostPROFUMO($matlab_args);" log_Msg "running matlab code: $matlabcode" "${matlab_interpreter[@]}" <<<"$matlabcode" diff --git a/PFM/scripts/PFMNotes.m b/PFM/scripts/RunPROFUMO.m similarity index 88% rename from PFM/scripts/PFMNotes.m rename to PFM/scripts/RunPROFUMO.m index 98e558e82..608c9f0a0 100755 --- a/PFM/scripts/PFMNotes.m +++ b/PFM/scripts/RunPROFUMO.m @@ -1,4 +1,4 @@ -function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRIName, PFMdimStr, OutputPrefix, RegString, LowResMesh, RunsXNumTimePointsStr, CIFTIVerticesStr, CIFTIVolumeStr, PFMFolder) +function RunPROFUMO(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRIName, PFMdimStr, OutputPrefix, RegString, LowResMesh, RunsXNumTimePointsStr, CIFTIVerticesStr, CIFTIVolumeStr, PFMFolder) % Parse string inputs Subjlist = strsplit(SubjListRaw, '@'); @@ -9,13 +9,13 @@ function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRINam c = 1; SubjFolderlist = {}; StudyFolderNumber = []; - for i = 1:length(Subjlist) + for i = 1:numel(Subjlist) SubjFolderlist{c} = [StudyFolder '/' Subjlist{i}]; StudyFolderNumber = [StudyFolderNumber 1]; c = c + 1; end - for i = 1:length(Subjlist) + for i = 1:numel(Subjlist) if ~isfile([StudyFolder '/' Subjlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii']) Subjlist{i} end @@ -23,13 +23,13 @@ function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRINam s1 = 0; m1 = 0; - TCSMask = zeros(PFMdim, RunsXNumTimePoints, length(SubjFolderlist), 'single'); - TCSAll = zeros(PFMdim, RunsXNumTimePoints, length(SubjFolderlist), 'single'); + TCSMask = zeros(PFMdim, RunsXNumTimePoints, numel(SubjFolderlist), 'single'); + TCSAll = zeros(PFMdim, RunsXNumTimePoints, numel(SubjFolderlist), 'single'); SpectraOne = []; PFMMapsOne = []; PFMVolMapsOne = []; - for i = 1:length(SubjFolderlist) + for i = 1:numel(SubjFolderlist) Subjlist{i} if exist([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii']) PFMMapsSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); @@ -45,7 +45,7 @@ function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRINam COND(i) = cond(TCSSub.cdata); if StudyFolderNumber(i) == 1 - if length(TCSSub.cdata) == RunsXNumTimePoints + if numel(TCSSub.cdata) == RunsXNumTimePoints TCSMask(:, :, i) = repmat(1, PFMdim, RunsXNumTimePoints, 1); if isempty(SpectraOne) SpectraOne = SpectraSub; @@ -68,9 +68,9 @@ function PFMNotesGroup(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRINam end TCSMaskConcat = TCSSub; - TCSMaskConcat.cdata = squeeze(reshape(TCSMask, PFMdim, RunsXNumTimePoints * length(SubjFolderlist))); + TCSMaskConcat.cdata = squeeze(reshape(TCSMask, PFMdim, RunsXNumTimePoints * numel(SubjFolderlist))); TCSFullConcat = TCSSub; - TCSFullConcat.cdata = squeeze(reshape(TCSAll, PFMdim, RunsXNumTimePoints * length(SubjFolderlist))); + TCSFullConcat.cdata = squeeze(reshape(TCSAll, PFMdim, RunsXNumTimePoints * numel(SubjFolderlist))); ciftisavereset(TCSMaskConcat, [PFMFolder '/PFM_TCSMASK_' num2str(PFMdim) '.sdseries.nii'], wbcommand); ciftisavereset(TCSFullConcat, [PFMFolder '/PFM_TCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); diff --git a/PFM/scripts/PFMNotes.sh b/PFM/scripts/RunPROFUMO.sh similarity index 94% rename from PFM/scripts/PFMNotes.sh rename to PFM/scripts/RunPROFUMO.sh index 31e541f3b..15e087f13 100755 --- a/PFM/scripts/PFMNotes.sh +++ b/PFM/scripts/RunPROFUMO.sh @@ -92,7 +92,7 @@ matlab_argarray=("$StudyFolder" "$SubjListRaw" "$GroupAverageName" "$OutputfMRIN case "$MatlabMode" in (0) - matlab_cmd=("$this_script_dir/Compiled_PFMNotesGroup/run_PFMNotesGroup.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + matlab_cmd=("$this_script_dir/Compiled_RunPROFUMO/run_RunPROFUMO.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") log_Msg "running compiled matlab command: ${matlab_cmd[*]}" "${matlab_cmd[@]}" ;; @@ -111,7 +111,7 @@ case "$MatlabMode" in addpath('$HCPPIPEDIR/global/matlab'); addpath('$this_script_dir'); addpath('$HCPCIFTIRWDIR'); - PFMNotesGroup($matlab_args);" + RunPROFUMO($matlab_args);" log_Msg "running matlab code: $matlabcode" "${matlab_interpreter[@]}" <<<"$matlabcode" @@ -119,4 +119,4 @@ case "$MatlabMode" in ;; esac -log_Msg "PFMNotesGroup step completed successfully" \ No newline at end of file +log_Msg "RunPROFUMO step completed successfully" \ No newline at end of file From 42f9c9892063c22d462dd0574b5f869ad64d3fc5 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 12:23:11 -0600 Subject: [PATCH 12/74] remove single regression option --- Examples/Scripts/RunPFMpipelineFullRun.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 543070edc..4d96e47ba 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -179,7 +179,6 @@ main() { LowResMesh="10" # RSN regression settings - RSNMethod="dual" # dual or single (default: dual) LowDims="6" FixLegacyBiasString="NO" ScaleFactor="0.01" @@ -233,7 +232,7 @@ main() { --concat-name="$ConcatName" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ - --rsn-method="$RSNMethod" \ + --rsn-method=dual \ --low-dims="$LowDims" \ --fix-legacy-bias="$FixLegacyBiasString" \ --scale-factor="$ScaleFactor" \ From da38924f9b6da756f5f6c63f8d2b92a5634405ae Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 12:23:37 -0600 Subject: [PATCH 13/74] remove single regression option --- PFM/PFMPipeline.sh | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 663f0c4f8..459d37e17 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -23,7 +23,7 @@ defaultStopAfter="${pipelineSteps[${#pipelineSteps[@]} - 1]}" stepsText="$(IFS=$'\n'; echo "${pipelineSteps[*]}")" #description to use in usage -opts_SetScriptDescription "implements complete PFM pipeline with four main steps: Run PROFUMO, Import PFM Notes, RSN Regression, and PFM Notes Group processing" +opts_SetScriptDescription "implements complete PFM pipeline with four main steps: Run PROFUMO, Post-PROFUMO, RSN Regression, and Group PFM processing" #mandatory parameters opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains all subjects" @@ -36,7 +36,7 @@ opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g. opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder containing Results.ppp" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" opts_AddMandatory '--profumo-config' 'ProfumoConfig' 'path' "path to PROFUMO JSON configuration file" -opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" "repetition time for PROFUMO analysis in seconds" +opts_AddMandatory '--profumo-tr' 'TR' "seconds" "repetition time for PROFUMO analysis" opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO postprocessing" opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs" "total timepoints across runs" opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' @@ -55,7 +55,6 @@ opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initiali opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' #RSN regression specific parameters -opts_AddOptional '--rsn-method' 'RSNMethod' 'weighted or dual' "RSN regression method: weighted (WR) or dual (DR) - default: dual" 'dual' opts_AddOptional '--low-dims' 'LowDims' 'string' "low dimensionalities for RSN regression" '7@8@9@10@11@12@13@14@15@16@17@18@19@20@21' opts_AddOptional '--low-dims-template-file' 'LowDimTemplate' 'path' "low dimensionality template name for RSN regression" '' opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' @@ -247,20 +246,10 @@ do continue fi - # Set maps and timeseries based on method (following your example exactly) - if [[ "$RSNMethod" == "single" ]] - then - # TimeSeries="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_${RegName}_ts.${LowResMesh}k_fs_LR.sdseries.nii" - TimeSeries="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_ts.${LowResMesh}k_fs_LR.sdseries.nii" - elif [[ "$RSNMethod" == "dual" ]] - then - # GroupMaps="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_${RegName}_origmaps.${LowResMesh}k_fs_LR.dscalar.nii" - # GroupMaps="${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.${OutputPrefix}_origmaps.${LowResMesh}k_fs_LR.dscalar.nii" - GroupMaps="${PFMFolder}/Results.ppp/Maps/Group.dscalar.nii" - TimeSeries="" - fi + # Set maps for dual regression + GroupMaps="${PFMFolder}/Results.ppp/Maps/Group.dscalar.nii" - # Build RSN regression command (exactly like your example) + # Build RSN regression command rsn_cmd=("$HCPPIPEDIR"/global/scripts/RSNregression.sh --study-folder="$StudyFolder" --subject="$Subject" @@ -277,18 +266,9 @@ do --output-z=1 --fix-legacy-bias="$FixLegacyBias" --scale-factor="$ScaleFactor" + --group-maps="$GroupMaps" ) - # Add conditional arguments based on method - if [[ "$RSNMethod" == "dual" ]] - then - rsn_cmd+=(--group-maps="$GroupMaps") - fi - if [[ "$RSNMethod" == "single" ]] - then - rsn_cmd+=(--timeseries="$TimeSeries") - fi - # Queue parallel job par_addjob "${rsn_cmd[@]}" done From 61dd140e2e6dee2a3ef6614316abcc7e15febd2a Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 12:47:44 -0600 Subject: [PATCH 14/74] improve commments --- PFM/scripts/GroupPFMs.m | 223 ++++++++++++++++++++------------------ PFM/scripts/PostPROFUMO.m | 149 ++++++++++++++++--------- 2 files changed, 213 insertions(+), 159 deletions(-) diff --git a/PFM/scripts/GroupPFMs.m b/PFM/scripts/GroupPFMs.m index 44b82c33f..f68b37756 100755 --- a/PFM/scripts/GroupPFMs.m +++ b/PFM/scripts/GroupPFMs.m @@ -1,122 +1,135 @@ function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) - - %% example parameters for 1071 3T MSMAll Rest - % StudyFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final'; - % SubjlistRaw = "100206@100307"; - % PFMdim = 92; - % OutputPrefix = ['rfMRI_REST_d92_WF5_S1200_MSMAll3T1071_PFMs_tclean']; - % RegName = '_MSMAll'; %e.g. _MSMAll - % LowResMesh = '32'; - % RunsXNumTimePoints = 4800; - % PFMFolder = '/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll3T1071/MNINonLinear/Results/rfMRI_REST/PFMs_chpc'; - - wbcommand = 'wb_command'; - - %% parse arguments - subjList = regexp(SubjlistRaw,'@','split'); - nS = numel(subjList); - RunsXNumTimePoints = str2double(RunsXNumTimePoints); - PFMdim = str2double(PFMdim); - if nargin < 8;error('All arguments are required.');end % failsafe - - %% preallocate - [TCSMask, TCSAll] = deal(zeros(PFMdim,RunsXNumTimePoints, nS,'single')); - [spectra,PFMmaps, PFMvolMaps] = deal([]); - %PFMMapsAll = zeros(CIFTIVertices,PFMdim, nS,'single'); - %PFMVolMapsAll = zeros(CIFTIVol,PFMdim, nS,'single'); - - %% loop over subjects - for iS = 1:nS +% GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) +% This function aggregates individual subject PFM results and computes +% group-level time course masks, spectra, maps, and statistics. +% +% Inputs: +% StudyFolder - Path to the study directory +% SubjlistRaw - Subject list as @ separated string +% PFMdim - PFM dimensionality +% OutputPrefix - Prefix for output files +% RegName - Registration string (e.g., '_MSMAll') +% LowResMesh - Mesh resolution (e.g., '32' for 32k_fs_LR) +% RunsXNumTimePoints - Total expected timepoints across runs +% PFMFolder - Output folder for group-level results +% +% Example: +% StudyFolder = '/media/myelin/brainmappers/Data/YA_HCP'; +% SubjlistRaw = "100206@100307@100408"; +% PFMdim = 92; +% OutputPrefix = 'rfMRI_REST_d92_MSMAll_PFMs_tclean'; +% RegName = '_MSMAll'; +% LowResMesh = '32'; +% RunsXNumTimePoints = 4800; + +%% Initialize parameters +wbcommand = 'wb_command'; + +%% Parse input arguments +subjList = regexp(SubjlistRaw,'@','split'); +nS = numel(subjList); +RunsXNumTimePoints = str2double(RunsXNumTimePoints); +PFMdim = str2double(PFMdim); +if nargin < 8; error('All arguments are required.'); end % Validate all inputs + +%% Preallocate arrays for group concatenation +% TCSMask: binary mask indicating valid timepoints per subject +% TCSAll: concatenated time courses across subjects +% spectra, PFMmaps, PFMvolMaps: accumulators for group averages +[TCSMask, TCSAll] = deal(zeros(PFMdim, RunsXNumTimePoints, nS, 'single')); +[spectra, PFMmaps, PFMvolMaps] = deal([]); + +%% Load and accumulate individual subject results +for iS = 1:nS subj = subjList{iS}; subjDir = [StudyFolder '/' subj '/MNINonLinear/fsaverage_LR' LowResMesh 'k']; - fprintf('processing %s ... \n', subj); + fprintf('Processing %s ... \n', subj); - % load subject data - PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'],wbcommand); - PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'],wbcommand); - TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'],wbcommand); - SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'],wbcommand); - - - % clean up NaNs and Infs in PFM VolMaps + %% Load individual PFM results from standard subject locations + % Load spatial maps, volume maps, time courses, and power spectra + PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + + %% Clean up NaNs and Infs in volume maps + % Replace invalid values with zeros to prevent propagation to group statistics infMask = isinf(PFMVolMapsSub.cdata); nanMask = isnan(PFMVolMapsSub.cdata); if any(infMask, 'all') - warning('Found Infs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); - PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = 0; + warning('Found Infs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = 0; end if any(nanMask, 'all') - warning('Found NaNs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); - PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; + warning('Found NaNs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; end - % store subject TCS in TCSALL array - TCSAll(1:size(TCSSub.cdata,1),1:size(TCSSub.cdata,2),iS) = TCSSub.cdata; - - % Perform running sum of spectra and PFM maps - if size(TCSSub.cdata,2) == RunsXNumTimePoints - TCSMask(:,:,iS) = repmat(1,PFMdim,RunsXNumTimePoints,1); - if isempty(spectra) - spectra = SpectraSub; - spectra.cdata = SpectraSub.cdata*0; - end - spectra.cdata = spectra.cdata + SpectraSub.cdata; + %% Store subject time course in concatenated array + TCSAll(1:size(TCSSub.cdata, 1), 1:size(TCSSub.cdata, 2), iS) = TCSSub.cdata; + + %% Accumulate spectra and maps for group averaging + % Only accumulate if subject has expected number of timepoints + if size(TCSSub.cdata, 2) == RunsXNumTimePoints + TCSMask(:, :, iS) = repmat(1, PFMdim, RunsXNumTimePoints, 1); + if isempty(spectra) + spectra = SpectraSub; + spectra.cdata = SpectraSub.cdata * 0; + end + spectra.cdata = spectra.cdata + SpectraSub.cdata; end + + % Accumulate spatial maps from all subjects with valid data if isempty(PFMmaps) - PFMmaps = PFMMapsSub; - PFMmaps.cdata = PFMMapsSub.cdata*0; - PFMvolMaps = PFMVolMapsSub; - PFMvolMaps.cdata = PFMVolMapsSub.cdata*0; + PFMmaps = PFMMapsSub; + PFMmaps.cdata = PFMMapsSub.cdata * 0; + PFMvolMaps = PFMVolMapsSub; + PFMvolMaps.cdata = PFMVolMapsSub.cdata * 0; end PFMmaps.cdata = PFMmaps.cdata + PFMMapsSub.cdata; PFMvolMaps.cdata = PFMvolMaps.cdata + PFMVolMapsSub.cdata; - end % for iS = 1:nS - - %% package and save outputs - if ~exist(PFMFolder,'dir');mkdir(PFMFolder);end - dimStr = num2str(PFMdim); - % timeseries - TCSMaskConcat = TCSSub; - TCSMaskConcat.cdata = squeeze(reshape(TCSMask,PFMdim,RunsXNumTimePoints* nS)); - TCSFullConcat = TCSSub; - TCSFullConcat.cdata = squeeze(reshape(TCSAll,PFMdim,RunsXNumTimePoints* nS)); - ciftisavereset(TCSMaskConcat,[PFMFolder '/PFM_TCSMASK_' dimStr '.sdseries.nii'],wbcommand); - ciftisavereset(TCSFullConcat,[PFMFolder '/PFM_TCS_' dimStr '.sdseries.nii'],wbcommand); - - TCSAVG = TCSSub; - TCSAVG.cdata = sum(TCSAll.*TCSMask,3)/nS; - TCSABSAVG = TCSSub; - TCSABSAVG.cdata = sum(abs(TCSAll.*TCSMask),3)/nS; - ciftisavereset(TCSAVG,[PFMFolder '/PFM_AVGTCS_' dimStr '.sdseries.nii'],wbcommand); - ciftisavereset(TCSABSAVG,[PFMFolder '/PFM_ABSAVGTCS_' dimStr '.sdseries.nii'],wbcommand); - - - % PFM stats - PFMTSTDs = std(TCSFullConcat.cdata,[],2); - PFMPercentVariances = (((PFMTSTDs.^2)/sum(PFMTSTDs.^2))*100); - dlmwrite([PFMFolder '/PFM_stats_' dimStr '.wb_annsub.csv'],[(1:PFMdim)' round(PFMPercentVariances,2)],','); - - % spectra - spectra.cdata = spectra.cdata/nS; - ciftisavereset(spectra,[PFMFolder '/PFM_Spectra_' dimStr '.sdseries.nii'],wbcommand); - - - % PFM maps - PFMmaps.cdata = PFMmaps.cdata/nS; - PFMvolMaps.cdata = PFMvolMaps.cdata/nS; - - %TRIM = repmat(squeeze(trimmean(PFMMapsAll,10,3)),1,1, nS); - %MAD = repmat(squeeze(mad(PFMMapsAll,1,3)*1.4826),1,1, nS); - %MASK = (PFMMapsAll>TRIM-MAD*2).*(PFMMapsAllTRIM-MAD*2).*(PFMVolMapsAll Date: Thu, 5 Feb 2026 12:48:34 -0600 Subject: [PATCH 15/74] remove unnecessary extra script copies --- PFM/scripts/RunPROFUMO.m | 96 ------------------------------ PFM/scripts/RunPROFUMO.sh | 122 -------------------------------------- 2 files changed, 218 deletions(-) delete mode 100755 PFM/scripts/RunPROFUMO.m delete mode 100755 PFM/scripts/RunPROFUMO.sh diff --git a/PFM/scripts/RunPROFUMO.m b/PFM/scripts/RunPROFUMO.m deleted file mode 100755 index 608c9f0a0..000000000 --- a/PFM/scripts/RunPROFUMO.m +++ /dev/null @@ -1,96 +0,0 @@ -function RunPROFUMO(StudyFolder, SubjListRaw, GroupAverageName, OutputfMRIName, PFMdimStr, OutputPrefix, RegString, LowResMesh, RunsXNumTimePointsStr, CIFTIVerticesStr, CIFTIVolumeStr, PFMFolder) - - % Parse string inputs - Subjlist = strsplit(SubjListRaw, '@'); - PFMdim = str2double(PFMdimStr); - RunsXNumTimePoints = str2double(RunsXNumTimePointsStr); - wbcommand = 'wb_command'; - - c = 1; - SubjFolderlist = {}; - StudyFolderNumber = []; - for i = 1:numel(Subjlist) - SubjFolderlist{c} = [StudyFolder '/' Subjlist{i}]; - StudyFolderNumber = [StudyFolderNumber 1]; - c = c + 1; - end - - for i = 1:numel(Subjlist) - if ~isfile([StudyFolder '/' Subjlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii']) - Subjlist{i} - end - end - - s1 = 0; - m1 = 0; - TCSMask = zeros(PFMdim, RunsXNumTimePoints, numel(SubjFolderlist), 'single'); - TCSAll = zeros(PFMdim, RunsXNumTimePoints, numel(SubjFolderlist), 'single'); - SpectraOne = []; - PFMMapsOne = []; - PFMVolMapsOne = []; - - for i = 1:numel(SubjFolderlist) - Subjlist{i} - if exist([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii']) - PFMMapsSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - PFMVolMapsSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix '_DR' RegString '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - TCSSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - SpectraSub = ciftiopen([SubjFolderlist{i} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{i} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - TCSAll(1:size(TCSSub.cdata,1), 1:size(TCSSub.cdata,2), i) = TCSSub.cdata; - - PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = NaN; - PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; - - RANK(i) = rank(TCSSub.cdata); - COND(i) = cond(TCSSub.cdata); - - if StudyFolderNumber(i) == 1 - if numel(TCSSub.cdata) == RunsXNumTimePoints - TCSMask(:, :, i) = repmat(1, PFMdim, RunsXNumTimePoints, 1); - if isempty(SpectraOne) - SpectraOne = SpectraSub; - SpectraOne.cdata = SpectraSub.cdata * 0; - end - SpectraOne.cdata = SpectraOne.cdata + SpectraSub.cdata; - s1 = s1 + 1; - end - if isempty(PFMMapsOne) - PFMMapsOne = PFMMapsSub; - PFMMapsOne.cdata = PFMMapsSub.cdata * 0; - PFMVolMapsOne = PFMVolMapsSub; - PFMVolMapsOne.cdata = PFMVolMapsSub.cdata * 0; - end - PFMMapsOne.cdata = PFMMapsOne.cdata + PFMMapsSub.cdata; - PFMVolMapsOne.cdata = PFMVolMapsOne.cdata + PFMVolMapsSub.cdata; - m1 = m1 + 1; - end - end - end - - TCSMaskConcat = TCSSub; - TCSMaskConcat.cdata = squeeze(reshape(TCSMask, PFMdim, RunsXNumTimePoints * numel(SubjFolderlist))); - TCSFullConcat = TCSSub; - TCSFullConcat.cdata = squeeze(reshape(TCSAll, PFMdim, RunsXNumTimePoints * numel(SubjFolderlist))); - ciftisavereset(TCSMaskConcat, [PFMFolder '/PFM_TCSMASK_' num2str(PFMdim) '.sdseries.nii'], wbcommand); - ciftisavereset(TCSFullConcat, [PFMFolder '/PFM_TCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); - - PFMTSTDs = std(TCSFullConcat.cdata, [], 2); - PFMPercentVariances = (((PFMTSTDs .^ 2) / sum(PFMTSTDs .^ 2)) * 100); - dlmwrite([PFMFolder '/PFM_stats_' num2str(PFMdim) '.wb_annsub.csv'], [round(PFMPercentVariances, 2)], ','); - - TCSAVGOne = TCSSub; - TCSAVGOne.cdata = sum(TCSAll .* single(TCSMask == 1), 3) / s1; - TCSABSAVGOne = TCSSub; - TCSABSAVGOne.cdata = sum(abs(TCSAll .* single(TCSMask == 1)), 3) / s1; - - SpectraOne.cdata = SpectraOne.cdata / s1; - PFMMapsOne.cdata = PFMMapsOne.cdata / m1; - PFMVolMapsOne.cdata = PFMVolMapsOne.cdata / m1; - - ciftisavereset(TCSAVGOne, [PFMFolder '/PFM_AVGTCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); - ciftisavereset(TCSABSAVGOne, [PFMFolder '/PFM_ABSAVGTCS_' num2str(PFMdim) '.sdseries.nii'], wbcommand); - - ciftisavereset(SpectraOne, [PFMFolder '/PFM_Spectra_' num2str(PFMdim) '.sdseries.nii'], wbcommand); - ciftisavereset(PFMMapsOne, [PFMFolder '/PFM_Maps_' num2str(PFMdim) '.dscalar.nii'], wbcommand); - ciftisavereset(PFMVolMapsOne, [PFMFolder '/PFM_VolMaps_' num2str(PFMdim) '.dscalar.nii'], wbcommand); -end \ No newline at end of file diff --git a/PFM/scripts/RunPROFUMO.sh b/PFM/scripts/RunPROFUMO.sh deleted file mode 100755 index 15e087f13..000000000 --- a/PFM/scripts/RunPROFUMO.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -set -eu - -pipedirguessed=0 -if [[ "${HCPPIPEDIR:-}" == "" ]] -then - pipedirguessed=1 - export HCPPIPEDIR="$(dirname -- "$0")/../.." -fi - -source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" -source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" -source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" -g_matlab_default_mode=1 - -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: Generate group-level PFM notes and statistics - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} - -#arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] -opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" -opts_AddMandatory '--subject-list' 'SubjListRaw' '100206@100307...' 'list of subject IDs separated by @s' -opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' -opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" -opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality" -opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" -opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string" -opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" -opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' 'integer' "total timepoints across runs" -opts_AddMandatory '--cifti-vertices' 'CIFTIVertices' 'integer' "number of CIFTI vertices" -opts_AddMandatory '--cifti-volume' 'CIFTIVolume' 'integer' "number of CIFTI volume vertices" -opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" - - -opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode -0 = compiled MATLAB -1 = interpreted MATLAB -2 = Octave" "$g_matlab_default_mode" - -opts_ParseArguments "$@" - -if ((pipedirguessed)) -then - log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" -fi - -#display the parsed/default values -opts_ShowValues - -RegString="" -if [[ "$RegName" != "" ]] -then - RegString="_$RegName" -fi - -case "$MatlabMode" in - (0) - if [[ "${MATLAB_COMPILER_RUNTIME:-}" == "" ]] - then - log_Err_Abort "to use compiled matlab, you must set and export the variable MATLAB_COMPILER_RUNTIME" - fi - ;; - (1) - matlab_interpreter=(matlab -nodisplay -nosplash) - ;; - (2) - matlab_interpreter=(octave-cli -q --no-window-system) - ;; - (*) - log_Err_Abort "unrecognized matlab mode '$MatlabMode', use 0, 1, or 2" - ;; -esac - -#shortcut in case the folder gets renamed -this_script_dir=$(dirname "$0") - -#matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$GroupAverageName" "$OutputfMRIName" "$PFMdim" "$OutputPrefix" "$RegString" "$LowResMesh" "$RunsXNumTimePoints" "$CIFTIVertices" "$CIFTIVolume" "$PFMFolder") - -case "$MatlabMode" in - (0) - matlab_cmd=("$this_script_dir/Compiled_RunPROFUMO/run_RunPROFUMO.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") - log_Msg "running compiled matlab command: ${matlab_cmd[*]}" - "${matlab_cmd[@]}" - ;; - (1 | 2) - #reformat argument array so matlab sees them as strings - matlab_args="" - for thisarg in "${matlab_argarray[@]}" - do - if [[ "$matlab_args" != "" ]] - then - matlab_args+=", " - fi - matlab_args+="'$thisarg'" - done - matlabcode=" - addpath('$HCPPIPEDIR/global/matlab'); - addpath('$this_script_dir'); - addpath('$HCPCIFTIRWDIR'); - RunPROFUMO($matlab_args);" - - log_Msg "running matlab code: $matlabcode" - "${matlab_interpreter[@]}" <<<"$matlabcode" - echo - ;; -esac - -log_Msg "RunPROFUMO step completed successfully" \ No newline at end of file From 4375d3598a01e79118c9c2143c3de17fd5e7c19d Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 13:00:52 -0600 Subject: [PATCH 16/74] improve commenting --- PFM/scripts/GroupPFMs.m | 97 +++++++++++------------- PFM/scripts/PostPROFUMO.m | 150 +++++++++++++++++++------------------- 2 files changed, 119 insertions(+), 128 deletions(-) diff --git a/PFM/scripts/GroupPFMs.m b/PFM/scripts/GroupPFMs.m index f68b37756..f056cf029 100755 --- a/PFM/scripts/GroupPFMs.m +++ b/PFM/scripts/GroupPFMs.m @@ -12,15 +12,6 @@ function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowR % LowResMesh - Mesh resolution (e.g., '32' for 32k_fs_LR) % RunsXNumTimePoints - Total expected timepoints across runs % PFMFolder - Output folder for group-level results -% -% Example: -% StudyFolder = '/media/myelin/brainmappers/Data/YA_HCP'; -% SubjlistRaw = "100206@100307@100408"; -% PFMdim = 92; -% OutputPrefix = 'rfMRI_REST_d92_MSMAll_PFMs_tclean'; -% RegName = '_MSMAll'; -% LowResMesh = '32'; -% RunsXNumTimePoints = 4800; %% Initialize parameters wbcommand = 'wb_command'; @@ -41,54 +32,54 @@ function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowR %% Load and accumulate individual subject results for iS = 1:nS - subj = subjList{iS}; - subjDir = [StudyFolder '/' subj '/MNINonLinear/fsaverage_LR' LowResMesh 'k']; - fprintf('Processing %s ... \n', subj); - - %% Load individual PFM results from standard subject locations - % Load spatial maps, volume maps, time courses, and power spectra - PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + subj = subjList{iS}; + subjDir = [StudyFolder '/' subj '/MNINonLinear/fsaverage_LR' LowResMesh 'k']; + fprintf('Processing %s ... \n', subj); + + %% Load individual PFM results from standard subject locations + % Load spatial maps, volume maps, time courses, and power spectra + PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - %% Clean up NaNs and Infs in volume maps - % Replace invalid values with zeros to prevent propagation to group statistics - infMask = isinf(PFMVolMapsSub.cdata); - nanMask = isnan(PFMVolMapsSub.cdata); - if any(infMask, 'all') - warning('Found Infs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); - PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = 0; - end - if any(nanMask, 'all') - warning('Found NaNs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); - PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; - end + %% Clean up NaNs and Infs in volume maps + % Replace invalid values with zeros to prevent propagation to group statistics + infMask = isinf(PFMVolMapsSub.cdata); + nanMask = isnan(PFMVolMapsSub.cdata); + if any(infMask, 'all') + warning('Found Infs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isinf(PFMVolMapsSub.cdata)) = 0; + end + if any(nanMask, 'all') + warning('Found NaNs in PFM VolMaps for subject %s. Replacing with zeros.\n', subj); + PFMVolMapsSub.cdata(isnan(PFMVolMapsSub.cdata)) = 0; + end - %% Store subject time course in concatenated array - TCSAll(1:size(TCSSub.cdata, 1), 1:size(TCSSub.cdata, 2), iS) = TCSSub.cdata; + %% Store subject time course in concatenated array + TCSAll(1:size(TCSSub.cdata, 1), 1:size(TCSSub.cdata, 2), iS) = TCSSub.cdata; - %% Accumulate spectra and maps for group averaging - % Only accumulate if subject has expected number of timepoints - if size(TCSSub.cdata, 2) == RunsXNumTimePoints - TCSMask(:, :, iS) = repmat(1, PFMdim, RunsXNumTimePoints, 1); - if isempty(spectra) - spectra = SpectraSub; - spectra.cdata = SpectraSub.cdata * 0; - end - spectra.cdata = spectra.cdata + SpectraSub.cdata; - end - - % Accumulate spatial maps from all subjects with valid data - if isempty(PFMmaps) - PFMmaps = PFMMapsSub; - PFMmaps.cdata = PFMMapsSub.cdata * 0; - PFMvolMaps = PFMVolMapsSub; - PFMvolMaps.cdata = PFMVolMapsSub.cdata * 0; + %% Accumulate spectra and maps for group averaging + % Only accumulate if subject has expected number of timepoints + if size(TCSSub.cdata, 2) == RunsXNumTimePoints + TCSMask(:, :, iS) = repmat(1, PFMdim, RunsXNumTimePoints, 1); + if isempty(spectra) + spectra = SpectraSub; + spectra.cdata = SpectraSub.cdata * 0; end - PFMmaps.cdata = PFMmaps.cdata + PFMMapsSub.cdata; - PFMvolMaps.cdata = PFMvolMaps.cdata + PFMVolMapsSub.cdata; -end % for iS = 1:nS + spectra.cdata = spectra.cdata + SpectraSub.cdata; + end + + % Accumulate spatial maps from all subjects with valid data + if isempty(PFMmaps) + PFMmaps = PFMMapsSub; + PFMmaps.cdata = PFMMapsSub.cdata * 0; + PFMvolMaps = PFMVolMapsSub; + PFMvolMaps.cdata = PFMVolMapsSub.cdata * 0; + end + PFMmaps.cdata = PFMmaps.cdata + PFMMapsSub.cdata; + PFMvolMaps.cdata = PFMvolMaps.cdata + PFMVolMapsSub.cdata; +end % for iS = 1:nS %% Create output directory if needed if ~exist(PFMFolder, 'dir'); mkdir(PFMFolder); end diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 481b93b48..0df613409 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -26,85 +26,85 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Main loop: Process each subject for s = 1:numel(Subjlist) - s % Display subject index - - %% Identify available fMRI runs for this subject - % Determine which fMRI runs exist for this subject - % If ConcatName is specified, use concatenated version; otherwise check individual runs - subfMRINames = {}; - if ~strcmp(ConcatName, '') - % Multi-run data: check if concatenated dataset exists - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii']) - c = 1; - for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) - subfMRINames{c} = fMRINames{r}; - c = c + 1; - end - end + s % Display subject index + + %% Identify available fMRI runs for this subject + % Determine which fMRI runs exist for this subject + % If ConcatName is specified, use concatenated version; otherwise check individual runs + subfMRINames = {}; + if ~strcmp(ConcatName, '') + % Multi-run data: check if concatenated dataset exists + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii']) + c = 1; + for r = 1:numel(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + subfMRINames{c} = fMRINames{r}; + c = c + 1; end - else - % Single-run data: check which runs exist - c = 1; - for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) - subfMRINames{c} = fMRINames{r}; - c = c + 1; - end - end + end % for r = 1:numel(fMRINames) end - - %% Process subject if valid runs found - if numel(subfMRINames) ~= 0 - %% Load and concatenate PFM time courses and amplitudes - % Load PROFUMO outputs and amplitude-modulate time courses - origTCS = []; % Original unmodulated time courses - TCS = []; % Amplitude-modulated time courses - for r = 1:numel(subfMRINames) - runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + else + % Single-run data: check which runs exist + c = 1; + for r = 1:numel(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + subfMRINames{c} = fMRINames{r}; + c = c + 1; + end + end % for r = 1:numel(fMRINames) + end + + %% Process subject if valid runs found + if numel(subfMRINames) ~= 0 + %% Load and concatenate PFM time courses and amplitudes + % Load PROFUMO outputs and amplitude-modulate time courses + origTCS = []; % Original unmodulated time courses + TCS = []; % Amplitude-modulated time courses + for r = 1:numel(subfMRINames) + runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', numel(runTCS), 1)]; - end - - %% Create original time course and spectral CIFTI files - % Generate CIFTI structure for unmodulated time courses - PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); - % PFMTCSorig.diminfo{1,2} = sICATCS.diminfo{1,2}; - - % Compute power spectra for original time courses - ts.Nnodes = size(origTCS, 2); - ts.Nsubjects = 1; - ts.ts = origTCS; - ts.NtimepointsPerSubject = size(origTCS, 1); - PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - % PFMSpectraorig.diminfo{1,2} = sICASpectra.diminfo{1,2}; - - %% Create amplitude-modulated time course and spectral CIFTI files - % Generate CIFTI structure for amplitude-modulated time courses - PFMTCS = cifti_struct_create_sdseries(TCS'); - % PFMTCS.diminfo{1,2} = sICATCS.diminfo{1,2}; - - % Compute power spectra for amplitude-modulated time courses - ts.Nnodes = size(TCS, 2); - ts.Nsubjects = 1; - ts.ts = TCS; - ts.NtimepointsPerSubject = size(TCS, 1); - PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - % PFMSpectra.diminfo{1,2} = sICASpectra.diminfo{1,2}; + origTCS = [origTCS ; runTCS]; + TCS = [TCS ; runTCS .* repmat(runAmp', numel(runTCS), 1)]; + end % for r = 1:numel(subfMRINames) + + %% Create original time course and spectral CIFTI files + % Generate CIFTI structure for unmodulated time courses + PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); + % PFMTCSorig.diminfo{1,2} = sICATCS.diminfo{1,2}; + + % Compute power spectra for original time courses + ts.Nnodes = size(origTCS, 2); + ts.Nsubjects = 1; + ts.ts = origTCS; + ts.NtimepointsPerSubject = size(origTCS, 1); + PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + % PFMSpectraorig.diminfo{1,2} = sICASpectra.diminfo{1,2}; + + %% Create amplitude-modulated time course and spectral CIFTI files + % Generate CIFTI structure for amplitude-modulated time courses + PFMTCS = cifti_struct_create_sdseries(TCS'); + % PFMTCS.diminfo{1,2} = sICATCS.diminfo{1,2}; + + % Compute power spectra for amplitude-modulated time courses + ts.Nnodes = size(TCS, 2); + ts.Nsubjects = 1; + ts.ts = TCS; + ts.NtimepointsPerSubject = size(TCS, 1); + PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + % PFMSpectra.diminfo{1,2} = sICASpectra.diminfo{1,2}; - %% Save individual-level results - % Save original and amplitude-modulated time courses and spectra - ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + %% Save individual-level results + % Save original and amplitude-modulated time courses and spectra + ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - %% Copy individual PFM maps - % Link PROFUMO spatial maps to subject's fsaverage space directory - copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); - end -end + %% Copy individual PFM maps + % Link PROFUMO spatial maps to subject's fsaverage space directory + copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); + end % if numel(subfMRINames) ~= 0 +end % for s = 1:numel(Subjlist) end \ No newline at end of file From e3d19869e5a0e0ab253d47f8af40c45d36c879ba Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 13:06:06 -0600 Subject: [PATCH 17/74] improve comments --- PFM/scripts/PostPROFUMO.m | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 0df613409..8a78b291c 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -71,28 +71,24 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Create original time course and spectral CIFTI files % Generate CIFTI structure for unmodulated time courses PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); - % PFMTCSorig.diminfo{1,2} = sICATCS.diminfo{1,2}; - % Compute power spectra for original time courses + % Store power spectra ts.Nnodes = size(origTCS, 2); ts.Nsubjects = 1; ts.ts = origTCS; ts.NtimepointsPerSubject = size(origTCS, 1); PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - % PFMSpectraorig.diminfo{1,2} = sICASpectra.diminfo{1,2}; - %% Create amplitude-modulated time course and spectral CIFTI files - % Generate CIFTI structure for amplitude-modulated time courses + %% Create time course and spectral CIFTI files + % Generate CIFTI structure for time courses PFMTCS = cifti_struct_create_sdseries(TCS'); - % PFMTCS.diminfo{1,2} = sICATCS.diminfo{1,2}; - % Compute power spectra for amplitude-modulated time courses + % Store power spectra ts.Nnodes = size(TCS, 2); ts.Nsubjects = 1; ts.ts = TCS; ts.NtimepointsPerSubject = size(TCS, 1); PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - % PFMSpectra.diminfo{1,2} = sICASpectra.diminfo{1,2}; %% Save individual-level results % Save original and amplitude-modulated time courses and spectra From 47e2c867836341749663b5c5c1269cb4b41f15e7 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 13:40:45 -0600 Subject: [PATCH 18/74] fix step name --- PFM/PFMPipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 459d37e17..d77b5fbd2 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -200,7 +200,7 @@ do "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" ;; - (ImportPFMNotes) + (PostPROFUMO) log_Msg "Running PostPROFUMO step" "$HCPPIPEDIR"/PFM/scripts/PostPROFUMO.sh \ --study-folder="$StudyFolder" \ From 1c17127317b0bf16cc4c2665ffb0fb50e1f5d36d Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 13:56:01 -0600 Subject: [PATCH 19/74] minor fixes --- PFM/PFMPipeline.sh | 2 +- PFM/scripts/PostPROFUMO.m | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index d77b5fbd2..e46f0cf20 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -257,7 +257,7 @@ do --surf-reg-name="$RegName" --low-res="$LowResMesh" --proc-string="_$fMRIProcSTRING" - --method="$RSNMethod" + --method="dual" --low-ica-dims="$LowDims" --low-ica-template-name="$LowDimTemplate" --output-string="$OutputPrefix" diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 8a78b291c..a88866874 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -26,7 +26,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Main loop: Process each subject for s = 1:numel(Subjlist) - s % Display subject index + fprintf('Processing subject %d/%d: %s\n', s, numel(Subjlist), Subjlist{s}); %% Identify available fMRI runs for this subject % Determine which fMRI runs exist for this subject @@ -34,10 +34,10 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc subfMRINames = {}; if ~strcmp(ConcatName, '') % Multi-run data: check if concatenated dataset exists - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii']) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii'],'file') c = 1; for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') subfMRINames{c} = fMRINames{r}; c = c + 1; end @@ -47,7 +47,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % Single-run data: check which runs exist c = 1; for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii']) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') subfMRINames{c} = fMRINames{r}; c = c + 1; end From 586b2e77d2750ec77555613e55faa0a4d54b0ffe Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 14:28:12 -0600 Subject: [PATCH 20/74] move output string definition to batch script --- Examples/Scripts/RunPFMpipelineFullRun.sh | 17 ++++++++++------- PFM/PFMPipeline.sh | 8 ++------ PFM/scripts/PostPROFUMO.m | 5 ++--- PFM/scripts/PostPROFUMO.sh | 3 +-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 4d96e47ba..fe63c625f 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -19,12 +19,12 @@ get_options() { # initialize variables StudyFolder="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" Subjlist="$(find ${StudyFolder} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" - EnvironmentScript="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5/Mac25Rhesus_v5_SetUpHCPPipeline.sh" + EnvironmentScript="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5_SetUpHCPPipeline.sh" GroupAverageName="Mac25Rhesus_v5" RegName="" MatlabMode=1 RunLocal=0 - QUEUE="matlabparallelhigh.q" + QUEUE="matlabparallelhigh.q@brainmappers-desktop3" # parse arguments local index argument @@ -143,7 +143,7 @@ main() { # StartStep="RSNRegression" # StopStep="RSNRegression" # StartStep="RunPROFUMO" - # StopStep="RSNRegression" + # StopStep="" StartStep="RunPROFUMO" StopStep="GroupPFMs" @@ -155,7 +155,7 @@ main() { # general inputs fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" - randSeed=2 # random seed for PROFUMO + randSeed=1 # random seed for PROFUMO OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs @@ -168,7 +168,7 @@ main() { # PFM settings for REST data PFMdim="16" # set the PFM dimensionality - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim}_s${randSeed}_M1k + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim}_s${randSeed}_test # Reference image for PROFUMO RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" @@ -178,6 +178,9 @@ main() { # set the mesh resolution, like '32' for 32k_fs_LR LowResMesh="10" + # Define OutputPrefix with seed designation + OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs_tclean" + # RSN regression settings LowDims="6" FixLegacyBiasString="NO" @@ -193,7 +196,7 @@ main() { ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" - nStarts="1000" # number of multi-start iterations for PROFUMO + nStarts="1" # number of multi-start iterations for PROFUMO RandomSeed="$randSeed" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below @@ -224,6 +227,7 @@ main() { --subject-list="$Subjlist" \ --fmri-names="$fMRINames" \ --output-fmri-name="$OutputfMRIName" \ + --output-prefix="$OutputPrefix" \ --proc-string="$fMRIProcSTRING" \ --group-average-name="$GroupAverageName" \ --pfm-dimension="$PFMdim" \ @@ -232,7 +236,6 @@ main() { --concat-name="$ConcatName" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ - --rsn-method=dual \ --low-dims="$LowDims" \ --fix-legacy-bias="$FixLegacyBiasString" \ --scale-factor="$ScaleFactor" \ diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index e46f0cf20..a34d144cf 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -30,6 +30,7 @@ opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains al opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of subject IDs separated by @s" opts_AddMandatory '--fmri-names' 'fMRINames' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' "list of fmri run names separated by @s" opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" +opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for individual subject files (typically includes dimension, group name, and seed)" opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing already done, e.g. '_Atlas_MSMAll_hp2000_clean_rclean_tclean'" opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" @@ -118,10 +119,6 @@ then RegString="_$RegName" fi -# Auto-generate output strings (include random seed for uniqueness) -OutputSTRING="_d${PFMdim}_${GroupAverageName}_seed${RandomSeed}_WR" -OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${RandomSeed}_PFMs_tclean" - # Volume template file path # VolumeTemplateFile="${StudyFolder}/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.2.dscalar.nii" @@ -209,7 +206,6 @@ do --concat-name="$ConcatName" \ --proc-string="$fMRIProcSTRING" \ --output-fmri-name="$OutputfMRIName" \ - --output-string="$OutputSTRING" \ --output-prefix="$OutputPrefix" \ --surf-reg-name="$RegName" \ --low-res-mesh="$LowResMesh" \ @@ -260,7 +256,7 @@ do --method="dual" --low-ica-dims="$LowDims" --low-ica-template-name="$LowDimTemplate" - --output-string="$OutputPrefix" + --output-prefix="$OutputPrefix" --output-spectra="$RunsXNumTimePoints" --volume-template-cifti="$VolumeTemplateFile" --output-z=1 diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index a88866874..0f1fdc63d 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,5 +1,5 @@ -function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) -% PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) +% PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) % This function imports PROFUMO results and generates CIFTI-format time courses % and power spectra for each subject. The outputs are used for subsequent % group-level PFM analysis. @@ -11,7 +11,6 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % ConcatName - Name of concatenated fMRI dataset (empty if single runs) % fMRIProcSTRING - Processing string component (e.g., '_Atlas_hp200_clean') % OutputfMRIName - Name of output fMRI dataset -% OutputSTRING - Output naming string % OutputPrefix - Prefix for output files % RegString - Registration string % LowResMesh - Mesh resolution (e.g., '10' for 10k) diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index 66e7aab55..f87e6d45c 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -36,7 +36,6 @@ opts_AddMandatory '--subject-list' 'SubjListRaw' '100206@100307...' 'list of sub opts_AddMandatory '--fmri-names' 'fMRIListRaw' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' 'list of fmri run names separated by @s' opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing" opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" -opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string identifier" opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string" opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" @@ -90,7 +89,7 @@ IFS='@' read -a fMRIList <<<"$fMRIListRaw" this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$OutputPrefix" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputPrefix" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") case "$MatlabMode" in (0) From f0573be0a21a8ba50cc27f7fe846dfe1da608faf Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 5 Feb 2026 15:22:14 -0600 Subject: [PATCH 21/74] Consolidate output-string and output-prefix --- Examples/Scripts/RunPFMpipelineFullRun.sh | 14 +++++++------- PFM/PFMPipeline.sh | 8 ++++---- PFM/scripts/GroupPFMs.m | 14 +++++++------- PFM/scripts/GroupPFMs.sh | 4 ++-- PFM/scripts/PostPROFUMO.m | 16 ++++++++-------- PFM/scripts/PostPROFUMO.sh | 4 ++-- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index fe63c625f..cd4f4c424 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -145,11 +145,11 @@ main() { # StartStep="RunPROFUMO" # StopStep="" - StartStep="RunPROFUMO" + StartStep="PostPROFUMO" StopStep="GroupPFMs" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' - parLimit=-1 + parLimit=5 # general inputs @@ -157,7 +157,7 @@ main() { # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" randSeed=1 # random seed for PROFUMO - OutputfMRIName="Mac25Rhesus_v5_BOLD_REST_CONCAT_PFM" + OutputfMRIName="BOLD_REST_CONCAT" # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs ConcatName="BOLD_REST_CONCAT" # set the output spectra size for individual projection, RunsXNumTimePoints @@ -168,7 +168,7 @@ main() { # PFM settings for REST data PFMdim="16" # set the PFM dimensionality - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_d${PFMdim}_s${randSeed}_test + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${GroupAverageName}_${OutputfMRIName}_PFM_d${PFMdim}_s${randSeed}_test # Reference image for PROFUMO RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" @@ -178,8 +178,8 @@ main() { # set the mesh resolution, like '32' for 32k_fs_LR LowResMesh="10" - # Define OutputPrefix with seed designation - OutputPrefix="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs_tclean" + # Define OutputSTRING with seed designation + OutputSTRING="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs" # RSN regression settings LowDims="6" @@ -227,7 +227,7 @@ main() { --subject-list="$Subjlist" \ --fmri-names="$fMRINames" \ --output-fmri-name="$OutputfMRIName" \ - --output-prefix="$OutputPrefix" \ + --output-string="$OutputSTRING" \ --proc-string="$fMRIProcSTRING" \ --group-average-name="$GroupAverageName" \ --pfm-dimension="$PFMdim" \ diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index a34d144cf..b9f3059df 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -30,7 +30,7 @@ opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains al opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of subject IDs separated by @s" opts_AddMandatory '--fmri-names' 'fMRINames' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' "list of fmri run names separated by @s" opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" -opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for individual subject files (typically includes dimension, group name, and seed)" +opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string for individual subject files (typically includes dimension, group name, and seed)" opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing already done, e.g. '_Atlas_MSMAll_hp2000_clean_rclean_tclean'" opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" @@ -206,7 +206,7 @@ do --concat-name="$ConcatName" \ --proc-string="$fMRIProcSTRING" \ --output-fmri-name="$OutputfMRIName" \ - --output-prefix="$OutputPrefix" \ + --output-string="$OutputSTRING" \ --surf-reg-name="$RegName" \ --low-res-mesh="$LowResMesh" \ --profumo-tr="$TR" \ @@ -256,7 +256,7 @@ do --method="dual" --low-ica-dims="$LowDims" --low-ica-template-name="$LowDimTemplate" - --output-prefix="$OutputPrefix" + --output-string="$OutputSTRING" --output-spectra="$RunsXNumTimePoints" --volume-template-cifti="$VolumeTemplateFile" --output-z=1 @@ -278,7 +278,7 @@ do --study-folder="$StudyFolder" \ --subject-list="$SubjlistRaw" \ --pfm-dimension="$PFMdim" \ - --output-prefix="$OutputPrefix" \ + --output-string="$OutputSTRING" \ --surf-reg-name="$RegName" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$RunsXNumTimePoints" \ diff --git a/PFM/scripts/GroupPFMs.m b/PFM/scripts/GroupPFMs.m index f056cf029..5f1950188 100755 --- a/PFM/scripts/GroupPFMs.m +++ b/PFM/scripts/GroupPFMs.m @@ -1,5 +1,5 @@ -function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) -% GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) +function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputSTRING, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) +% GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputSTRING, RegName, LowResMesh, RunsXNumTimePoints, PFMFolder) % This function aggregates individual subject PFM results and computes % group-level time course masks, spectra, maps, and statistics. % @@ -7,7 +7,7 @@ function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowR % StudyFolder - Path to the study directory % SubjlistRaw - Subject list as @ separated string % PFMdim - PFM dimensionality -% OutputPrefix - Prefix for output files +% OutputSTRING - Output string for files % RegName - Registration string (e.g., '_MSMAll') % LowResMesh - Mesh resolution (e.g., '32' for 32k_fs_LR) % RunsXNumTimePoints - Total expected timepoints across runs @@ -38,10 +38,10 @@ function GroupPFMs(StudyFolder, SubjlistRaw, PFMdim, OutputPrefix, RegName, LowR %% Load individual PFM results from standard subject locations % Load spatial maps, volume maps, time courses, and power spectra - PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - TCSSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - SpectraSub = ciftiopen([subjDir '/' subj '.' OutputPrefix '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + PFMMapsSub = ciftiopen([subjDir '/' subj '.' OutputSTRING '_DR' RegName '.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + PFMVolMapsSub = ciftiopen([subjDir '/' subj '.' OutputSTRING '_DR' RegName '_vol.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + TCSSub = ciftiopen([subjDir '/' subj '.' OutputSTRING '_DR' RegName '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + SpectraSub = ciftiopen([subjDir '/' subj '.' OutputSTRING '_DR' RegName '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); %% Clean up NaNs and Infs in volume maps % Replace invalid values with zeros to prevent propagation to group statistics diff --git a/PFM/scripts/GroupPFMs.sh b/PFM/scripts/GroupPFMs.sh index 1c943b597..761b70717 100755 --- a/PFM/scripts/GroupPFMs.sh +++ b/PFM/scripts/GroupPFMs.sh @@ -34,7 +34,7 @@ PARAMETERs are [ ] = optional; < > = user supplied value opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains all subjects" opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of subject IDs separated by @s" opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" -opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" +opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string for files" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string corresponding to the input files" opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' 'integer' "total timepoints across runs (e.g., 4800 for rest, 3880 for task)" @@ -82,7 +82,7 @@ esac this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjlistRaw" "$PFMdim" "$OutputPrefix" "$RegString" "$LowResMesh" "$RunsXNumTimePoints" "$PFMFolder") +matlab_argarray=("$StudyFolder" "$SubjlistRaw" "$PFMdim" "$OutputSTRING" "$RegString" "$LowResMesh" "$RunsXNumTimePoints" "$PFMFolder") case "$MatlabMode" in (0) diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 0f1fdc63d..6fb0bec71 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,5 +1,5 @@ -function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) -% PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputPrefix, RegString, LowResMesh, TR, PFMFolder) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) +% PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) % This function imports PROFUMO results and generates CIFTI-format time courses % and power spectra for each subject. The outputs are used for subsequent % group-level PFM analysis. @@ -11,7 +11,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % ConcatName - Name of concatenated fMRI dataset (empty if single runs) % fMRIProcSTRING - Processing string component (e.g., '_Atlas_hp200_clean') % OutputfMRIName - Name of output fMRI dataset -% OutputPrefix - Prefix for output files +% OutputSTRING - Output string for files % RegString - Registration string % LowResMesh - Mesh resolution (e.g., '10' for 10k) % TR - Repetition time in seconds @@ -91,15 +91,15 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Save individual-level results % Save original and amplitude-modulated time courses and spectra - ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); %% Copy individual PFM maps % Link PROFUMO spatial maps to subject's fsaverage space directory - copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputPrefix RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); + copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) end \ No newline at end of file diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index f87e6d45c..7c04f6a47 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -36,7 +36,7 @@ opts_AddMandatory '--subject-list' 'SubjListRaw' '100206@100307...' 'list of sub opts_AddMandatory '--fmri-names' 'fMRIListRaw' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' 'list of fmri run names separated by @s' opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing" opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" -opts_AddMandatory '--output-prefix' 'OutputPrefix' 'string' "output prefix for files" +opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string for files" opts_AddMandatory '--surf-reg-name' 'RegName' 'MSMAll' "the registration string" opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" @@ -89,7 +89,7 @@ IFS='@' read -a fMRIList <<<"$fMRIListRaw" this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputPrefix" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") case "$MatlabMode" in (0) From 5ce14787e455ee678fa8e3ecbb6891721dab136c Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 10 Feb 2026 10:06:35 -0600 Subject: [PATCH 22/74] remove low ica-dims arguments, because weighted regression no longer supported. --- PFM/PFMPipeline.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index b9f3059df..37b6f40ce 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -56,8 +56,6 @@ opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initiali opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' #RSN regression specific parameters -opts_AddOptional '--low-dims' 'LowDims' 'string' "low dimensionalities for RSN regression" '7@8@9@10@11@12@13@14@15@16@17@18@19@20@21' -opts_AddOptional '--low-dims-template-file' 'LowDimTemplate' 'path' "low dimensionality template name for RSN regression" '' opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' opts_AddOptional '--scale-factor' 'ScaleFactor' 'float' 'scale factor for RSN regression' '0.01' @@ -254,8 +252,6 @@ do --low-res="$LowResMesh" --proc-string="_$fMRIProcSTRING" --method="dual" - --low-ica-dims="$LowDims" - --low-ica-template-name="$LowDimTemplate" --output-string="$OutputSTRING" --output-spectra="$RunsXNumTimePoints" --volume-template-cifti="$VolumeTemplateFile" From 3d5eee29536484e7f7c619014202ebd94c9827f2 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 16 Feb 2026 15:03:21 -0600 Subject: [PATCH 23/74] clear PFM folder in RunPROFUMO step to prevent + file creation on reruns --- PFM/PFMPipeline.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 37b6f40ce..6e1e08afb 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -149,6 +149,13 @@ do # Calculate low rank data parameter LowRankData=$((PFMdim * 5)) + # if PFM output directory exists, clear it, becuase otherwise PROFUMO create + files instead of overwriting existing ones + if [[ -d "${PFMFolder}" ]] + then + log_Warn "PFM output folder ${PFMFolder} already exists, clearing contents" + rm -rf "${PFMFolder:?}" + fi + # Create output directory mkdir -p "${PFMFolder}" From 08533a12211f50859a4e7fe831a5bd6e98834c12 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 16 Feb 2026 17:41:04 -0600 Subject: [PATCH 24/74] revise PFMdir clearing to leave in the dataLocations.json created by the batch script --- PFM/PFMPipeline.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 6e1e08afb..cef723d13 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -149,14 +149,12 @@ do # Calculate low rank data parameter LowRankData=$((PFMdim * 5)) - # if PFM output directory exists, clear it, becuase otherwise PROFUMO create + files instead of overwriting existing ones + # if PFM output directory exists, clear it (except dataLocations.json) because PROFUMO otherwise creates "+" files instead of overwriting if [[ -d "${PFMFolder}" ]] then log_Warn "PFM output folder ${PFMFolder} already exists, clearing contents" - rm -rf "${PFMFolder:?}" + find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -delete fi - - # Create output directory mkdir -p "${PFMFolder}" # Build optional initialMaps argument From 1d3a04e649de1f58b9bf030321141809f672c70b Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 24 Feb 2026 11:23:00 -0600 Subject: [PATCH 25/74] add --loadSequentially argument to PROFUMO call by default, convert PROFUMO input dtseries ciftis to cifti v1 to work around weird PROFUMO file loading bug where it thinks the files aren't the same size. --- PFM/PFMPipeline.sh | 55 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index cef723d13..5093355ac 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -51,6 +51,7 @@ opts_AddOptional '--profumo-singularity' 'ProfumoSingularity' 'path' "path to PR opts_AddOptional '--profumo-random-seed' 'RandomSeed' 'integer' "random seed for PROFUMO" '123' opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'integer' "number of iterations of group-level spatial decomposition before inferring full model" '5' opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" +opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -164,6 +165,34 @@ do InitialMapsArg="--initialMaps ${InitialMaps}" fi + # Build optional loadSequentially argument + LoadSequentiallyArg="" + LoadSequentiallyBool=$(opts_StringToBool "$LoadSequentially") + if ((LoadSequentiallyBool)) + then + LoadSequentiallyArg="--loadSequentially" + fi + + ## CIFTI v2 to v1 conversion workaround for Armadillo SIMD buffer alignment issue in PROFUMO + # Extract all file paths from the JSON + cat "${ProfumoConfig}" | while IFS= read -r line; do + # Only process lines that contain .nii" (file paths) + if [[ "$line" != *'.nii"'* ]]; then continue;fi + + # Extract the file path from JSON value (text between colons and quotes) + # Pattern: "RUNNAME": "/path/to/file.nii" + filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') + + # Check CIFTI version + ciftiVersion=$(wb_command -nifti-information "$filePath" -print-xml -version 2 2>/dev/null | grep -oP 'CIFTI.*Version="\K[0-9]' || echo "") + if ((ciftiVersion == 1)); then + continue # If already v1, leave it be + else # convert cifti to v1 cifti + log_Msg "Converting CIFTI v2 to v1: $filePath" + wb_command -file-convert -cifti-version-convert "$filePath" 1 "$filePath" + fi + done + # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ @@ -172,7 +201,7 @@ do "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" \ - --multiStartIterations "${MultiStartIterations}" ${InitialMapsArg} + --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ @@ -180,8 +209,28 @@ do "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ - --multiStartIterations "${MultiStartIterations}" ${InitialMapsArg} + --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} + ## Undo CIFTI v2 to v1 conversion workaround for Armadillo SIMD buffer alignment issue in PROFUMO + # currently comment out to just leave the files as v1 ciftis + # cat "${ProfumoConfig}" | while IFS= read -r line; do + # # Only process lines that contain .nii" (file paths) + # if [[ "$line" != *'.nii"'* ]]; then continue;fi + + # # Extract the file path from JSON value (text between colons and quotes) + # # Pattern: "RUNNAME": "/path/to/file.nii" + # filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') + + # # Check CIFTI version + # ciftiVersion=$(wb_command -nifti-information "$filePath" -print-xml -version 2 2>/dev/null | grep -oP 'CIFTI.*Version="\K[0-9]' || echo "") + # if ((ciftiVersion == 2)); then + # continue # If already v2, leave it be + # else # convert cifti to v2 cifti + # log_Msg "Converting CIFTI v1 to v2: $filePath" + # wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" + # fi + # done + log_Msg "Running PROFUMO postprocessing" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ @@ -199,6 +248,8 @@ do "${PFM_PATH}" \ "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" + + ;; (PostPROFUMO) log_Msg "Running PostPROFUMO step" From ed0e699ab0b91cf94429bc4bcc07e9ddb537d570 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 24 Feb 2026 12:21:51 -0600 Subject: [PATCH 26/74] change the cifti version conversion log message to warning --- PFM/PFMPipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 5093355ac..c31d7c2d0 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -188,7 +188,7 @@ do if ((ciftiVersion == 1)); then continue # If already v1, leave it be else # convert cifti to v1 cifti - log_Msg "Converting CIFTI v2 to v1: $filePath" + log_Warn "Converting CIFTI v2 to v1: $filePath" wb_command -file-convert -cifti-version-convert "$filePath" 1 "$filePath" fi done @@ -226,7 +226,7 @@ do # if ((ciftiVersion == 2)); then # continue # If already v2, leave it be # else # convert cifti to v2 cifti - # log_Msg "Converting CIFTI v1 to v2: $filePath" + # log_Warn "Converting CIFTI v1 to v2: $filePath" # wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" # fi # done From e354d0f47c7932673c987e81fd1d5062cc7a5a3a Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 25 Feb 2026 12:06:56 -0600 Subject: [PATCH 27/74] update PFM usages and refactor usages for newopts. --- PFM/scripts/GroupPFMs.sh | 17 +---------- PFM/scripts/PostPROFUMO.sh | 18 ++---------- PostFreeSurfer/PostFreeSurferPipeline.sh | 20 ++----------- PostFreeSurfer/PostFreeSurferPipeline_1res.sh | 20 ++----------- .../scripts/GenerateStructuralScenes.sh | 18 ++---------- TaskfMRIAnalysis/TaskfMRIAnalysis.sh | 27 +++++------------ .../scripts/makeSubjectTaskSummary.sh | 29 +++++-------------- fMRISurface/scripts/GenerateFMRIScenes.sh | 18 ++---------- global/scripts/ExtractFromMRFIXConcat.sh | 20 ++----------- global/scripts/RSNregression.sh | 16 ++-------- tICA/scripts/ComputeGroupTICA.sh | 18 ++---------- tICA/scripts/ComputeTICAFeatures.sh | 18 ++---------- tICA/scripts/ConcatGroupSICA.sh | 18 ++---------- tICA/scripts/GroupSICA.sh | 18 ++---------- tICA/scripts/MIGP.sh | 18 ++---------- tICA/scripts/tICACleanData.sh | 16 ++-------- 16 files changed, 44 insertions(+), 265 deletions(-) diff --git a/PFM/scripts/GroupPFMs.sh b/PFM/scripts/GroupPFMs.sh index 761b70717..fe0429de4 100755 --- a/PFM/scripts/GroupPFMs.sh +++ b/PFM/scripts/GroupPFMs.sh @@ -13,22 +13,7 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: Import PFM notes and create time courses, spectra, and maps - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +opts_SetScriptDescription "Aggregate individual subject PFM results and compute group-level time course masks, spectra, maps, and statistics" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder that contains all subjects" diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index 7c04f6a47..699c67c6e 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -13,22 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: Import PFM notes and create time courses, spectra, and maps - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "Import PROFUMO outputs and create time courses, spectra, and maps" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" diff --git a/PostFreeSurfer/PostFreeSurferPipeline.sh b/PostFreeSurfer/PostFreeSurferPipeline.sh index f2e0944a8..03be00da1 100755 --- a/PostFreeSurfer/PostFreeSurferPipeline.sh +++ b/PostFreeSurfer/PostFreeSurferPipeline.sh @@ -50,27 +50,13 @@ source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/processingmodecheck.shlib" "$@" # Check processing mode requirements +#description of this script to use in usage +opts_SetScriptDescription "takes FreeSurfer output folder and converts files into HCP format/organization, etc." + log_Msg "Platform Information Follows: " uname -a "$HCPPIPEDIR"/show_version -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: takes FreeSurfer output folder and converts files into HCP format/organization, etc. - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} - defaultSigma=$(echo "sqrt(200)" | bc -l) opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" "--path" diff --git a/PostFreeSurfer/PostFreeSurferPipeline_1res.sh b/PostFreeSurfer/PostFreeSurferPipeline_1res.sh index 365d4b6b3..3ed080274 100755 --- a/PostFreeSurfer/PostFreeSurferPipeline_1res.sh +++ b/PostFreeSurfer/PostFreeSurferPipeline_1res.sh @@ -26,27 +26,13 @@ source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/processingmodecheck.shlib" "$@" # Check processing mode requirements +#description of this script to use in usage +opts_SetScriptDescription "takes FreeSurfer output folder and converts files into HCP format/organization, etc." + log_Msg "Platform Information Follows: " uname -a "$HCPPIPEDIR"/show_version -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: takes FreeSurfer output folder and converts files into HCP format/organization, etc. - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} - defaultSigma=$(echo "sqrt(200)" | bc -l) #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] diff --git a/PostFreeSurfer/scripts/GenerateStructuralScenes.sh b/PostFreeSurfer/scripts/GenerateStructuralScenes.sh index e6bbdd26b..2cd2cb022 100755 --- a/PostFreeSurfer/scripts/GenerateStructuralScenes.sh +++ b/PostFreeSurfer/scripts/GenerateStructuralScenes.sh @@ -13,22 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" "$@" source "$HCPPIPEDIR/global/scripts/relativePath.shlib" "$@" -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: makes QC scenes and captures for HCP FreeSurfer pipelines - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "makes QC scenes and captures for HCP FreeSurfer pipelines" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/TaskfMRIAnalysis/TaskfMRIAnalysis.sh b/TaskfMRIAnalysis/TaskfMRIAnalysis.sh index 9f187fb10..75a6ff9a2 100755 --- a/TaskfMRIAnalysis/TaskfMRIAnalysis.sh +++ b/TaskfMRIAnalysis/TaskfMRIAnalysis.sh @@ -57,29 +57,16 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/fsl_version.shlib" # Function for getting FSL version -# -------------------------------------------------------------------------------- -# Usage Description Function -# -------------------------------------------------------------------------------- +# ------------------------------------------------------- +# Usage Description +# ------------------------------------------------------- -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: Run TaskfMRIAnalysis pipeline for a subject. Pipeline will run Level1 (scan-level) analyses, and Level2 (single subject-level) analysis as specified. - -Usage: $log_ToolName arguments... -[ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "Run TaskfMRIAnalysis pipeline for a subject. Pipeline will run Level1 (scan-level) analyses, and Level2 (single subject-level) analysis as specified." -# ------------------------------------------------------------------------------ +# ------------------------------------------- # Parse Command Line Options -# ------------------------------------------------------------------------------ +# ------------------------------------------- #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/TaskfMRIAnalysis/scripts/makeSubjectTaskSummary.sh b/TaskfMRIAnalysis/scripts/makeSubjectTaskSummary.sh index ed6866b87..d35633738 100755 --- a/TaskfMRIAnalysis/scripts/makeSubjectTaskSummary.sh +++ b/TaskfMRIAnalysis/scripts/makeSubjectTaskSummary.sh @@ -55,29 +55,16 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/fsl_version.shlib" # Function for getting FSL version -# -------------------------------------------------------------------------------- -# Usage Description Function -# -------------------------------------------------------------------------------- - -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: Run TaskfMRIAnalysis pipeline for a subject. Pipeline will run Level1 (scan-level) analyses, and Level2 (single subject-level) analysis as specified. - -Usage: $log_ToolName arguments... -[ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +# ------------------------------------------- +# Usage Description +# ------------------------------------------- -# ------------------------------------------------------------------------------ +#description of this script to use in usage +opts_SetScriptDescription "Run TaskfMRIAnalysis pipeline for a subject. Pipeline will run Level1 (scan-level) analyses, and Level2 (single subject-level) analysis as specified." + +# ------------------------------------------- # Parse Command Line Options -# ------------------------------------------------------------------------------ +# ------------------------------------------- #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/fMRISurface/scripts/GenerateFMRIScenes.sh b/fMRISurface/scripts/GenerateFMRIScenes.sh index 31c98860b..d63654163 100755 --- a/fMRISurface/scripts/GenerateFMRIScenes.sh +++ b/fMRISurface/scripts/GenerateFMRIScenes.sh @@ -12,22 +12,8 @@ source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/relativePath.shlib" "$@" -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: makes QC scenes and captures for HCP fMRIVolume pipeline - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "makes QC scenes and captures for HCP fMRIVolume pipeline" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/global/scripts/ExtractFromMRFIXConcat.sh b/global/scripts/ExtractFromMRFIXConcat.sh index f0a33a8a5..192e8d878 100755 --- a/global/scripts/ExtractFromMRFIXConcat.sh +++ b/global/scripts/ExtractFromMRFIXConcat.sh @@ -11,24 +11,8 @@ fi source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: extract a specified set of runs from an MR FIX - concatenated file and reconcatenate them. Typically this is used to extract - the resting state runs from a combined task and resting state MR FIX run. - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "extract a specified set of runs from an MR FIX concatenated file and reconcatenate them. Typically this is used to extract the resting state runs from a combined task and resting state MR FIX run." #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/global/scripts/RSNregression.sh b/global/scripts/RSNregression.sh index 5e901a941..09f23a98f 100755 --- a/global/scripts/RSNregression.sh +++ b/global/scripts/RSNregression.sh @@ -13,20 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: regresses group ICA spatial maps into individual data in order to obtain individual spatial maps of where the subject's similar function is - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments -} +#description of this script to use in usage +opts_SetScriptDescription "regresses group ICA spatial maps into individual data in order to obtain individual spatial maps of where the subject's similar function is" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value other than empty string if AddOptional], [compatibility flag, ...] opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" diff --git a/tICA/scripts/ComputeGroupTICA.sh b/tICA/scripts/ComputeGroupTICA.sh index 89b0c4ae4..e995bae7a 100755 --- a/tICA/scripts/ComputeGroupTICA.sh +++ b/tICA/scripts/ComputeGroupTICA.sh @@ -13,22 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: does stuff - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "does stuff" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/tICA/scripts/ComputeTICAFeatures.sh b/tICA/scripts/ComputeTICAFeatures.sh index 486ae6472..a5890bd1b 100755 --- a/tICA/scripts/ComputeTICAFeatures.sh +++ b/tICA/scripts/ComputeTICAFeatures.sh @@ -14,22 +14,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: does stuff - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "does stuff" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/tICA/scripts/ConcatGroupSICA.sh b/tICA/scripts/ConcatGroupSICA.sh index d39e4063a..e39ad6de5 100755 --- a/tICA/scripts/ConcatGroupSICA.sh +++ b/tICA/scripts/ConcatGroupSICA.sh @@ -13,22 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: does stuff - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "does stuff" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/tICA/scripts/GroupSICA.sh b/tICA/scripts/GroupSICA.sh index 40e08789a..8cac4e9f2 100755 --- a/tICA/scripts/GroupSICA.sh +++ b/tICA/scripts/GroupSICA.sh @@ -12,22 +12,8 @@ source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: does stuff - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "does stuff" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/tICA/scripts/MIGP.sh b/tICA/scripts/MIGP.sh index c4affb1e0..51d2a5503 100755 --- a/tICA/scripts/MIGP.sh +++ b/tICA/scripts/MIGP.sh @@ -13,22 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: does stuff - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments - - #do not use exit, the parsing code takes care of it -} +#description of this script to use in usage +opts_SetScriptDescription "does stuff" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value if AddOptional], [compatibility flag, ...] #help info for option gets printed like "--foo=<$3> - $4" diff --git a/tICA/scripts/tICACleanData.sh b/tICA/scripts/tICACleanData.sh index 16ae27452..bbfcc2e73 100755 --- a/tICA/scripts/tICACleanData.sh +++ b/tICA/scripts/tICACleanData.sh @@ -13,20 +13,8 @@ source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" "$@" g_matlab_default_mode=1 -#this function gets called by opts_ParseArguments when --help is specified -function usage() -{ - #header text - echo " -$log_ToolName: regresses noise group temporal ICA components out of CIFTI and optionaly volume timeseries data and optionally correct the bias legacy field - -Usage: $log_ToolName PARAMETER... - -PARAMETERs are [ ] = optional; < > = user supplied value -" - #automatic argument descriptions - opts_ShowArguments -} +#description of this script to use in usage +opts_SetScriptDescription "regresses noise group temporal ICA components out of CIFTI and optionaly volume timeseries data and optionally correct the bias legacy field" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value other than empty string if AddOptional], [compatibility flag, ...] opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" From d1f57f9d798d13547d1d385e4fa459d0573f00fc Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 25 Feb 2026 12:15:43 -0600 Subject: [PATCH 28/74] re-write PROFUMO inputs (as cifti 2) with wb_command, unconditionally, to avoid SIMD buffer alignment issue in PROFUMO if files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. --- PFM/PFMPipeline.sh | 48 ++++++++-------------------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index c31d7c2d0..f01626c3a 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -173,25 +173,14 @@ do LoadSequentiallyArg="--loadSequentially" fi - ## CIFTI v2 to v1 conversion workaround for Armadillo SIMD buffer alignment issue in PROFUMO - # Extract all file paths from the JSON - cat "${ProfumoConfig}" | while IFS= read -r line; do - # Only process lines that contain .nii" (file paths) - if [[ "$line" != *'.nii"'* ]]; then continue;fi - - # Extract the file path from JSON value (text between colons and quotes) - # Pattern: "RUNNAME": "/path/to/file.nii" - filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') - - # Check CIFTI version - ciftiVersion=$(wb_command -nifti-information "$filePath" -print-xml -version 2 2>/dev/null | grep -oP 'CIFTI.*Version="\K[0-9]' || echo "") - if ((ciftiVersion == 1)); then - continue # If already v1, leave it be - else # convert cifti to v1 cifti - log_Warn "Converting CIFTI v2 to v1: $filePath" - wb_command -file-convert -cifti-version-convert "$filePath" 1 "$filePath" - fi - done + ## Rewrite input files with wb_command, as a failsafe to avoid SIMD buffer alignment issue in PROFUMO if + ## files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. + cat "${ProfumoConfig}" | \ + while IFS= read -r line; do + if [[ "$line" != *'.nii"'* ]]; then continue;fi # Only process lines that contain .nii" + filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') + wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" + done # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ @@ -210,27 +199,6 @@ do --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} - - ## Undo CIFTI v2 to v1 conversion workaround for Armadillo SIMD buffer alignment issue in PROFUMO - # currently comment out to just leave the files as v1 ciftis - # cat "${ProfumoConfig}" | while IFS= read -r line; do - # # Only process lines that contain .nii" (file paths) - # if [[ "$line" != *'.nii"'* ]]; then continue;fi - - # # Extract the file path from JSON value (text between colons and quotes) - # # Pattern: "RUNNAME": "/path/to/file.nii" - # filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') - - # # Check CIFTI version - # ciftiVersion=$(wb_command -nifti-information "$filePath" -print-xml -version 2 2>/dev/null | grep -oP 'CIFTI.*Version="\K[0-9]' || echo "") - # if ((ciftiVersion == 2)); then - # continue # If already v2, leave it be - # else # convert cifti to v2 cifti - # log_Warn "Converting CIFTI v1 to v2: $filePath" - # wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" - # fi - # done - log_Msg "Running PROFUMO postprocessing" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ From 85cceed084c5eaa5879b848ebea22241bca9cc9e Mon Sep 17 00:00:00 2001 From: arkky68 Date: Thu, 9 Apr 2026 20:25:23 -0500 Subject: [PATCH 29/74] Update PFM pipeline scripts (cherry picked from commit af161ed185433a29c6e3fb6cd22b34a9c7742c35) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 88 ++++++++++++----------- PFM/PFMPipeline.sh | 42 ++++++----- PFM/scripts/PostPROFUMO.m | 2 +- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index cd4f4c424..7ed5d9a3f 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -17,14 +17,19 @@ get_options() { local arguments=("$@") # initialize variables - StudyFolder="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" - Subjlist="$(find ${StudyFolder} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" - EnvironmentScript="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5_SetUpHCPPipeline.sh" - GroupAverageName="Mac25Rhesus_v5" - RegName="" + StudyFolder="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final" + #Subjlist="100610" + Subjlist="100610@102311@102816@104416@105923@108323@109123@111312@111514@114823@115017@115825@116726@118225@125525@126426@128935@130518@131217@131722@132118@134627@134829@135124@137128@140117@144226@145834@146129@146432@146735@146937@148133@150423@155938@156334@157336@158035@158136@159239@162935@164131@164636@165436@167036@167440@169343@169444@169747@171633@172130@173334@175237@176542@177140@177645@177746@178142@178243@178647@180533@181232@181636@182436@182739@185442@186949@187345@191033@191336@191841@192439@192641@193845@195041@196144@197348@198653@199655@200210@200311@200614@201515@203418@204521@205220@209228@212419@214019@214524@221319@233326@239136@246133@249947@251833@257845@263436@283543@318637@320826@330324@346137@352738@360030@365343@380036@381038@389357@393247@395756@397760@406836@412528@429040@436845@463040@467351@473952@525541@536647@541943@547046@550439@552241@562345@572045@573249@581450@585256@601127@617748@627549@638049@654552@671855@680957@690152@706040@724446@725751@732243@751550@757764@765864@770352@771354@782561@783462@789373@814649@818859@825048@826353@833249@859671@861456@871762@872764@878776@878877@898176@899885@901139@901442@905147@910241@926862@927359@942658@951457@958976@966975@973770@995174" + #ret exclude 473952 + #Subjlist="100610@102311@102816@104416@105923@108323@109123@111312@111514@114823@115017@115825@116726@118225@125525@126426@128935@130518@131217@131722@132118@134627@134829@135124@137128@140117@144226@145834@146129@146432@146735@146937@148133@150423@155938@156334@157336@158035@158136@159239@162935@164131@164636@165436@167036@167440@169343@169444@169747@171633@172130@173334@175237@176542@177140@177645@177746@178142@178243@178647@180533@181232@181636@182436@182739@185442@186949@187345@191033@191336@191841@192439@192641@193845@195041@196144@197348@198653@199655@200210@200311@200614@201515@203418@204521@205220@209228@212419@214019@214524@221319@233326@239136@246133@249947@251833@257845@263436@283543@318637@320826@330324@346137@352738@360030@365343@380036@381038@389357@393247@395756@397760@406836@412528@429040@436845@463040@467351@525541@536647@541943@547046@550439@552241@562345@572045@573249@581450@585256@601127@617748@627549@638049@654552@671855@680957@690152@706040@724446@725751@732243@751550@757764@765864@770352@771354@782561@783462@789373@814649@818859@825048@826353@833249@859671@861456@871762@872764@878776@878877@898176@899885@901139@901442@905147@910241@926862@927359@942658@951457@958976@966975@973770@995174" + + EnvironmentScript="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/Scripts/SetUpHCPPipeline_PFM_AY.sh" + GroupAverageName="S1200_MSMAll7T175" + + RegName="MSMAll" MatlabMode=1 RunLocal=0 - QUEUE="matlabparallelhigh.q@brainmappers-desktop3" + QUEUE="matlabparallelhigh.q" # parse arguments local index argument @@ -135,78 +140,75 @@ main() { # general settings # set the start step beginning from RunPROFUMO which is by default the first step - # StartStep="RunPROFUMO" - # StopStep="RunPROFUMO" - # StartStep="PostPROFUMO" - # StopStep="PostPROFUMO" - # StartStep="PostPROFUMO" - # StartStep="RSNRegression" - # StopStep="RSNRegression" - # StartStep="RunPROFUMO" - # StopStep="" - - StartStep="PostPROFUMO" + StartStep="RunPROFUMO" StopStep="GroupPFMs" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' - parLimit=5 + parLimit=-1 # general inputs - fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" - # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" - randSeed=1 # random seed for PROFUMO + fMRINames="rfMRI_REST1_LR@rfMRI_REST1_RL@rfMRI_REST2_LR@rfMRI_REST2_RL" - OutputfMRIName="BOLD_REST_CONCAT" + randSeed=123 # random seed for PROFUMO + + OutputfMRIName="rfMRI_REST" # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs - ConcatName="BOLD_REST_CONCAT" - # set the output spectra size for individual projection, RunsXNumTimePoints - subjectExpectedTimepoints="8508" + ConcatName="" + + # set the output spectra size for individual projection, RunsXNumTimePoints #subjectExpectedTimepoints="3655" + subjectExpectedTimepoints="4800" + # set temporal highpass full-width (2*sigma) used in preprocessing - HighPass="pd2" + HighPass="2000" + #set fMRIResolution of data, like '2','1.60' or '2.40' + fMRIResolution="2.0" # PFM settings for REST data - PFMdim="16" # set the PFM dimensionality - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${GroupAverageName}_${OutputfMRIName}_PFM_d${PFMdim}_s${randSeed}_test - # Reference image for PROFUMO - RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" + # set the PFM dimensionality + PFMdim="99" + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} + # Reference image for PROFUMO + RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_rclean_tclean_meanvn.dscalar.nii" + # set the file name component representing the preprocessing already done - fMRIProcSTRING="hp${HighPass}_clean" + fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" # set the mesh resolution, like '32' for 32k_fs_LR - LowResMesh="10" + LowResMesh="32" + # Define OutputSTRING with seed designation OutputSTRING="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs" # RSN regression settings - LowDims="6" FixLegacyBiasString="NO" ScaleFactor="0.01" + LowDims="7@8@9@10@11@12@13@14@15@16@17@18@19@20@21" # Volume template file - VolumeTemplateCIFTI="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus/Mac25Rhesus_v5/MNINonLinear/Results/Mac25Rhesus_v5_VolMaps_16_template.dscalar.nii" - + VolumeTemplateCIFTI="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll7T175/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" + # PROFUMO settings - ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" + ProfumoSingularity="/media/myelin/andrea/HCPpipelines/PFM/profumo_v2.sif" ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="0.702" + TR="1.0" ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" - nStarts="1" # number of multi-start iterations for PROFUMO + nStarts="5" # number of multi-start iterations for PROFUMO RandomSeed="$randSeed" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below - + # build Profumo data location json mkdir -p $PFMFolder echo '{' > $ProfumoConfig for Subject in $(echo $Subjlist | tr "@" "\n"); do echo -e "\t\"$Subject\": {" >> $ProfumoConfig for fMRIName in $(echo $fMRINames | tr "@" "\n"); do - runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${fMRIProcSTRING}.dtseries.nii" + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" if [[ -e $runFile ]]; then echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig fi @@ -234,6 +236,7 @@ main() { --pfm-folder="$PFMFolder" \ --surf-reg-name="$RegName" \ --concat-name="$ConcatName" \ + --fmri-resolution="$fMRIResolution" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ --low-dims="$LowDims" \ @@ -249,7 +252,7 @@ main() { --profumo-threads="$ProfumoThreads" \ --profumo-dof-correction="$DOFCorrection" \ --profumo-cov-model="$CovModel" \ - --profumo-multi-start-iterations="$nStarts"\ + --profumo-multi-start-iterations="$nStarts" \ --profumo-random-seed="$RandomSeed" \ --ref-image="$RefImage" \ --volume-template-file="$VolumeTemplateCIFTI" @@ -260,4 +263,5 @@ main() { # # Invoke the main function to get things started # -main "$@" \ No newline at end of file +main "$@" + diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index f01626c3a..caadf2137 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -42,6 +42,7 @@ opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO p opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs" "total timepoints across runs" opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' +opts_AddMandatory '--fmri-resolution' 'fMRIResolution' "fMRI resolution string for template selection, like '2','1.60' or '2.40'" '' #PROFUMO specific parameters opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threads for PROFUMO" '25' @@ -59,6 +60,8 @@ opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like ' #RSN regression specific parameters opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' opts_AddOptional '--scale-factor' 'ScaleFactor' 'float' 'scale factor for RSN regression' '0.01' +opts_AddOptional '--low-dims' 'LowDims' 'dims' "low ICA dimensions separated by @s for RSN regression" '' + #general settings opts_AddOptional '--starting-step' 'startStep' 'step' "what step to start processing at, one of: @@ -226,7 +229,7 @@ do --subject-list="$SubjlistRaw" \ --fmri-names="$fMRINames" \ --concat-name="$ConcatName" \ - --proc-string="$fMRIProcSTRING" \ + --proc-string="_Atlas_${RegName}_$fMRIProcSTRING" \ --output-fmri-name="$OutputfMRIName" \ --output-string="$OutputSTRING" \ --surf-reg-name="$RegName" \ @@ -239,25 +242,30 @@ do log_Msg "Running RSNRegression step" # Set up template paths - # LowDimTemplate="${StudyFolder}/${GroupAverageName}/MNINonLinear/Results/${OutputfMRIName}/sICA/melodic_oIC_${PFMdim}.dscalar.nii" + LowDimTemplate="${StudyFolder}/${GroupAverageName}/MNINonLinear/Results/${OutputfMRIName}/sICA/melodic_oIC_${PFMdim}.dscalar.nii" for Subject in "${Subjlist[@]}" do - # Build list of existing fMRI files for this subject (same logic as your example) - fMRINamesForSub="" - for fMRIName in "${fMRINamesArray[@]}" - do - if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}${fMRIProcSTRING}.dtseries.nii" ]] - then - if [[ "$fMRINamesForSub" != "" ]] + if [[ "$ConcatName" != "" ]] + then + fMRINamesForSub="${ConcatName}" + else + # Build list of existing fMRI files for this subject (same logic as your example) + fMRINamesForSub="" + for fMRIName in "${fMRINamesArray[@]}" + do + if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" ]] then - fMRINamesForSub="${fMRINamesForSub}@${fMRIName}" - else - fMRINamesForSub="${fMRIName}" + if [[ "$fMRINamesForSub" != "" ]] + then + fMRINamesForSub="${fMRINamesForSub}@${fMRIName}" + else + fMRINamesForSub="${fMRIName}" + fi fi - fi - done - + done + fi + if [[ "$fMRINamesForSub" == "" ]] then log_Warn "No valid fMRI runs found for subject $Subject, skipping" @@ -271,10 +279,12 @@ do rsn_cmd=("$HCPPIPEDIR"/global/scripts/RSNregression.sh --study-folder="$StudyFolder" --subject="$Subject" - --subject-timeseries="$ConcatName" # "$fMRINamesForSub" + --subject-timeseries="$fMRINamesForSub" # "$fMRINamesForSub" --surf-reg-name="$RegName" + --low-ica-dims="$LowDims" --low-res="$LowResMesh" --proc-string="_$fMRIProcSTRING" + --low-ica-template-name="$LowDimTemplate" --method="dual" --output-string="$OutputSTRING" --output-spectra="$RunsXNumTimePoints" diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 6fb0bec71..2845bcb85 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -64,7 +64,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', numel(runTCS), 1)]; + TCS=[TCS ; runTCS.*repmat(runAmp',length(runTCS),1)]; end % for r = 1:numel(subfMRINames) %% Create original time course and spectral CIFTI files From 9922f3722043a7f6b894b53a406ac002a76e721c Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 9 Apr 2026 20:36:58 -0500 Subject: [PATCH 30/74] Update subject list and environment script path (cherry picked from commit f4f0e7bf6afffa692265aa85c505349fac29b0a4) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 7ed5d9a3f..39ac46d44 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -18,12 +18,8 @@ get_options() { # initialize variables StudyFolder="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final" - #Subjlist="100610" - Subjlist="100610@102311@102816@104416@105923@108323@109123@111312@111514@114823@115017@115825@116726@118225@125525@126426@128935@130518@131217@131722@132118@134627@134829@135124@137128@140117@144226@145834@146129@146432@146735@146937@148133@150423@155938@156334@157336@158035@158136@159239@162935@164131@164636@165436@167036@167440@169343@169444@169747@171633@172130@173334@175237@176542@177140@177645@177746@178142@178243@178647@180533@181232@181636@182436@182739@185442@186949@187345@191033@191336@191841@192439@192641@193845@195041@196144@197348@198653@199655@200210@200311@200614@201515@203418@204521@205220@209228@212419@214019@214524@221319@233326@239136@246133@249947@251833@257845@263436@283543@318637@320826@330324@346137@352738@360030@365343@380036@381038@389357@393247@395756@397760@406836@412528@429040@436845@463040@467351@473952@525541@536647@541943@547046@550439@552241@562345@572045@573249@581450@585256@601127@617748@627549@638049@654552@671855@680957@690152@706040@724446@725751@732243@751550@757764@765864@770352@771354@782561@783462@789373@814649@818859@825048@826353@833249@859671@861456@871762@872764@878776@878877@898176@899885@901139@901442@905147@910241@926862@927359@942658@951457@958976@966975@973770@995174" - #ret exclude 473952 - #Subjlist="100610@102311@102816@104416@105923@108323@109123@111312@111514@114823@115017@115825@116726@118225@125525@126426@128935@130518@131217@131722@132118@134627@134829@135124@137128@140117@144226@145834@146129@146432@146735@146937@148133@150423@155938@156334@157336@158035@158136@159239@162935@164131@164636@165436@167036@167440@169343@169444@169747@171633@172130@173334@175237@176542@177140@177645@177746@178142@178243@178647@180533@181232@181636@182436@182739@185442@186949@187345@191033@191336@191841@192439@192641@193845@195041@196144@197348@198653@199655@200210@200311@200614@201515@203418@204521@205220@209228@212419@214019@214524@221319@233326@239136@246133@249947@251833@257845@263436@283543@318637@320826@330324@346137@352738@360030@365343@380036@381038@389357@393247@395756@397760@406836@412528@429040@436845@463040@467351@525541@536647@541943@547046@550439@552241@562345@572045@573249@581450@585256@601127@617748@627549@638049@654552@671855@680957@690152@706040@724446@725751@732243@751550@757764@765864@770352@771354@782561@783462@789373@814649@818859@825048@826353@833249@859671@861456@871762@872764@878776@878877@898176@899885@901139@901442@905147@910241@926862@927359@942658@951457@958976@966975@973770@995174" - - EnvironmentScript="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/Scripts/SetUpHCPPipeline_PFM_AY.sh" + Subjlist="100610@102311" + EnvironmentScript="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/Scripts/SetUpHCPPipeline_PFM.sh" GroupAverageName="S1200_MSMAll7T175" RegName="MSMAll" From f7d1d3d57fe463998199ec91b2db0aa9bcdc6484 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Wed, 15 Apr 2026 15:36:27 -0500 Subject: [PATCH 31/74] Update Examples/Scripts/RunPFMpipelineFullRun.sh Co-authored-by: Burke Rosen (cherry picked from commit 16152f6329fcaa951cb32841db727ccb64a96167) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 39ac46d44..d8da37884 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -19,7 +19,7 @@ get_options() { # initialize variables StudyFolder="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final" Subjlist="100610@102311" - EnvironmentScript="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/Scripts/SetUpHCPPipeline_PFM.sh" + EnvironmentScript="${HOME}/projects/HCPpipelines/Examples/Scripts/SetUpHCPPipeline.sh" GroupAverageName="S1200_MSMAll7T175" RegName="MSMAll" From 5e26211b4fe603e09520f7d216626fdc67f5697a Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Tue, 28 Apr 2026 15:43:50 -0500 Subject: [PATCH 32/74] Update Examples/Scripts/RunPFMpipelineFullRun.sh Co-authored-by: Burke Rosen (cherry picked from commit b1b7120c0a6b9b952c921738e1565a54b8c35806) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index d8da37884..10579b643 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -17,7 +17,7 @@ get_options() { local arguments=("$@") # initialize variables - StudyFolder="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final" + StudyFolder="${HOME}/data/HCPpipelines_ExampleData" Subjlist="100610@102311" EnvironmentScript="${HOME}/projects/HCPpipelines/Examples/Scripts/SetUpHCPPipeline.sh" GroupAverageName="S1200_MSMAll7T175" From 8cf3c8ae6153f0793660aabaa828053f2372664b Mon Sep 17 00:00:00 2001 From: arkky68 Date: Tue, 28 Apr 2026 15:48:22 -0500 Subject: [PATCH 33/74] Ignore local sif file (cherry picked from commit 06673e3315d95c2049ba876c123bc1a0b4e5f5af) --- PFM/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 PFM/.gitignore diff --git a/PFM/.gitignore b/PFM/.gitignore new file mode 100644 index 000000000..4adcc2a24 --- /dev/null +++ b/PFM/.gitignore @@ -0,0 +1 @@ +profumo_v2.sif From 1764c3f3ad09fcdbfbd348aaf86514a814d484a4 Mon Sep 17 00:00:00 2001 From: arkky68 Date: Thu, 7 May 2026 22:39:08 -0500 Subject: [PATCH 34/74] Add Wishart filter prefiltering step to PFM pipeline (cherry picked from commit 28d1c024cc94f2d9835546966c9f11e4b356f9e7) --- PFM/PFMPipeline.sh | 87 +++++++++-- PFM/scripts/WishartFilter.m | 281 +++++++++++++++++++++++++++++++++++ PFM/scripts/WishartFilter.sh | 85 +++++++++++ 3 files changed, 437 insertions(+), 16 deletions(-) create mode 100644 PFM/scripts/WishartFilter.m create mode 100755 PFM/scripts/WishartFilter.sh diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index caadf2137..892f09179 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -53,6 +53,7 @@ opts_AddOptional '--profumo-random-seed' 'RandomSeed' 'integer' "random seed for opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'integer' "number of iterations of group-level spatial decomposition before inferring full model" '5' opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' +opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filter iterations for prefiltering (0 to skip)" '0' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -60,7 +61,6 @@ opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like ' #RSN regression specific parameters opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' opts_AddOptional '--scale-factor' 'ScaleFactor' 'float' 'scale factor for RSN regression' '0.01' -opts_AddOptional '--low-dims' 'LowDims' 'dims' "low ICA dimensions separated by @s for RSN regression" '' #general settings @@ -145,6 +145,58 @@ do log_Err_Abort "Reference image must be specified with --ref-image" fi + ProfumoConfigToUse="${ProfumoConfig}" + if [[ "$NumWishart" -gt 0 ]] + then + log_Msg "Running Wishart filtering with ${NumWishart} iterations before PROFUMO" + WFTempDir=$(mktemp -d "${PFMFolder}/WF_tmp.XXXXXX") + for Subject in "${Subjlist[@]}" + do + mkdir -p "${WFTempDir}/${Subject}" + inputList="" + outputList="" + for fMRIName in "${fMRINamesArray[@]}" + do + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" + outputFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] + then + if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi + inputList+="$inputFile" + outputList+="$outputFile" + fi + done + log_Msg "Applying Wishart filter for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/WishartFilter.sh \ + --input="$inputList" \ + --output="$outputList" \ + --num-wishart="$NumWishart" \ + --pfm-dimension="$PFMdim" \ + --matlab-run-mode="$MatlabMode" + done + + ProfumoConfigToUse="${WFTempDir}/wishart_dataLocations.json" + echo '{' > "$ProfumoConfigToUse" + for Subject in "${Subjlist[@]}" + do + echo -e "\t\"$Subject\": {" >> "$ProfumoConfigToUse" + for fMRIName in "${fMRINamesArray[@]}" + do + WFFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$WFFile" ]] + then + echo -e "\t\t\"$fMRIName\": \"$WFFile\"," >> "$ProfumoConfigToUse" + fi + done + + perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" + echo -e "}," >> "$ProfumoConfigToUse" + done + perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" + echo "}" >> "$ProfumoConfigToUse" + log_Msg "WF prefiltering complete" + fi + # Set up PROFUMO paths PFM_PATH="${PFMFolder}/Analysis.pfm" RESULTS_PATH="${PFMFolder}/Results.ppp" @@ -178,18 +230,21 @@ do ## Rewrite input files with wb_command, as a failsafe to avoid SIMD buffer alignment issue in PROFUMO if ## files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. - cat "${ProfumoConfig}" | \ - while IFS= read -r line; do - if [[ "$line" != *'.nii"'* ]]; then continue;fi # Only process lines that contain .nii" - filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') - wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" - done - + if [[ "${NumWishart}" -eq 0 ]] + then + cat "${ProfumoConfig}" | \ + while IFS= read -r line; do + if [[ "$line" != *'.nii"'* ]]; then continue;fi # Only process lines that contain .nii" + filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') + wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" + done + fi + # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ - /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" \ @@ -197,7 +252,7 @@ do apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ - /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ @@ -220,7 +275,11 @@ do "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" - + if [[ "${NumWishart}" -gt 0 ]] + then + log_Msg "Cleaning up temporary Wishart filtered files" + rm -rf "${WFTempDir}" + fi ;; (PostPROFUMO) log_Msg "Running PostPROFUMO step" @@ -241,9 +300,7 @@ do (RSNRegression) log_Msg "Running RSNRegression step" - # Set up template paths - LowDimTemplate="${StudyFolder}/${GroupAverageName}/MNINonLinear/Results/${OutputfMRIName}/sICA/melodic_oIC_${PFMdim}.dscalar.nii" - + # Set up template paths for Subject in "${Subjlist[@]}" do if [[ "$ConcatName" != "" ]] @@ -281,10 +338,8 @@ do --subject="$Subject" --subject-timeseries="$fMRINamesForSub" # "$fMRINamesForSub" --surf-reg-name="$RegName" - --low-ica-dims="$LowDims" --low-res="$LowResMesh" --proc-string="_$fMRIProcSTRING" - --low-ica-template-name="$LowDimTemplate" --method="dual" --output-string="$OutputSTRING" --output-spectra="$RunsXNumTimePoints" diff --git a/PFM/scripts/WishartFilter.m b/PFM/scripts/WishartFilter.m new file mode 100644 index 000000000..ddc890fae --- /dev/null +++ b/PFM/scripts/WishartFilter.m @@ -0,0 +1,281 @@ +function WishartFilter(inputFiles, outputFiles, numWisharts, pfmDim) + numWisharts = str2double(numWisharts); + pfmDim = str2double(pfmDim); + inList = strsplit(inputFiles, ','); + outList = strsplit(outputFiles, ','); + + allData = []; + tpCounts = []; + for i = 1:length(inList) + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + tpCounts(i) = size(cii.cdata, 2); + allData = [allData cii.cdata]; + end + + Ntp = size(allData, 2); + + mask = range(allData, 2) > 0; + data = allData(mask, :); + + [u, EigS, v] = nets_svds(data', 0); + DOF = sum(diag(EigS) > (std(diag(EigS)) * 0.1)); + u(isnan(u)) = 0; + v(isnan(v)) = 0; + + noise_unst = (u(:, pfmDim:DOF) * EigS(pfmDim:DOF, pfmDim:DOF) * v(:, pfmDim:DOF)')'; + noise_unst_std = max(std(noise_unst, [], 2), 0.001); + clear noise_unst; + data_vn = data ./ repmat(noise_unst_std, 1, Ntp); + + lambda = flipud(eig(cov(data_vn))); + + lnb = 0.5; + MaxX = size(data_vn, 1); + origDOF = DOF; + clear EN + for i = 1:numWisharts + [x, en] = FitWishart(lnb, 0, DOF, MaxX, lambda(1:DOF)); + lambda = lambda(1:DOF) - en(1:DOF); + en_padded = zeros(origDOF, 1, 'single'); + en_padded(1:min(length(en), origDOF)) = en(1:min(length(en), origDOF)); + EN(:,i) = en_padded; + MaxX = x; + DOF = min(find(lambda <= 0)); + if isempty(DOF) + break; + end + lambda = [lambda(1:DOF-1); zeros(origDOF - DOF + 1, 1, 'single')]; + lambda = lambda(1:origDOF); + end + EN_all = sum(EN, 2); + + [u2, EigS2, v2] = nets_svds(data_vn', 0); + u2(isnan(u2)) = 0; + v2(isnan(v2)) = 0; + + grot = diag(EigS2); + grot_sq = grot.^2; + grot_scaled = (grot_sq ./ max(grot_sq)) .* max(lambda); + grot_adj = grot_scaled(1:length(EN_all)) - EN_all; + firstZero = min(find(grot_adj <= 0)); + if ~isempty(firstZero) + grot_adj(firstZero:end) = 0; + end + EigSAdj = zeros(length(grot), 1, 'single'); + EigSAdj(1:length(grot_adj)) = sqrt(max(grot_adj, 0)); + + filtered = (u2 * diag(EigSAdj) * v2')'; + filtered = filtered .* repmat(noise_unst_std, 1, Ntp); + + fullFiltered = zeros(size(allData), 'single'); + fullFiltered(mask, :) = filtered; + + startTP = 1; + for i = 1:length(outList) + endTP = startTP + tpCounts(i) - 1; + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + cii.cdata = fullFiltered(:, startTP:endTP); + ciftisave(cii, strtrim(outList{i}), 'wb_command'); + startTP = endTP + 1; + end +end + +%% ----------- Internal Helper Functions(From icaDim.m) ------------ %% + +function [out] = lpdist(in) + out=pdist(log(in)); +end + +function [x,EN] = FitWishart(lnb,S,DOF,MaxX,lambda) %FitWishart(lnb,step,DOF,MaxX,lambda) + EigDn1=round(DOF*lnb); %Isolate search to noise + EigDn2=round(DOF-1); %Reqd for post MR+FIX deconcatinated tcs + %EigDn2=round(DOF*0.75); %Reqd for post MR+FIX deconcatinated tcs + + a = DOF; %Lower bound for search range + b = MaxX; %Upper bound for search range + epsilon = 1; %Accuracy/stopping criterion + iter = 500; %# iterations/secondary stopping criterion + tau = double((sqrt(5)-1)/2); %Golden ratio (constant), 0.618... + k = 1; %Iteration count + + %Initial section ranges to instantiate optimization + x1 = a+(1-tau)*(b-a); + x2 = a+tau*(b-a); + %x1=a; + %x2=b; + + %Calculate initial null spectra + EN_x1 = iFeta([0:0.001:5],DOF,x1)'; %Call feta to calc null spectrum + %EN_x1=flipud(eig(cov(Smooth(randn(round(x1),DOF),S)))); + EN_x1=EN_x1*median(lambda(EigDn1:EigDn2)./EN_x1(EigDn1:EigDn2)); %Remove offset between null & data + f_x1 = lpdist([EN_x1(EigDn1:EigDn2)'; lambda(EigDn1:EigDn2)']); %Compute pairwise distance b/w null & data + + EN_x2 = iFeta([0:0.001:5],DOF,x2)'; + %EN_x2=flipud(eig(cov(Smooth(randn(round(x2),DOF),S)))); + EN_x2=EN_x2*median(lambda(EigDn1:EigDn2)./EN_x2(EigDn1:EigDn2)); + f_x2 = lpdist([EN_x2(EigDn1:EigDn2)'; lambda(EigDn1:EigDn2)']); + + disp(['golden search initial range: ' num2str(a) ' to ' num2str(b)]); + + while (abs(b-a)>epsilon) && (kepsilon) && (k Date: Thu, 7 May 2026 22:40:11 -0500 Subject: [PATCH 35/74] Add NumWishart parameter to PFM launcher (cherry picked from commit 7842a4e0749c48644d5d36a3b636aace03845b8c) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 10579b643..322ff1caa 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -138,6 +138,7 @@ main() { # set the start step beginning from RunPROFUMO which is by default the first step StartStep="RunPROFUMO" StopStep="GroupPFMs" + NumWishart="5" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' parLimit=-1 @@ -188,7 +189,7 @@ main() { VolumeTemplateCIFTI="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll7T175/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" # PROFUMO settings - ProfumoSingularity="/media/myelin/andrea/HCPpipelines/PFM/profumo_v2.sif" + ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" ProfumoConfig="${PFMFolder}/dataLocations.json" TR="1.0" ProfumoThreads="14" @@ -235,8 +236,8 @@ main() { --fmri-resolution="$fMRIResolution" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ - --low-dims="$LowDims" \ --fix-legacy-bias="$FixLegacyBiasString" \ + --num-wishart="$NumWishart"\ --scale-factor="$ScaleFactor" \ --starting-step="$StartStep" \ --stop-after-step="$StopStep" \ @@ -250,6 +251,7 @@ main() { --profumo-cov-model="$CovModel" \ --profumo-multi-start-iterations="$nStarts" \ --profumo-random-seed="$RandomSeed" \ + --num-wishart="$NumWishart" \ --ref-image="$RefImage" \ --volume-template-file="$VolumeTemplateCIFTI" From 9b9618fc113f390db96c5dec610a9de2cedb4ca9 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 7 May 2026 22:58:18 -0500 Subject: [PATCH 36/74] Fix matrix dimension issue in TCS calculation (cherry picked from commit 82cf3926d5a35388d0cb78b0705d4e8d4c44aa9e) --- PFM/scripts/PostPROFUMO.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 2845bcb85..b61d3d909 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -64,7 +64,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); origTCS = [origTCS ; runTCS]; - TCS=[TCS ; runTCS.*repmat(runAmp',length(runTCS),1)]; + TCS = [TCS ; runTCS.*repmat(runAmp',size(runTCS),1)]; end % for r = 1:numel(subfMRINames) %% Create original time course and spectral CIFTI files @@ -102,4 +102,4 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) -end \ No newline at end of file +end From bb787d6826427fb6f4e74a8212af3f130757f698 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 7 May 2026 23:01:40 -0500 Subject: [PATCH 37/74] Update VolumeTemplateCIFTI path to use HOME directory (cherry picked from commit ba11f0c0c50adffd5dab2fe4b492c700f9a8e542) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 322ff1caa..e2cc086db 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -186,7 +186,7 @@ main() { LowDims="7@8@9@10@11@12@13@14@15@16@17@18@19@20@21" # Volume template file - VolumeTemplateCIFTI="/media/myelin/brainmappers/Connectome_Project/YA_HCP_Final/S1200_MSMAll7T175/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" + VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" # PROFUMO settings ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" From 73efef6fbf9a22ba5c1ba8e8b309bf3cd99fb58e Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 7 May 2026 23:07:39 -0500 Subject: [PATCH 38/74] Remove LowDims variable from script Removed LowDims variable from RSN regression settings. (cherry picked from commit caeb87c0bd485b6a2183284e5f357e049d418553) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index e2cc086db..10ec62c2e 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -183,7 +183,6 @@ main() { # RSN regression settings FixLegacyBiasString="NO" ScaleFactor="0.01" - LowDims="7@8@9@10@11@12@13@14@15@16@17@18@19@20@21" # Volume template file VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" From 1d5ad7c214136368f050463eba6ed12595c59253 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 7 May 2026 23:16:56 -0500 Subject: [PATCH 39/74] Cleaning up and add comments (cherry picked from commit 77542624cf88f48739d07f4faebaec38d1f9c291) --- PFM/PFMPipeline.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 892f09179..13f462054 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -121,8 +121,6 @@ then RegString="_$RegName" fi -# Volume template file path -# VolumeTemplateFile="${StudyFolder}/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.2.dscalar.nii" for ((stepInd = startInd; stepInd <= stopAfterInd; ++stepInd)) do @@ -146,6 +144,7 @@ do fi ProfumoConfigToUse="${ProfumoConfig}" + # Applying WF if [[ "$NumWishart" -gt 0 ]] then log_Msg "Running Wishart filtering with ${NumWishart} iterations before PROFUMO" @@ -174,7 +173,7 @@ do --pfm-dimension="$PFMdim" \ --matlab-run-mode="$MatlabMode" done - + # Create config file(location of files) for WF files ProfumoConfigToUse="${WFTempDir}/wishart_dataLocations.json" echo '{' > "$ProfumoConfigToUse" for Subject in "${Subjlist[@]}" @@ -190,11 +189,11 @@ do done perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" - echo -e "}," >> "$ProfumoConfigToUse" + echo -e "\t}," >> "$ProfumoConfigToUse" done perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" echo "}" >> "$ProfumoConfigToUse" - log_Msg "WF prefiltering complete" + log_Msg "WF complete" fi # Set up PROFUMO paths @@ -274,7 +273,8 @@ do "${PFM_PATH}" \ "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" - + + #Cleaning up temporary WF files if [[ "${NumWishart}" -gt 0 ]] then log_Msg "Cleaning up temporary Wishart filtered files" @@ -375,4 +375,4 @@ do ;; esac log_Msg "step $stepName complete" -done \ No newline at end of file +done From cead749aff172eb3ac1754ede99f3d66e334c97b Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 7 May 2026 23:19:07 -0500 Subject: [PATCH 40/74] Clean temp WF files before PROFUMO postprocessing Removed redundant cleanup of temporary Wishart filtered files from the PROFUMO step. (cherry picked from commit fc2629d707b7bc8aeeac73109f0abc3bd7d1530c) --- PFM/PFMPipeline.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 13f462054..50e1d7f5f 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -256,6 +256,14 @@ do --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} + + #Cleanup temp WF files + if [[ "${NumWishart}" -gt 0 ]] + then + log_Msg "Cleaning up temporary Wishart filtered files" + rm -rf "${WFTempDir}" + fi + log_Msg "Running PROFUMO postprocessing" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ @@ -274,12 +282,7 @@ do "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" - #Cleaning up temporary WF files - if [[ "${NumWishart}" -gt 0 ]] - then - log_Msg "Cleaning up temporary Wishart filtered files" - rm -rf "${WFTempDir}" - fi + ;; (PostPROFUMO) log_Msg "Running PostPROFUMO step" From 4f8df646c539db78fc987378eff796f9b4e7dbfa Mon Sep 17 00:00:00 2001 From: arkky68 Date: Mon, 11 May 2026 14:44:38 -0500 Subject: [PATCH 41/74] Address review( move postprocessing to PostPROFUMO, fix descriptions, use RegString, fix size(runTCS,1)) --- Examples/Scripts/RunPFMpipelineFullRun.sh | 3 +-- PFM/PFMPipeline.sh | 22 ++++++++++++---------- PFM/scripts/PostPROFUMO.m | 2 +- PFM/scripts/WishartFilter.sh | 10 +++++----- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 10ec62c2e..947ae0c5e 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -190,7 +190,7 @@ main() { # PROFUMO settings ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="1.0" + TR="0.72" ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" @@ -232,7 +232,6 @@ main() { --pfm-folder="$PFMFolder" \ --surf-reg-name="$RegName" \ --concat-name="$ConcatName" \ - --fmri-resolution="$fMRIResolution" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ --fix-legacy-bias="$FixLegacyBiasString" \ diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 50e1d7f5f..98f412e19 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -42,7 +42,6 @@ opts_AddMandatory '--ref-image' 'RefImage' 'path' "reference image for PROFUMO p opts_AddMandatory '--runs-timepoints' 'RunsXNumTimePoints' "total timepoints across runs" "total timepoints across runs" opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using multi-run data" '' opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' -opts_AddMandatory '--fmri-resolution' 'fMRIResolution' "fMRI resolution string for template selection, like '2','1.60' or '2.40'" '' #PROFUMO specific parameters opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threads for PROFUMO" '25' @@ -156,8 +155,8 @@ do outputList="" for fMRIName in "${fMRINamesArray[@]}" do - inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" - outputFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}_WF.dtseries.nii" + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + outputFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$inputFile" ]] then if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi @@ -181,7 +180,7 @@ do echo -e "\t\"$Subject\": {" >> "$ProfumoConfigToUse" for fMRIName in "${fMRINamesArray[@]}" do - WFFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}_WF.dtseries.nii" + WFFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$WFFile" ]] then echo -e "\t\t\"$fMRIName\": \"$WFFile\"," >> "$ProfumoConfigToUse" @@ -262,9 +261,15 @@ do then log_Msg "Cleaning up temporary Wishart filtered files" rm -rf "${WFTempDir}" - fi + fi + ;; + (PostPROFUMO) log_Msg "Running PROFUMO postprocessing" + PFM_PATH="${PFMFolder}/Analysis.pfm" + RESULTS_PATH="${PFMFolder}/Results.ppp" + REAL_REF_IMAGE=$(readlink -f "${RefImage}") + echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ @@ -282,16 +287,13 @@ do "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" - - ;; - (PostPROFUMO) log_Msg "Running PostPROFUMO step" "$HCPPIPEDIR"/PFM/scripts/PostPROFUMO.sh \ --study-folder="$StudyFolder" \ --subject-list="$SubjlistRaw" \ --fmri-names="$fMRINames" \ --concat-name="$ConcatName" \ - --proc-string="_Atlas_${RegName}_$fMRIProcSTRING" \ + --proc-string="_Atlas${RegString}_${fMRIProcSTRING}" \ --output-fmri-name="$OutputfMRIName" \ --output-string="$OutputSTRING" \ --surf-reg-name="$RegName" \ @@ -314,7 +316,7 @@ do fMRINamesForSub="" for fMRIName in "${fMRINamesArray[@]}" do - if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" ]] + if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" ]] then if [[ "$fMRINamesForSub" != "" ]] then diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index b61d3d909..5c5ccdd58 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -64,7 +64,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS.*repmat(runAmp',size(runTCS),1)]; + TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; end % for r = 1:numel(subfMRINames) %% Create original time course and spectral CIFTI files diff --git a/PFM/scripts/WishartFilter.sh b/PFM/scripts/WishartFilter.sh index 621f07567..ffbe758ea 100755 --- a/PFM/scripts/WishartFilter.sh +++ b/PFM/scripts/WishartFilter.sh @@ -12,12 +12,12 @@ source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" g_matlab_default_mode=1 -opts_SetScriptDescription "applies Wishart filter to a single CIFTI file using icaDim" +opts_SetScriptDescription "applies Wishart filter to CIFTI dtseries files for PROFUMO" -opts_AddMandatory '--input' 'inputFile' 'file' "the input dtseries file" -opts_AddMandatory '--output' 'outputFile' 'file' "the output wishart-filtered dtseries file" -opts_AddMandatory '--num-wishart' 'numWisharts' 'integer' "how many wisharts to use in icaDim" -opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "the PFM dimension to use for the low-dim template in icaDim" +opts_AddMandatory '--input' 'inputFile' 'file' "comma-separated list of input dtseries files" +opts_AddMandatory '--output' 'outputFile' 'file' "comma-separated list of output wishart-filtered dtseries files" +opts_AddMandatory '--num-wishart' 'numWisharts' 'integer' "number of Wishart distributions to fit" +opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode 0 = compiled MATLAB From 64b0d78ecf0201138e06711b5a27fea0f35392d5 Mon Sep 17 00:00:00 2001 From: arkky68 Date: Sat, 23 May 2026 23:38:42 -0500 Subject: [PATCH 42/74] Update with ApplyWFProfumo scripts --- Examples/Scripts/RunPFMpipelineFullRun.sh | 2 + PFM/PFMPipeline.sh | 90 ++++-- PFM/scripts/ApplyWFProfumo.m | 24 ++ .../{WishartFilter.sh => ApplyWFProfumo.sh} | 5 +- PFM/scripts/WishartFilter.m | 281 ------------------ 5 files changed, 85 insertions(+), 317 deletions(-) create mode 100644 PFM/scripts/ApplyWFProfumo.m rename PFM/scripts/{WishartFilter.sh => ApplyWFProfumo.sh} (92%) delete mode 100644 PFM/scripts/WishartFilter.m diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 947ae0c5e..2015a6bb1 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -139,6 +139,7 @@ main() { StartStep="RunPROFUMO" StopStep="GroupPFMs" NumWishart="5" + KeepWishartFiles="YES" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' parLimit=-1 @@ -244,6 +245,7 @@ main() { --profumo-config="$ProfumoConfig" \ --profumo-singularity="$ProfumoSingularity" \ --profumo-tr="$TR" \ + --keep-wishart-files="$KeepWishartFiles" \ --profumo-threads="$ProfumoThreads" \ --profumo-dof-correction="$DOFCorrection" \ --profumo-cov-model="$CovModel" \ diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 98f412e19..22ab53c93 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -53,6 +53,7 @@ opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'inte opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filter iterations for prefiltering (0 to skip)" '0' +opts_AddOptional '--keep-wishart-files' 'KeepWishartFiles' 'YES or NO' "keep Wishart-filtered files after PROFUMO instead of deleting (default NO)" 'NO' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -143,51 +144,70 @@ do fi ProfumoConfigToUse="${ProfumoConfig}" - # Applying WF if [[ "$NumWishart" -gt 0 ]] then - log_Msg "Running Wishart filtering with ${NumWishart} iterations before PROFUMO" - WFTempDir=$(mktemp -d "${PFMFolder}/WF_tmp.XXXXXX") + WFDir="${PFMFolder}/WishartFilter_WF${NumWishart}" + # Check if WF files already exist + wfComplete=true for Subject in "${Subjlist[@]}" do - mkdir -p "${WFTempDir}/${Subject}" - inputList="" - outputList="" for fMRIName in "${fMRINamesArray[@]}" do inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - outputFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - if [[ -f "$inputFile" ]] + wfFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] && [[ ! -f "$wfFile" ]] then - if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi - inputList+="$inputFile" - outputList+="$outputFile" + wfComplete=false + break 2 fi done - log_Msg "Applying Wishart filter for subject $Subject" - "$HCPPIPEDIR"/PFM/scripts/WishartFilter.sh \ - --input="$inputList" \ - --output="$outputList" \ - --num-wishart="$NumWishart" \ - --pfm-dimension="$PFMdim" \ - --matlab-run-mode="$MatlabMode" done - # Create config file(location of files) for WF files - ProfumoConfigToUse="${WFTempDir}/wishart_dataLocations.json" + + if $wfComplete + then + log_Msg "WF files already exist in ${WFDir}" + else + log_Msg "Running Wishart filtering with ${NumWishart} iterations" + for Subject in "${Subjlist[@]}" + do + mkdir -p "${WFDir}/${Subject}" + inputList="" + outputList="" + for fMRIName in "${fMRINamesArray[@]}" + do + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] + then + if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi + inputList+="$inputFile" + outputList+="$outputFile" + fi + done + log_Msg "Applying Wishart filter for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ + --input="$inputList" \ + --output="$outputList" \ + --num-wishart="$NumWishart" \ + --matlab-run-mode="$MatlabMode" + done + fi + + # Build JSON pointing at WF files + ProfumoConfigToUse="${WFDir}/wishart_dataLocations.json" echo '{' > "$ProfumoConfigToUse" for Subject in "${Subjlist[@]}" do echo -e "\t\"$Subject\": {" >> "$ProfumoConfigToUse" for fMRIName in "${fMRINamesArray[@]}" do - WFFile="${WFTempDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + WFFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$WFFile" ]] then echo -e "\t\t\"$fMRIName\": \"$WFFile\"," >> "$ProfumoConfigToUse" fi done - - perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" + perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" echo -e "\t}," >> "$ProfumoConfigToUse" done perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" @@ -207,10 +227,9 @@ do if [[ -d "${PFMFolder}" ]] then log_Warn "PFM output folder ${PFMFolder} already exists, clearing contents" - find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -delete + find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -not -path "*/WishartFilter_WF*" -delete fi - mkdir -p "${PFMFolder}" - + # Build optional initialMaps argument InitialMapsArg="" if [[ -n "${InitialMaps}" && -f "${InitialMaps}" ]] @@ -226,8 +245,7 @@ do LoadSequentiallyArg="--loadSequentially" fi - ## Rewrite input files with wb_command, as a failsafe to avoid SIMD buffer alignment issue in PROFUMO if - ## files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. + # files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. if [[ "${NumWishart}" -eq 0 ]] then cat "${ProfumoConfig}" | \ @@ -256,12 +274,18 @@ do --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} - #Cleanup temp WF files - if [[ "${NumWishart}" -gt 0 ]] + #Cleanup WF files + if [[ "$NumWishart" -gt 0 ]] then - log_Msg "Cleaning up temporary Wishart filtered files" - rm -rf "${WFTempDir}" - fi + KeepWishartBool=$(opts_StringToBool "$KeepWishartFiles") + if ((KeepWishartBool)) + then + log_Msg "Keeping Wishart filtered files in ${WFDir}" + else + log_Msg "Cleaning up Wishart filtered files" + rm -rf "${WFDir}" + fi + fi ;; (PostPROFUMO) diff --git a/PFM/scripts/ApplyWFProfumo.m b/PFM/scripts/ApplyWFProfumo.m new file mode 100644 index 000000000..9fbc1533f --- /dev/null +++ b/PFM/scripts/ApplyWFProfumo.m @@ -0,0 +1,24 @@ +function ApplyWishartFilterProfumo(inputFiles, outputFiles, numWisharts) + numWisharts = str2double(numWisharts); + inList = strsplit(inputFiles, ','); + outList = strsplit(outputFiles, ','); + + allData = []; + tpCounts = []; + for i = 1:length(inList) + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + tpCounts(i) = size(cii.cdata, 2); + allData = [allData cii.cdata]; + end + + Out = icaDim(allData, 0, 1, -1, numWisharts); + + startTP = 1; + for i = 1:length(outList) + endTP = startTP + tpCounts(i) - 1; + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + cii.cdata = Out.data(:, startTP:endTP); + ciftisave(cii, strtrim(outList{i}), 'wb_command'); + startTP = endTP + 1; + end +end \ No newline at end of file diff --git a/PFM/scripts/WishartFilter.sh b/PFM/scripts/ApplyWFProfumo.sh similarity index 92% rename from PFM/scripts/WishartFilter.sh rename to PFM/scripts/ApplyWFProfumo.sh index ffbe758ea..8b860f221 100755 --- a/PFM/scripts/WishartFilter.sh +++ b/PFM/scripts/ApplyWFProfumo.sh @@ -17,7 +17,6 @@ opts_SetScriptDescription "applies Wishart filter to CIFTI dtseries files for PR opts_AddMandatory '--input' 'inputFile' 'file' "comma-separated list of input dtseries files" opts_AddMandatory '--output' 'outputFile' 'file' "comma-separated list of output wishart-filtered dtseries files" opts_AddMandatory '--num-wishart' 'numWisharts' 'integer' "number of Wishart distributions to fit" -opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode 0 = compiled MATLAB @@ -53,7 +52,7 @@ esac this_script_dir=$(dirname "$0") -matlab_argarray=("$inputFile" "$outputFile" "$numWisharts" "$PFMdim") +matlab_argarray=("$inputFile" "$outputFile" "$numWisharts") case "$MatlabMode" in (0) @@ -76,7 +75,7 @@ case "$MatlabMode" in addpath('$HCPPIPEDIR/global/matlab'); addpath('$this_script_dir'); addpath('$HCPCIFTIRWDIR'); - WishartFilter($matlab_args);" + ApplyWFProfumo($matlab_args);" log_Msg "running matlab code: $matlabcode" "${matlab_interpreter[@]}" <<<"$matlabcode" diff --git a/PFM/scripts/WishartFilter.m b/PFM/scripts/WishartFilter.m deleted file mode 100644 index ddc890fae..000000000 --- a/PFM/scripts/WishartFilter.m +++ /dev/null @@ -1,281 +0,0 @@ -function WishartFilter(inputFiles, outputFiles, numWisharts, pfmDim) - numWisharts = str2double(numWisharts); - pfmDim = str2double(pfmDim); - inList = strsplit(inputFiles, ','); - outList = strsplit(outputFiles, ','); - - allData = []; - tpCounts = []; - for i = 1:length(inList) - cii = ciftiopen(strtrim(inList{i}), 'wb_command'); - tpCounts(i) = size(cii.cdata, 2); - allData = [allData cii.cdata]; - end - - Ntp = size(allData, 2); - - mask = range(allData, 2) > 0; - data = allData(mask, :); - - [u, EigS, v] = nets_svds(data', 0); - DOF = sum(diag(EigS) > (std(diag(EigS)) * 0.1)); - u(isnan(u)) = 0; - v(isnan(v)) = 0; - - noise_unst = (u(:, pfmDim:DOF) * EigS(pfmDim:DOF, pfmDim:DOF) * v(:, pfmDim:DOF)')'; - noise_unst_std = max(std(noise_unst, [], 2), 0.001); - clear noise_unst; - data_vn = data ./ repmat(noise_unst_std, 1, Ntp); - - lambda = flipud(eig(cov(data_vn))); - - lnb = 0.5; - MaxX = size(data_vn, 1); - origDOF = DOF; - clear EN - for i = 1:numWisharts - [x, en] = FitWishart(lnb, 0, DOF, MaxX, lambda(1:DOF)); - lambda = lambda(1:DOF) - en(1:DOF); - en_padded = zeros(origDOF, 1, 'single'); - en_padded(1:min(length(en), origDOF)) = en(1:min(length(en), origDOF)); - EN(:,i) = en_padded; - MaxX = x; - DOF = min(find(lambda <= 0)); - if isempty(DOF) - break; - end - lambda = [lambda(1:DOF-1); zeros(origDOF - DOF + 1, 1, 'single')]; - lambda = lambda(1:origDOF); - end - EN_all = sum(EN, 2); - - [u2, EigS2, v2] = nets_svds(data_vn', 0); - u2(isnan(u2)) = 0; - v2(isnan(v2)) = 0; - - grot = diag(EigS2); - grot_sq = grot.^2; - grot_scaled = (grot_sq ./ max(grot_sq)) .* max(lambda); - grot_adj = grot_scaled(1:length(EN_all)) - EN_all; - firstZero = min(find(grot_adj <= 0)); - if ~isempty(firstZero) - grot_adj(firstZero:end) = 0; - end - EigSAdj = zeros(length(grot), 1, 'single'); - EigSAdj(1:length(grot_adj)) = sqrt(max(grot_adj, 0)); - - filtered = (u2 * diag(EigSAdj) * v2')'; - filtered = filtered .* repmat(noise_unst_std, 1, Ntp); - - fullFiltered = zeros(size(allData), 'single'); - fullFiltered(mask, :) = filtered; - - startTP = 1; - for i = 1:length(outList) - endTP = startTP + tpCounts(i) - 1; - cii = ciftiopen(strtrim(inList{i}), 'wb_command'); - cii.cdata = fullFiltered(:, startTP:endTP); - ciftisave(cii, strtrim(outList{i}), 'wb_command'); - startTP = endTP + 1; - end -end - -%% ----------- Internal Helper Functions(From icaDim.m) ------------ %% - -function [out] = lpdist(in) - out=pdist(log(in)); -end - -function [x,EN] = FitWishart(lnb,S,DOF,MaxX,lambda) %FitWishart(lnb,step,DOF,MaxX,lambda) - EigDn1=round(DOF*lnb); %Isolate search to noise - EigDn2=round(DOF-1); %Reqd for post MR+FIX deconcatinated tcs - %EigDn2=round(DOF*0.75); %Reqd for post MR+FIX deconcatinated tcs - - a = DOF; %Lower bound for search range - b = MaxX; %Upper bound for search range - epsilon = 1; %Accuracy/stopping criterion - iter = 500; %# iterations/secondary stopping criterion - tau = double((sqrt(5)-1)/2); %Golden ratio (constant), 0.618... - k = 1; %Iteration count - - %Initial section ranges to instantiate optimization - x1 = a+(1-tau)*(b-a); - x2 = a+tau*(b-a); - %x1=a; - %x2=b; - - %Calculate initial null spectra - EN_x1 = iFeta([0:0.001:5],DOF,x1)'; %Call feta to calc null spectrum - %EN_x1=flipud(eig(cov(Smooth(randn(round(x1),DOF),S)))); - EN_x1=EN_x1*median(lambda(EigDn1:EigDn2)./EN_x1(EigDn1:EigDn2)); %Remove offset between null & data - f_x1 = lpdist([EN_x1(EigDn1:EigDn2)'; lambda(EigDn1:EigDn2)']); %Compute pairwise distance b/w null & data - - EN_x2 = iFeta([0:0.001:5],DOF,x2)'; - %EN_x2=flipud(eig(cov(Smooth(randn(round(x2),DOF),S)))); - EN_x2=EN_x2*median(lambda(EigDn1:EigDn2)./EN_x2(EigDn1:EigDn2)); - f_x2 = lpdist([EN_x2(EigDn1:EigDn2)'; lambda(EigDn1:EigDn2)']); - - disp(['golden search initial range: ' num2str(a) ' to ' num2str(b)]); - - while (abs(b-a)>epsilon) && (kepsilon) && (k Date: Mon, 8 Jun 2026 07:20:56 -0500 Subject: [PATCH 43/74] Address review: use concat file for WF, add per-run demeaning, fix RegString, remove duplicates --- Examples/Scripts/RunPFMpipelineFullRun.sh | 9 ++- PFM/PFMPipeline.sh | 73 +++++++++++++++++------ PFM/scripts/ApplyWFProfumo.m | 6 +- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 2015a6bb1..3b2975796 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -105,7 +105,7 @@ get_options() { echo " GroupAverageName: ${GroupAverageName}" echo " RegName: ${RegName}" echo " MatlabMode: ${MatlabMode}" - echo "-- ${scriptName}: Specified Command-Line Options: -- End --" + echo "-- ${scriptName}: Specified Command-Line Options: -- End --"ß } @@ -139,7 +139,7 @@ main() { StartStep="RunPROFUMO" StopStep="GroupPFMs" NumWishart="5" - KeepWishartFiles="YES" + KeepWishartFiles="NO" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' parLimit=-1 @@ -205,7 +205,7 @@ main() { for Subject in $(echo $Subjlist | tr "@" "\n"); do echo -e "\t\"$Subject\": {" >> $ProfumoConfig for fMRIName in $(echo $fMRINames | tr "@" "\n"); do - runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}_${fMRIProcSTRING}.dtseries.nii" + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" if [[ -e $runFile ]]; then echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig fi @@ -217,7 +217,7 @@ main() { echo "}" >> $ProfumoConfig # PFM pipeline execution - echo "Starting PFM postprocessing pipeline..." + echo "Starting PFM postprocessing pipeline" echo "Data type: ${OutputfMRIName}" echo "PFM dimension: ${PFMdim}" @@ -251,7 +251,6 @@ main() { --profumo-cov-model="$CovModel" \ --profumo-multi-start-iterations="$nStarts" \ --profumo-random-seed="$RandomSeed" \ - --num-wishart="$NumWishart" \ --ref-image="$RefImage" \ --volume-template-file="$VolumeTemplateCIFTI" diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 22ab53c93..a72f7d296 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -88,7 +88,7 @@ IFS='@' read -a Subjlist <<<"$SubjlistRaw" IFS='@' read -a fMRINamesArray <<<"$fMRINames" FixLegacyBiasBool=$(opts_StringToBool "$FixLegacyBias") - +KeepWishartBool=$(opts_StringToBool "$KeepWishartFiles") if ! [[ "$parLimit" == "-1" || "$parLimit" =~ [1-9][0-9]* ]] then log_Err_Abort "--parallel-limit must be a positive integer or -1, provided value: '$parLimit'" @@ -171,25 +171,61 @@ do for Subject in "${Subjlist[@]}" do mkdir -p "${WFDir}/${Subject}" - inputList="" - outputList="" - for fMRIName in "${fMRINamesArray[@]}" - do - inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - if [[ -f "$inputFile" ]] + + if [[ "$ConcatName" != "" ]] + then + # Use already concatenated file + concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + + if [[ -f "$concatFile" ]] then - if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi - inputList+="$inputFile" - outputList+="$outputFile" + log_Msg "Applying Wishart filter to concat file for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + --input="$concatFile" \ + --output="$concatOutFile" \ + --num-wishart="$NumWishart" \ + --matlab-run-mode="$MatlabMode" + + # Split back into individual runs + cumTP=0 + for fMRIName in "${fMRINamesArray[@]}"ßß + do + origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + if [[ -f "$origFile" ]] + then + nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) + startIdx=$((cumTP + 1)) + endIdx=$((cumTP + nTP)) + outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + wb_command -cifti-merge "$outFile" -direction ROW \ + -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + cumTP=$endIdx + fi + done fi - done - log_Msg "Applying Wishart filter for subject $Subject" - "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ - --input="$inputList" \ - --output="$outputList" \ - --num-wishart="$NumWishart" \ - --matlab-run-mode="$MatlabMode" + else + # No concat file then pass individual runs + inputList="" + outputList="" + for fMRIName in "${fMRINamesArray[@]}" + do + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] + then + if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi + inputList+="$inputFile" + outputList+="$outputFile" + fi + done + log_Msg "Applying Wishart filter for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + --input="$inputList" \ + --output="$outputList" \ + --num-wishart="$NumWishart" \ + --matlab-run-mode="$MatlabMode" + fi done fi @@ -277,7 +313,6 @@ do #Cleanup WF files if [[ "$NumWishart" -gt 0 ]] then - KeepWishartBool=$(opts_StringToBool "$KeepWishartFiles") if ((KeepWishartBool)) then log_Msg "Keeping Wishart filtered files in ${WFDir}" diff --git a/PFM/scripts/ApplyWFProfumo.m b/PFM/scripts/ApplyWFProfumo.m index 9fbc1533f..909fd9e79 100644 --- a/PFM/scripts/ApplyWFProfumo.m +++ b/PFM/scripts/ApplyWFProfumo.m @@ -5,10 +5,12 @@ function ApplyWishartFilterProfumo(inputFiles, outputFiles, numWisharts) allData = []; tpCounts = []; + runMeans = {}; for i = 1:length(inList) cii = ciftiopen(strtrim(inList{i}), 'wb_command'); tpCounts(i) = size(cii.cdata, 2); - allData = [allData cii.cdata]; + runMeans{i} = mean(cii.cdata, 2); + allData = [allData (cii.cdata - runMeans{i})]; end Out = icaDim(allData, 0, 1, -1, numWisharts); @@ -17,7 +19,7 @@ function ApplyWishartFilterProfumo(inputFiles, outputFiles, numWisharts) for i = 1:length(outList) endTP = startTP + tpCounts(i) - 1; cii = ciftiopen(strtrim(inList{i}), 'wb_command'); - cii.cdata = Out.data(:, startTP:endTP); + cii.cdata = Out.data(:, startTP:endTP) + runMeans{i}; ciftisave(cii, strtrim(outList{i}), 'wb_command'); startTP = endTP + 1; end From c377ddb04dba9ed2629e157957d2cd4a2464c2b8 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Mon, 8 Jun 2026 20:24:08 +0800 Subject: [PATCH 44/74] Correct Typo --- Examples/Scripts/RunPFMpipelineFullRun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 3b2975796..61fb4b30b 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -105,7 +105,7 @@ get_options() { echo " GroupAverageName: ${GroupAverageName}" echo " RegName: ${RegName}" echo " MatlabMode: ${MatlabMode}" - echo "-- ${scriptName}: Specified Command-Line Options: -- End --"ß + echo "-- ${scriptName}: Specified Command-Line Options: -- End --" } From 1c07db55c26222143e9275720956178d2bc0e62a Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 7 Jul 2026 09:18:11 -0500 Subject: [PATCH 45/74] Make PRUFUMO container ignore the local python environmnet. Clean up exisiting results dirs in PostPROFUMO step. Make both RunPROFUMO and PostPROFUMO existing data clears more robust. --- PFM/PFMPipeline.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index a72f7d296..dd9521cdc 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -181,7 +181,7 @@ do if [[ -f "$concatFile" ]] then log_Msg "Applying Wishart filter to concat file for subject $Subject" - "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ --input="$concatFile" \ --output="$concatOutFile" \ --num-wishart="$NumWishart" \ @@ -189,7 +189,7 @@ do # Split back into individual runs cumTP=0 - for fMRIName in "${fMRINamesArray[@]}"ßß + for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" if [[ -f "$origFile" ]] @@ -263,7 +263,8 @@ do if [[ -d "${PFMFolder}" ]] then log_Warn "PFM output folder ${PFMFolder} already exists, clearing contents" - find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -not -path "*/WishartFilter_WF*" -delete + find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -not -name ".*" -not -path "*/WishartFilter_WF*" -delete 2>/dev/null || true + # ignore errors due to nfs silly renamed files, or similar fi # Build optional initialMaps argument @@ -295,6 +296,7 @@ do # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ + --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ @@ -303,6 +305,7 @@ do --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ + --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ @@ -329,8 +332,17 @@ do RESULTS_PATH="${PFMFolder}/Results.ppp" REAL_REF_IMAGE=$(readlink -f "${RefImage}") + # Remove any existing Results.ppp directory (and ++ variants created by re-runs) + # so postprocess_results.py writes fresh output to Results.ppp + if [[ -d "${PFMFolder}/Results.ppp" ]] + then + log_Warn "Results.ppp folder ${PFMFolder}/Results.ppp(+) already exists, clearing before postprocessing" + rm -rf "${PFMFolder}"/Results.ppp* 2>/dev/null || true # ignore errors due to nfs silly renamed files, or similar + fi + echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ + --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ /opt/fsl/fslpython/envs/profumo/bin/python3 /opt/profumo/Python/postprocess_results.py \ --web-report \ @@ -339,6 +351,7 @@ do "${REAL_REF_IMAGE}" apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ + --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ /opt/fsl/fslpython/envs/profumo/bin/python3 /opt/profumo/Python/postprocess_results.py \ --web-report \ From ce5480e41effea8a741b48c53823e655042cb994 Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Thu, 9 Jul 2026 21:26:57 +0800 Subject: [PATCH 46/74] Pfm Update with human subjects (#392) PFM pipeline development --------- Co-authored-by: Burke Rosen --- Examples/Scripts/RunPFMpipelineFullRun.sh | 89 +++++----- PFM/.gitignore | 1 + PFM/PFMPipeline.sh | 207 ++++++++++++++++++---- PFM/scripts/ApplyWFProfumo.m | 26 +++ PFM/scripts/ApplyWFProfumo.sh | 84 +++++++++ PFM/scripts/PostPROFUMO.m | 4 +- 6 files changed, 326 insertions(+), 85 deletions(-) create mode 100644 PFM/.gitignore create mode 100644 PFM/scripts/ApplyWFProfumo.m create mode 100755 PFM/scripts/ApplyWFProfumo.sh diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index cd4f4c424..61fb4b30b 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -17,14 +17,15 @@ get_options() { local arguments=("$@") # initialize variables - StudyFolder="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus" - Subjlist="$(find ${StudyFolder} -maxdepth 1 -type d -name "A*" -exec basename {} \; | paste -sd@ -)" - EnvironmentScript="/media/myelin/burke/projects/Mac25Rhesus/scripts/Mac25Rhesus_v5_SetUpHCPPipeline.sh" - GroupAverageName="Mac25Rhesus_v5" - RegName="" + StudyFolder="${HOME}/data/HCPpipelines_ExampleData" + Subjlist="100610@102311" + EnvironmentScript="${HOME}/projects/HCPpipelines/Examples/Scripts/SetUpHCPPipeline.sh" + GroupAverageName="S1200_MSMAll7T175" + + RegName="MSMAll" MatlabMode=1 RunLocal=0 - QUEUE="matlabparallelhigh.q@brainmappers-desktop3" + QUEUE="matlabparallelhigh.q" # parse arguments local index argument @@ -135,78 +136,76 @@ main() { # general settings # set the start step beginning from RunPROFUMO which is by default the first step - # StartStep="RunPROFUMO" - # StopStep="RunPROFUMO" - # StartStep="PostPROFUMO" - # StopStep="PostPROFUMO" - # StartStep="PostPROFUMO" - # StartStep="RSNRegression" - # StopStep="RSNRegression" - # StartStep="RunPROFUMO" - # StopStep="" - - StartStep="PostPROFUMO" + StartStep="RunPROFUMO" StopStep="GroupPFMs" + NumWishart="5" + KeepWishartFiles="NO" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' - parLimit=5 + parLimit=-1 # general inputs - fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR@BOLD_REST_3_AP@BOLD_REST_4_PA" - # fMRINames="BOLD_REST_1_RL@BOLD_REST_2_LR" - randSeed=1 # random seed for PROFUMO + fMRINames="rfMRI_REST1_LR@rfMRI_REST1_RL@rfMRI_REST2_LR@rfMRI_REST2_RL" + + randSeed=123 # random seed for PROFUMO - OutputfMRIName="BOLD_REST_CONCAT" + OutputfMRIName="rfMRI_REST" # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs - ConcatName="BOLD_REST_CONCAT" - # set the output spectra size for individual projection, RunsXNumTimePoints - subjectExpectedTimepoints="8508" + ConcatName="" + + # set the output spectra size for individual projection, RunsXNumTimePoints #subjectExpectedTimepoints="3655" + subjectExpectedTimepoints="4800" + # set temporal highpass full-width (2*sigma) used in preprocessing - HighPass="pd2" + HighPass="2000" + #set fMRIResolution of data, like '2','1.60' or '2.40' + fMRIResolution="2.0" # PFM settings for REST data - PFMdim="16" # set the PFM dimensionality - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${GroupAverageName}_${OutputfMRIName}_PFM_d${PFMdim}_s${randSeed}_test - # Reference image for PROFUMO - RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP/Mac25Rhesus_v5_BOLD_REST_CONCAT_MIGP_Atlas_hppd2_clean_meanvn.dscalar.nii" + # set the PFM dimensionality + PFMdim="99" + PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} + # Reference image for PROFUMO + RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_rclean_tclean_meanvn.dscalar.nii" + # set the file name component representing the preprocessing already done - fMRIProcSTRING="hp${HighPass}_clean" + fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" # set the mesh resolution, like '32' for 32k_fs_LR - LowResMesh="10" + LowResMesh="32" + # Define OutputSTRING with seed designation OutputSTRING="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs" # RSN regression settings - LowDims="6" FixLegacyBiasString="NO" ScaleFactor="0.01" # Volume template file - VolumeTemplateCIFTI="/media/myelin/brainmappers/BICAN/Macaque/MacaqueRhesus/Mac25Rhesus_v5/MNINonLinear/Results/Mac25Rhesus_v5_VolMaps_16_template.dscalar.nii" - + VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" + # PROFUMO settings - ProfumoSingularity="/media/myelin/burke/projects/Mac25Rhesus/HCPpipelines/PFM/profumo_v2.sif" + ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="0.702" + TR="0.72" ProfumoThreads="14" DOFCorrection="0.5" CovModel="Subject" - nStarts="1" # number of multi-start iterations for PROFUMO + nStarts="5" # number of multi-start iterations for PROFUMO RandomSeed="$randSeed" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below - + # build Profumo data location json mkdir -p $PFMFolder echo '{' > $ProfumoConfig for Subject in $(echo $Subjlist | tr "@" "\n"); do echo -e "\t\"$Subject\": {" >> $ProfumoConfig for fMRIName in $(echo $fMRINames | tr "@" "\n"); do - runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${fMRIProcSTRING}.dtseries.nii" + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" if [[ -e $runFile ]]; then echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig fi @@ -218,7 +217,7 @@ main() { echo "}" >> $ProfumoConfig # PFM pipeline execution - echo "Starting PFM postprocessing pipeline..." + echo "Starting PFM postprocessing pipeline" echo "Data type: ${OutputfMRIName}" echo "PFM dimension: ${PFMdim}" @@ -236,8 +235,8 @@ main() { --concat-name="$ConcatName" \ --low-res-mesh="$LowResMesh" \ --runs-timepoints="$subjectExpectedTimepoints" \ - --low-dims="$LowDims" \ --fix-legacy-bias="$FixLegacyBiasString" \ + --num-wishart="$NumWishart"\ --scale-factor="$ScaleFactor" \ --starting-step="$StartStep" \ --stop-after-step="$StopStep" \ @@ -246,10 +245,11 @@ main() { --profumo-config="$ProfumoConfig" \ --profumo-singularity="$ProfumoSingularity" \ --profumo-tr="$TR" \ + --keep-wishart-files="$KeepWishartFiles" \ --profumo-threads="$ProfumoThreads" \ --profumo-dof-correction="$DOFCorrection" \ --profumo-cov-model="$CovModel" \ - --profumo-multi-start-iterations="$nStarts"\ + --profumo-multi-start-iterations="$nStarts" \ --profumo-random-seed="$RandomSeed" \ --ref-image="$RefImage" \ --volume-template-file="$VolumeTemplateCIFTI" @@ -260,4 +260,5 @@ main() { # # Invoke the main function to get things started # -main "$@" \ No newline at end of file +main "$@" + diff --git a/PFM/.gitignore b/PFM/.gitignore new file mode 100644 index 000000000..4adcc2a24 --- /dev/null +++ b/PFM/.gitignore @@ -0,0 +1 @@ +profumo_v2.sif diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index f01626c3a..a72f7d296 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -52,6 +52,8 @@ opts_AddOptional '--profumo-random-seed' 'RandomSeed' 'integer' "random seed for opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'integer' "number of iterations of group-level spatial decomposition before inferring full model" '5' opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' +opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filter iterations for prefiltering (0 to skip)" '0' +opts_AddOptional '--keep-wishart-files' 'KeepWishartFiles' 'YES or NO' "keep Wishart-filtered files after PROFUMO instead of deleting (default NO)" 'NO' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -60,6 +62,7 @@ opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like ' opts_AddOptional '--fix-legacy-bias' 'FixLegacyBias' 'YES or NO' 'whether the input data used legacy bias correction' 'NO' opts_AddOptional '--scale-factor' 'ScaleFactor' 'float' 'scale factor for RSN regression' '0.01' + #general settings opts_AddOptional '--starting-step' 'startStep' 'step' "what step to start processing at, one of: $stepsText" "$defaultStart" @@ -85,7 +88,7 @@ IFS='@' read -a Subjlist <<<"$SubjlistRaw" IFS='@' read -a fMRINamesArray <<<"$fMRINames" FixLegacyBiasBool=$(opts_StringToBool "$FixLegacyBias") - +KeepWishartBool=$(opts_StringToBool "$KeepWishartFiles") if ! [[ "$parLimit" == "-1" || "$parLimit" =~ [1-9][0-9]* ]] then log_Err_Abort "--parallel-limit must be a positive integer or -1, provided value: '$parLimit'" @@ -118,8 +121,6 @@ then RegString="_$RegName" fi -# Volume template file path -# VolumeTemplateFile="${StudyFolder}/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.2.dscalar.nii" for ((stepInd = startInd; stepInd <= stopAfterInd; ++stepInd)) do @@ -142,6 +143,114 @@ do log_Err_Abort "Reference image must be specified with --ref-image" fi + ProfumoConfigToUse="${ProfumoConfig}" + if [[ "$NumWishart" -gt 0 ]] + then + WFDir="${PFMFolder}/WishartFilter_WF${NumWishart}" + # Check if WF files already exist + wfComplete=true + for Subject in "${Subjlist[@]}" + do + for fMRIName in "${fMRINamesArray[@]}" + do + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + wfFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] && [[ ! -f "$wfFile" ]] + then + wfComplete=false + break 2 + fi + done + done + + if $wfComplete + then + log_Msg "WF files already exist in ${WFDir}" + else + log_Msg "Running Wishart filtering with ${NumWishart} iterations" + for Subject in "${Subjlist[@]}" + do + mkdir -p "${WFDir}/${Subject}" + + if [[ "$ConcatName" != "" ]] + then + # Use already concatenated file + concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + + if [[ -f "$concatFile" ]] + then + log_Msg "Applying Wishart filter to concat file for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + --input="$concatFile" \ + --output="$concatOutFile" \ + --num-wishart="$NumWishart" \ + --matlab-run-mode="$MatlabMode" + + # Split back into individual runs + cumTP=0 + for fMRIName in "${fMRINamesArray[@]}"ßß + do + origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + if [[ -f "$origFile" ]] + then + nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) + startIdx=$((cumTP + 1)) + endIdx=$((cumTP + nTP)) + outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + wb_command -cifti-merge "$outFile" -direction ROW \ + -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + cumTP=$endIdx + fi + done + fi + else + # No concat file then pass individual runs + inputList="" + outputList="" + for fMRIName in "${fMRINamesArray[@]}" + do + inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$inputFile" ]] + then + if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi + inputList+="$inputFile" + outputList+="$outputFile" + fi + done + log_Msg "Applying Wishart filter for subject $Subject" + "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + --input="$inputList" \ + --output="$outputList" \ + --num-wishart="$NumWishart" \ + --matlab-run-mode="$MatlabMode" + fi + done + fi + + # Build JSON pointing at WF files + ProfumoConfigToUse="${WFDir}/wishart_dataLocations.json" + echo '{' > "$ProfumoConfigToUse" + for Subject in "${Subjlist[@]}" + do + echo -e "\t\"$Subject\": {" >> "$ProfumoConfigToUse" + for fMRIName in "${fMRINamesArray[@]}" + do + WFFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + if [[ -f "$WFFile" ]] + then + echo -e "\t\t\"$fMRIName\": \"$WFFile\"," >> "$ProfumoConfigToUse" + fi + done + perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" + echo -e "\t}," >> "$ProfumoConfigToUse" + done + perl -pi -e 'if (eof) { s/,$// }' "$ProfumoConfigToUse" + echo "}" >> "$ProfumoConfigToUse" + log_Msg "WF complete" + fi + # Set up PROFUMO paths PFM_PATH="${PFMFolder}/Analysis.pfm" RESULTS_PATH="${PFMFolder}/Results.ppp" @@ -154,10 +263,9 @@ do if [[ -d "${PFMFolder}" ]] then log_Warn "PFM output folder ${PFMFolder} already exists, clearing contents" - find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -delete + find "${PFMFolder}" -mindepth 1 -not -name "dataLocations.json" -not -path "*/WishartFilter_WF*" -delete fi - mkdir -p "${PFMFolder}" - + # Build optional initialMaps argument InitialMapsArg="" if [[ -n "${InitialMaps}" && -f "${InitialMaps}" ]] @@ -173,20 +281,22 @@ do LoadSequentiallyArg="--loadSequentially" fi - ## Rewrite input files with wb_command, as a failsafe to avoid SIMD buffer alignment issue in PROFUMO if - ## files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. - cat "${ProfumoConfig}" | \ - while IFS= read -r line; do - if [[ "$line" != *'.nii"'* ]]; then continue;fi # Only process lines that contain .nii" - filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') - wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" - done - + # files were written with an older version of cifti-matlab with 8-byte instead of 16-byte alignment. + if [[ "${NumWishart}" -eq 0 ]] + then + cat "${ProfumoConfig}" | \ + while IFS= read -r line; do + if [[ "$line" != *'.nii"'* ]]; then continue;fi # Only process lines that contain .nii" + filePath=$(echo "$line" | sed -E 's/^[[:space:]]*"[^"]*"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/') + wb_command -file-convert -cifti-version-convert "$filePath" 2 "$filePath" + done + fi + # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ - /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" \ @@ -194,12 +304,31 @@ do apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ - /opt/profumo/C++/PROFUMO "${ProfumoConfig}" \ + /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} + + #Cleanup WF files + if [[ "$NumWishart" -gt 0 ]] + then + if ((KeepWishartBool)) + then + log_Msg "Keeping Wishart filtered files in ${WFDir}" + else + log_Msg "Cleaning up Wishart filtered files" + rm -rf "${WFDir}" + fi + fi + ;; + + (PostPROFUMO) log_Msg "Running PROFUMO postprocessing" + PFM_PATH="${PFMFolder}/Analysis.pfm" + RESULTS_PATH="${PFMFolder}/Results.ppp" + REAL_REF_IMAGE=$(readlink -f "${RefImage}") + echo apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ "${ProfumoSingularity}" \ @@ -216,17 +345,14 @@ do "${PFM_PATH}" \ "${RESULTS_PATH}" \ "${REAL_REF_IMAGE}" - - ;; - (PostPROFUMO) log_Msg "Running PostPROFUMO step" "$HCPPIPEDIR"/PFM/scripts/PostPROFUMO.sh \ --study-folder="$StudyFolder" \ --subject-list="$SubjlistRaw" \ --fmri-names="$fMRINames" \ --concat-name="$ConcatName" \ - --proc-string="$fMRIProcSTRING" \ + --proc-string="_Atlas${RegString}_${fMRIProcSTRING}" \ --output-fmri-name="$OutputfMRIName" \ --output-string="$OutputSTRING" \ --surf-reg-name="$RegName" \ @@ -238,26 +364,29 @@ do (RSNRegression) log_Msg "Running RSNRegression step" - # Set up template paths - # LowDimTemplate="${StudyFolder}/${GroupAverageName}/MNINonLinear/Results/${OutputfMRIName}/sICA/melodic_oIC_${PFMdim}.dscalar.nii" - + # Set up template paths for Subject in "${Subjlist[@]}" do - # Build list of existing fMRI files for this subject (same logic as your example) - fMRINamesForSub="" - for fMRIName in "${fMRINamesArray[@]}" - do - if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_${RegName}${fMRIProcSTRING}.dtseries.nii" ]] - then - if [[ "$fMRINamesForSub" != "" ]] + if [[ "$ConcatName" != "" ]] + then + fMRINamesForSub="${ConcatName}" + else + # Build list of existing fMRI files for this subject (same logic as your example) + fMRINamesForSub="" + for fMRIName in "${fMRINamesArray[@]}" + do + if [[ -f "${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" ]] then - fMRINamesForSub="${fMRINamesForSub}@${fMRIName}" - else - fMRINamesForSub="${fMRIName}" + if [[ "$fMRINamesForSub" != "" ]] + then + fMRINamesForSub="${fMRINamesForSub}@${fMRIName}" + else + fMRINamesForSub="${fMRIName}" + fi fi - fi - done - + done + fi + if [[ "$fMRINamesForSub" == "" ]] then log_Warn "No valid fMRI runs found for subject $Subject, skipping" @@ -271,7 +400,7 @@ do rsn_cmd=("$HCPPIPEDIR"/global/scripts/RSNregression.sh --study-folder="$StudyFolder" --subject="$Subject" - --subject-timeseries="$ConcatName" # "$fMRINamesForSub" + --subject-timeseries="$fMRINamesForSub" # "$fMRINamesForSub" --surf-reg-name="$RegName" --low-res="$LowResMesh" --proc-string="_$fMRIProcSTRING" @@ -310,4 +439,4 @@ do ;; esac log_Msg "step $stepName complete" -done \ No newline at end of file +done diff --git a/PFM/scripts/ApplyWFProfumo.m b/PFM/scripts/ApplyWFProfumo.m new file mode 100644 index 000000000..909fd9e79 --- /dev/null +++ b/PFM/scripts/ApplyWFProfumo.m @@ -0,0 +1,26 @@ +function ApplyWishartFilterProfumo(inputFiles, outputFiles, numWisharts) + numWisharts = str2double(numWisharts); + inList = strsplit(inputFiles, ','); + outList = strsplit(outputFiles, ','); + + allData = []; + tpCounts = []; + runMeans = {}; + for i = 1:length(inList) + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + tpCounts(i) = size(cii.cdata, 2); + runMeans{i} = mean(cii.cdata, 2); + allData = [allData (cii.cdata - runMeans{i})]; + end + + Out = icaDim(allData, 0, 1, -1, numWisharts); + + startTP = 1; + for i = 1:length(outList) + endTP = startTP + tpCounts(i) - 1; + cii = ciftiopen(strtrim(inList{i}), 'wb_command'); + cii.cdata = Out.data(:, startTP:endTP) + runMeans{i}; + ciftisave(cii, strtrim(outList{i}), 'wb_command'); + startTP = endTP + 1; + end +end \ No newline at end of file diff --git a/PFM/scripts/ApplyWFProfumo.sh b/PFM/scripts/ApplyWFProfumo.sh new file mode 100755 index 000000000..8b860f221 --- /dev/null +++ b/PFM/scripts/ApplyWFProfumo.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -eu + +pipedirguessed=0 +if [[ "${HCPPIPEDIR:-}" == "" ]] +then + pipedirguessed=1 + export HCPPIPEDIR="$(dirname -- "$0")/../.." +fi + +source "$HCPPIPEDIR/global/scripts/newopts.shlib" "$@" +source "$HCPPIPEDIR/global/scripts/debug.shlib" "$@" +g_matlab_default_mode=1 + +opts_SetScriptDescription "applies Wishart filter to CIFTI dtseries files for PROFUMO" + +opts_AddMandatory '--input' 'inputFile' 'file' "comma-separated list of input dtseries files" +opts_AddMandatory '--output' 'outputFile' 'file' "comma-separated list of output wishart-filtered dtseries files" +opts_AddMandatory '--num-wishart' 'numWisharts' 'integer' "number of Wishart distributions to fit" +opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode + +0 = compiled MATLAB +1 = interpreted MATLAB +2 = Octave" "$g_matlab_default_mode" + +opts_ParseArguments "$@" + +if ((pipedirguessed)) +then + log_Err_Abort "HCPPIPEDIR is not set, you must first source your edited copy of Examples/Scripts/SetUpHCPPipeline.sh" +fi + +opts_ShowValues + +case "$MatlabMode" in + (0) + if [[ "${MATLAB_COMPILER_RUNTIME:-}" == "" ]] + then + log_Err_Abort "to use compiled matlab, you must set and export the variable MATLAB_COMPILER_RUNTIME" + fi + ;; + (1) + matlab_interpreter=(matlab -nodisplay -nosplash) + ;; + (2) + matlab_interpreter=(octave-cli -q --no-window-system) + ;; + (*) + log_Err_Abort "unrecognized matlab mode '$MatlabMode', use 0, 1, or 2" + ;; +esac + +this_script_dir=$(dirname "$0") + +matlab_argarray=("$inputFile" "$outputFile" "$numWisharts") + +case "$MatlabMode" in + (0) + matlab_cmd=("$this_script_dir/Compiled_WishartFilter/run_WishartFilter.sh" "$MATLAB_COMPILER_RUNTIME" "${matlab_argarray[@]}") + log_Msg "running compiled matlab command: ${matlab_cmd[*]}" + "${matlab_cmd[@]}" + ;; + (1 | 2) + matlab_args="" + for thisarg in "${matlab_argarray[@]}" + do + if [[ "$matlab_args" != "" ]] + then + matlab_args+=", " + fi + matlab_args+="'$thisarg'" + done + matlabcode=" + addpath('$HCPPIPEDIR/global/matlab/icaDim'); + addpath('$HCPPIPEDIR/global/matlab'); + addpath('$this_script_dir'); + addpath('$HCPCIFTIRWDIR'); + ApplyWFProfumo($matlab_args);" + + log_Msg "running matlab code: $matlabcode" + "${matlab_interpreter[@]}" <<<"$matlabcode" + echo + ;; +esac \ No newline at end of file diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 6fb0bec71..5c5ccdd58 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -64,7 +64,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', numel(runTCS), 1)]; + TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; end % for r = 1:numel(subfMRINames) %% Create original time course and spectral CIFTI files @@ -102,4 +102,4 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) -end \ No newline at end of file +end From 774d7d482daca2bb7d92179254a146e9b0d5b85e Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Fri, 17 Jul 2026 08:24:11 -0500 Subject: [PATCH 47/74] Update tICA/scripts/tICACleanData.sh Co-authored-by: Tim Coalson --- tICA/scripts/tICACleanData.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tICA/scripts/tICACleanData.sh b/tICA/scripts/tICACleanData.sh index bbfcc2e73..e1702688d 100755 --- a/tICA/scripts/tICACleanData.sh +++ b/tICA/scripts/tICACleanData.sh @@ -14,7 +14,7 @@ source "$HCPPIPEDIR/global/scripts/tempfiles.shlib" "$@" g_matlab_default_mode=1 #description of this script to use in usage -opts_SetScriptDescription "regresses noise group temporal ICA components out of CIFTI and optionaly volume timeseries data and optionally correct the bias legacy field" +opts_SetScriptDescription "regresses noise group temporal ICA components out of CIFTI and optionally volume timeseries data and optionally correct the bias legacy field" #arguments to opts_Add*: switch, variable to set, name for inside of <> in help text, description, [default value other than empty string if AddOptional], [compatibility flag, ...] opts_AddMandatory '--study-folder' 'StudyFolder' 'path' "folder containing all subjects" From a888a267c4711df13fd41cc59d0d6315da0ac74c Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Fri, 17 Jul 2026 08:24:33 -0500 Subject: [PATCH 48/74] Update PFM/PFMPipeline.sh Co-authored-by: Tim Coalson --- PFM/PFMPipeline.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index dd9521cdc..7cb25cb3f 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -340,16 +340,7 @@ do rm -rf "${PFMFolder}"/Results.ppp* 2>/dev/null || true # ignore errors due to nfs silly renamed files, or similar fi - echo apptainer exec --bind $(dirname "${StudyFolder}") \ - --env PROFUMODIR=/opt/profumo \ - --env PYTHONNOUSERSITE=1 \ - "${ProfumoSingularity}" \ - /opt/fsl/fslpython/envs/profumo/bin/python3 /opt/profumo/Python/postprocess_results.py \ - --web-report \ - "${PFM_PATH}" \ - "${RESULTS_PATH}" \ - "${REAL_REF_IMAGE}" - apptainer exec --bind $(dirname "${StudyFolder}") \ + cmd=(apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ @@ -357,7 +348,9 @@ do --web-report \ "${PFM_PATH}" \ "${RESULTS_PATH}" \ - "${REAL_REF_IMAGE}" + "${REAL_REF_IMAGE}") + log_Msg "Running command: ${cmd[*]}" + "${cmd[@]}" log_Msg "Running PostPROFUMO step" "$HCPPIPEDIR"/PFM/scripts/PostPROFUMO.sh \ From d5edd76205a9ba7faf83ec105e99e77c42abe375 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Fri, 17 Jul 2026 08:24:51 -0500 Subject: [PATCH 49/74] Update PFM/PFMPipeline.sh Co-authored-by: Tim Coalson --- PFM/PFMPipeline.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 7cb25cb3f..503659a7a 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -294,16 +294,7 @@ do fi # log_Msg "Running PROFUMO decomposition with dimension ${PFMdim}" - echo apptainer exec --bind $(dirname "${StudyFolder}") \ - --env PROFUMODIR=/opt/profumo \ - --env PYTHONNOUSERSITE=1 \ - "${ProfumoSingularity}" \ - /opt/profumo/C++/PROFUMO "${ProfumoConfigToUse}" \ - "${PFMdim}" "${PFM_PATH}" \ - --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ - --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" \ - --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} - apptainer exec --bind $(dirname "${StudyFolder}") \ + cmd=(apptainer exec --bind $(dirname "${StudyFolder}") \ --env PROFUMODIR=/opt/profumo \ --env PYTHONNOUSERSITE=1 \ "${ProfumoSingularity}" \ @@ -311,7 +302,9 @@ do "${PFMdim}" "${PFM_PATH}" \ --useHRF "${TR}" --covModel "${CovModel}" --dofCorrection "${DOFCorrection}" \ --nThreads "${ProfumoThreads}" --lowRankData "${LowRankData}" --randomSeed "${RandomSeed}" \ - --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg} + --multiStartIterations "${MultiStartIterations}" ${LoadSequentiallyArg} ${InitialMapsArg}) + log_Msg "running command: ${cmd[*]}" + "${cmd[@]}" #Cleanup WF files if [[ "$NumWishart" -gt 0 ]] From 1f7bcd99a94b3e0d53a258121c01188d0c7b00cd Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Fri, 17 Jul 2026 14:39:46 -0500 Subject: [PATCH 50/74] default profumo-threads to # physical cores. add url to download .sif --- Examples/Scripts/RunPFMpipelineFullRun.sh | 10 +++++++--- PFM/PFMPipeline.sh | 6 +++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 61fb4b30b..54a504936 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -188,16 +188,20 @@ main() { # Volume template file VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" - # PROFUMO settings - ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" + ## PROFUMO settings ProfumoConfig="${PFMFolder}/dataLocations.json" TR="0.72" - ProfumoThreads="14" + ProfumoThreads="-1" # number of threads for PROFUMO, -1 means auto-detect physical cores DOFCorrection="0.5" CovModel="Subject" nStarts="5" # number of multi-start iterations for PROFUMO RandomSeed="$randSeed" # random seed for PROFUMO reproducibility # RefImage will be auto-set based on data type below + + # Download the PROFUMO Singularity image from the following link and place it in the PFM folder, + # or change the path below to point to your own copy of the image + # https://balsa.wustl.edu/myelin/download?dirName=public&filepath=profumo_v2.sif + ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" # build Profumo data location json mkdir -p $PFMFolder diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 503659a7a..a536cc944 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -44,7 +44,7 @@ opts_AddMandatory '--concat-name' 'ConcatName' "concatenated fMRI name if using opts_AddMandatory '--volume-template-file' 'VolumeTemplateFile' "volume template file path" '' #PROFUMO specific parameters -opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threads for PROFUMO" '25' +opts_AddOptional '--profumo-threads' 'ProfumoThreads' 'integer' "number of threads for PROFUMO" '-1' opts_AddOptional '--profumo-dof-correction' 'DOFCorrection' 'float' "DOF correction for PROFUMO" '0.5' opts_AddOptional '--profumo-cov-model' 'CovModel' 'string' "covariance model for PROFUMO" 'Subject' opts_AddOptional '--profumo-singularity' 'ProfumoSingularity' 'path' "path to PROFUMO singularity container" @@ -83,6 +83,10 @@ fi #display the parsed/default values opts_ShowValues +if [[ "$ProfumoThreads" == "-1" ]]; then + ProfumoThreads=$(par_numphys) +fi + #processing code goes here IFS='@' read -a Subjlist <<<"$SubjlistRaw" IFS='@' read -a fMRINamesArray <<<"$fMRINames" From 0a7af013d06fdea32fb21f20a67f85d549d5ec4c Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 20 Jul 2026 12:34:12 -0500 Subject: [PATCH 51/74] update HCP-YA example parameters and filenames. fix function name when concatnames not supplied. --- Examples/Scripts/RunPFMpipelineFullRun.sh | 8 ++++---- PFM/PFMPipeline.sh | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 54a504936..47fa8e5b9 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -138,7 +138,7 @@ main() { # set the start step beginning from RunPROFUMO which is by default the first step StartStep="RunPROFUMO" StopStep="GroupPFMs" - NumWishart="5" + NumWishart="6" KeepWishartFiles="NO" # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' @@ -158,18 +158,18 @@ main() { subjectExpectedTimepoints="4800" # set temporal highpass full-width (2*sigma) used in preprocessing - HighPass="2000" + HighPass="0" #set fMRIResolution of data, like '2','1.60' or '2.40' fMRIResolution="2.0" # PFM settings for REST data # set the PFM dimensionality - PFMdim="99" + PFMdim="76" PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} # Reference image for PROFUMO - RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_rclean_tclean_meanvn.dscalar.nii" + RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_tclean_meanvn.dscalar.nii" # set the file name component representing the preprocessing already done fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index a536cc944..f8ca395cf 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -31,7 +31,7 @@ opts_AddMandatory '--subject-list' 'SubjlistRaw' '100206@100307...' "list of sub opts_AddMandatory '--fmri-names' 'fMRINames' 'rfMRI_REST1_LR@rfMRI_REST1_RL...' "list of fmri run names separated by @s" opts_AddMandatory '--output-fmri-name' 'OutputfMRIName' 'rfMRI_REST' "name to use for PFM pipeline outputs" opts_AddMandatory '--output-string' 'OutputSTRING' 'string' "output string for individual subject files (typically includes dimension, group name, and seed)" -opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing already done, e.g. '_Atlas_MSMAll_hp2000_clean_rclean_tclean'" +opts_AddMandatory '--proc-string' 'fMRIProcSTRING' 'string' "file name component representing the preprocessing already done, e.g. '_Atlas_MSMAll_hp0_clean_tclean'" opts_AddMandatory '--group-average-name' 'GroupAverageName' 'string' 'name to use for the group output folder' opts_AddMandatory '--pfm-dimension' 'PFMdim' 'integer' "PFM dimensionality (e.g., 76, 92, 65)" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder containing Results.ppp" @@ -223,12 +223,17 @@ do outputList+="$outputFile" fi done + log_Msg "Applying Wishart filter for subject $Subject" - "$HCPPIPEDIR"/PFM/scripts/ApplyWishartFilterProfumo.sh \ + "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ --input="$inputList" \ --output="$outputList" \ --num-wishart="$NumWishart" \ --matlab-run-mode="$MatlabMode" + + + + fi done fi From acdca9611e69c2ca8602890074daa4705ea68e94 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 20 Jul 2026 12:39:27 -0500 Subject: [PATCH 52/74] defuctionalize --- Examples/Scripts/RunPFMpipelineFullRun.sh | 290 ++++++++++------------ 1 file changed, 135 insertions(+), 155 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index 47fa8e5b9..ff6194424 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -108,161 +108,141 @@ get_options() { echo "-- ${scriptName}: Specified Command-Line Options: -- End --" } - - -# -# Function Description -# Main processing of this script -# -# Gets user specified command line options and runs PFM postprocessing pipeline -# (please make sure the PROFUMO, ICA-FIX, MSMAll and MakeAverageDataset are finished before running this script) -# - -main() { - - # get command line options - get_options "$@" - - # set up pipeline environment variables and software - source "${EnvironmentScript}" - - if ((RunLocal)) || [[ "$QUEUE" == "" ]]; then - echo "running locally" - queuing_command=("$HCPPIPEDIR"/global/scripts/captureoutput.sh) - else - echo "queueing with fsl_sub to $QUEUE" - queuing_command=("$FSLDIR/bin/fsl_sub" -q "$QUEUE") - fi - - # general settings - # set the start step beginning from RunPROFUMO which is by default the first step - StartStep="RunPROFUMO" - StopStep="GroupPFMs" - NumWishart="6" - KeepWishartFiles="NO" - - # set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' - parLimit=-1 - - - # general inputs - fMRINames="rfMRI_REST1_LR@rfMRI_REST1_RL@rfMRI_REST2_LR@rfMRI_REST2_RL" - - randSeed=123 # random seed for PROFUMO - - OutputfMRIName="rfMRI_REST" - # set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs - ConcatName="" - - # set the output spectra size for individual projection, RunsXNumTimePoints #subjectExpectedTimepoints="3655" - subjectExpectedTimepoints="4800" - - # set temporal highpass full-width (2*sigma) used in preprocessing - HighPass="0" - - #set fMRIResolution of data, like '2','1.60' or '2.40' - fMRIResolution="2.0" - - # PFM settings for REST data - # set the PFM dimensionality - PFMdim="76" - - PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} - # Reference image for PROFUMO - RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_tclean_meanvn.dscalar.nii" - - # set the file name component representing the preprocessing already done - fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" - - # set the mesh resolution, like '32' for 32k_fs_LR - LowResMesh="32" - - - # Define OutputSTRING with seed designation - OutputSTRING="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs" - - # RSN regression settings - FixLegacyBiasString="NO" - ScaleFactor="0.01" - - # Volume template file - VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" - - ## PROFUMO settings - ProfumoConfig="${PFMFolder}/dataLocations.json" - TR="0.72" - ProfumoThreads="-1" # number of threads for PROFUMO, -1 means auto-detect physical cores - DOFCorrection="0.5" - CovModel="Subject" - nStarts="5" # number of multi-start iterations for PROFUMO - RandomSeed="$randSeed" # random seed for PROFUMO reproducibility - # RefImage will be auto-set based on data type below - - # Download the PROFUMO Singularity image from the following link and place it in the PFM folder, - # or change the path below to point to your own copy of the image - # https://balsa.wustl.edu/myelin/download?dirName=public&filepath=profumo_v2.sif - ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" - - # build Profumo data location json - mkdir -p $PFMFolder - echo '{' > $ProfumoConfig - for Subject in $(echo $Subjlist | tr "@" "\n"); do - echo -e "\t\"$Subject\": {" >> $ProfumoConfig - for fMRIName in $(echo $fMRINames | tr "@" "\n"); do - runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - if [[ -e $runFile ]]; then - echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig - fi - done - perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma - echo -e "\t}," >> $ProfumoConfig +# get command line options +get_options "$@" + +# set up pipeline environment variables and software +source "${EnvironmentScript}" + +if ((RunLocal)) || [[ "$QUEUE" == "" ]]; then + echo "running locally" + queuing_command=("$HCPPIPEDIR"/global/scripts/captureoutput.sh) +else + echo "queueing with fsl_sub to $QUEUE" + queuing_command=("$FSLDIR/bin/fsl_sub" -q "$QUEUE") +fi + +# Download the PROFUMO Singularity image from the following link and place it in the PFM folder, +# or change the path below to point to your own copy of the image +# https://balsa.wustl.edu/myelin/download?dirName=public&filepath=profumo_v2.sif +ProfumoSingularity="$HCPPIPEDIR/PFM/profumo_v2.sif" + +# general settings +# set the start step beginning from RunPROFUMO which is by default the first step +StartStep="RunPROFUMO" +StopStep="GroupPFMs" +NumWishart="6" +KeepWishartFiles="NO" + +# set how many subjects to do in parallel (local, not cluster-distributed) during RSN regression, defaults to all detected physical cores, '-1' +parLimit=-1 + +# general inputs +fMRINames="rfMRI_REST1_LR@rfMRI_REST1_RL@rfMRI_REST2_LR@rfMRI_REST2_RL" + +randSeed=123 # random seed for PROFUMO + +OutputfMRIName="rfMRI_REST" +# set the MR concat fMRI name, if multi-run FIX was used, leave empty for single runs +ConcatName="" + +# set the output spectra size for individual projection, RunsXNumTimePoints #subjectExpectedTimepoints="3655" +subjectExpectedTimepoints="4800" + +# set temporal highpass full-width (2*sigma) used in preprocessing +HighPass="0" + +#set fMRIResolution of data, like '2','1.60' or '2.40' +fMRIResolution="2.0" + +# PFM settings for REST data +# set the PFM dimensionality +PFMdim="76" + +PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} +# Reference image for PROFUMO +RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_tclean_meanvn.dscalar.nii" + +# set the file name component representing the preprocessing already done +fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" + +# set the mesh resolution, like '32' for 32k_fs_LR +LowResMesh="32" + +# Define OutputSTRING with seed designation +OutputSTRING="${OutputfMRIName}_d${PFMdim}_${GroupAverageName}_seed${randSeed}_PFMs" + +# RSN regression settings +FixLegacyBiasString="NO" +ScaleFactor="0.01" + +# Volume template file +VolumeTemplateCIFTI="${HOME}/data/HCPpipelines_ExampleData/${GroupAverageName}/MNINonLinear/${GroupAverageName}_CIFTIVolumeTemplate_${OutputfMRIName}.${fMRIResolution}.dscalar.nii" + +## PROFUMO settings +ProfumoConfig="${PFMFolder}/dataLocations.json" +TR="0.72" +ProfumoThreads="-1" # number of threads for PROFUMO, -1 means auto-detect physical cores +DOFCorrection="0.5" +CovModel="Subject" +nStarts="5" # number of multi-start iterations for PROFUMO +RandomSeed="$randSeed" # random seed for PROFUMO reproducibility +# RefImage will be auto-set based on data type below + +# build Profumo data location json +mkdir -p $PFMFolder +echo '{' > $ProfumoConfig +for Subject in $(echo $Subjlist | tr "@" "\n"); do + echo -e "\t\"$Subject\": {" >> $ProfumoConfig + for fMRIName in $(echo $fMRINames | tr "@" "\n"); do + runFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + if [[ -e $runFile ]]; then + echo -e "\t\t\"$fMRIName\": \"$runFile\"," >> $ProfumoConfig + fi done perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma - echo "}" >> $ProfumoConfig - - # PFM pipeline execution - echo "Starting PFM postprocessing pipeline" - echo "Data type: ${OutputfMRIName}" - echo "PFM dimension: ${PFMdim}" - - "${queuing_command[@]}" "$HCPPIPEDIR"/PFM/PFMPipeline.sh \ - --study-folder="$StudyFolder" \ - --subject-list="$Subjlist" \ - --fmri-names="$fMRINames" \ - --output-fmri-name="$OutputfMRIName" \ - --output-string="$OutputSTRING" \ - --proc-string="$fMRIProcSTRING" \ - --group-average-name="$GroupAverageName" \ - --pfm-dimension="$PFMdim" \ - --pfm-folder="$PFMFolder" \ - --surf-reg-name="$RegName" \ - --concat-name="$ConcatName" \ - --low-res-mesh="$LowResMesh" \ - --runs-timepoints="$subjectExpectedTimepoints" \ - --fix-legacy-bias="$FixLegacyBiasString" \ - --num-wishart="$NumWishart"\ - --scale-factor="$ScaleFactor" \ - --starting-step="$StartStep" \ - --stop-after-step="$StopStep" \ - --parallel-limit="$parLimit" \ - --matlab-run-mode="$MatlabMode" \ - --profumo-config="$ProfumoConfig" \ - --profumo-singularity="$ProfumoSingularity" \ - --profumo-tr="$TR" \ - --keep-wishart-files="$KeepWishartFiles" \ - --profumo-threads="$ProfumoThreads" \ - --profumo-dof-correction="$DOFCorrection" \ - --profumo-cov-model="$CovModel" \ - --profumo-multi-start-iterations="$nStarts" \ - --profumo-random-seed="$RandomSeed" \ - --ref-image="$RefImage" \ - --volume-template-file="$VolumeTemplateCIFTI" - - echo "PFM pipeline submitted successfully!" -} - -# -# Invoke the main function to get things started -# -main "$@" + echo -e "\t}," >> $ProfumoConfig +done +perl -pi -e 'if (eof) { s/,$// }' $ProfumoConfig # remove trailing comma +echo "}" >> $ProfumoConfig + +# PFM pipeline execution +echo "Starting PFM postprocessing pipeline" +echo "Data type: ${OutputfMRIName}" +echo "PFM dimension: ${PFMdim}" + +"${queuing_command[@]}" "$HCPPIPEDIR"/PFM/PFMPipeline.sh \ + --study-folder="$StudyFolder" \ + --subject-list="$Subjlist" \ + --fmri-names="$fMRINames" \ + --output-fmri-name="$OutputfMRIName" \ + --output-string="$OutputSTRING" \ + --proc-string="$fMRIProcSTRING" \ + --group-average-name="$GroupAverageName" \ + --pfm-dimension="$PFMdim" \ + --pfm-folder="$PFMFolder" \ + --surf-reg-name="$RegName" \ + --concat-name="$ConcatName" \ + --low-res-mesh="$LowResMesh" \ + --runs-timepoints="$subjectExpectedTimepoints" \ + --fix-legacy-bias="$FixLegacyBiasString" \ + --num-wishart="$NumWishart"\ + --scale-factor="$ScaleFactor" \ + --starting-step="$StartStep" \ + --stop-after-step="$StopStep" \ + --parallel-limit="$parLimit" \ + --matlab-run-mode="$MatlabMode" \ + --profumo-config="$ProfumoConfig" \ + --profumo-singularity="$ProfumoSingularity" \ + --profumo-tr="$TR" \ + --keep-wishart-files="$KeepWishartFiles" \ + --profumo-threads="$ProfumoThreads" \ + --profumo-dof-correction="$DOFCorrection" \ + --profumo-cov-model="$CovModel" \ + --profumo-multi-start-iterations="$nStarts" \ + --profumo-random-seed="$RandomSeed" \ + --ref-image="$RefImage" \ + --volume-template-file="$VolumeTemplateCIFTI" + +echo "PFM pipeline submitted successfully!" From ddb9f13521f6465e350366b07871788499a5d8ab Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 20 Jul 2026 13:17:54 -0500 Subject: [PATCH 53/74] remove only "Results.ppp" dir --- PFM/PFMPipeline.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index f8ca395cf..3d8b00ae6 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -323,7 +323,7 @@ do log_Msg "Keeping Wishart filtered files in ${WFDir}" else log_Msg "Cleaning up Wishart filtered files" - rm -rf "${WFDir}" + rm -rf "${WFDir}" 2>/dev/null || true # ignore errors due to nfs silly renamed files, or similar fi fi ;; @@ -334,12 +334,12 @@ do RESULTS_PATH="${PFMFolder}/Results.ppp" REAL_REF_IMAGE=$(readlink -f "${RefImage}") - # Remove any existing Results.ppp directory (and ++ variants created by re-runs) + # Remove any existing Results.ppp directory # so postprocess_results.py writes fresh output to Results.ppp if [[ -d "${PFMFolder}/Results.ppp" ]] then - log_Warn "Results.ppp folder ${PFMFolder}/Results.ppp(+) already exists, clearing before postprocessing" - rm -rf "${PFMFolder}"/Results.ppp* 2>/dev/null || true # ignore errors due to nfs silly renamed files, or similar + log_Warn "Results.ppp folder ${PFMFolder}/Results.ppp already exists, clearing before postprocessing" + rm -rf "${PFMFolder}"/Results.ppp 2>/dev/null || true # ignore errors due to nfs silly renamed files, or similar fi cmd=(apptainer exec --bind $(dirname "${StudyFolder}") \ From bb4826b4ddc464d383c304a432464ff413fa80e1 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 20 Jul 2026 14:07:20 -0500 Subject: [PATCH 54/74] un-variance normalize and un-demean deconcatenated post-WF data --- PFM/PFMPipeline.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 3d8b00ae6..37f173664 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -180,8 +180,9 @@ do then # Use already concatenated file concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + vnScalar="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + vnSeries="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dtseries.nii" concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - if [[ -f "$concatFile" ]] then log_Msg "Applying Wishart filter to concat file for subject $Subject" @@ -191,11 +192,15 @@ do --num-wishart="$NumWishart" \ --matlab-run-mode="$MatlabMode" + # un-variance normalize post-WF concatenated data + wb_command -cifti-math "TCS / VN" "${vnSeries}" -var TCS "${concatOutFile}" -var VN "${vnScalar}" -select 1 1 -repeat # un-variance normalize concatenated data + # Split back into individual runs cumTP=0 for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_mean.dscalar.nii" if [[ -f "$origFile" ]] then nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) @@ -204,7 +209,11 @@ do outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" wb_command -cifti-merge "$outFile" -direction ROW \ -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + + # un-demean the deconcatenated WF data by adding back the mean from the original run + wb_command -cifti-math "TCS + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -select 1 1 -repeat cumTP=$endIdx + fi done fi From 47a38db3f8da7d5916702f4de8af0faa790a17a1 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 20 Jul 2026 14:13:33 -0500 Subject: [PATCH 55/74] moving the concatenation and deconcatenation to PFMPipeline.sh so ApplyWFProfumo just needs to run icaDim --- PFM/scripts/ApplyWFProfumo.m | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/PFM/scripts/ApplyWFProfumo.m b/PFM/scripts/ApplyWFProfumo.m index 909fd9e79..65a07b5ff 100644 --- a/PFM/scripts/ApplyWFProfumo.m +++ b/PFM/scripts/ApplyWFProfumo.m @@ -1,26 +1,7 @@ -function ApplyWishartFilterProfumo(inputFiles, outputFiles, numWisharts) - numWisharts = str2double(numWisharts); - inList = strsplit(inputFiles, ','); - outList = strsplit(outputFiles, ','); - - allData = []; - tpCounts = []; - runMeans = {}; - for i = 1:length(inList) - cii = ciftiopen(strtrim(inList{i}), 'wb_command'); - tpCounts(i) = size(cii.cdata, 2); - runMeans{i} = mean(cii.cdata, 2); - allData = [allData (cii.cdata - runMeans{i})]; - end - - Out = icaDim(allData, 0, 1, -1, numWisharts); - - startTP = 1; - for i = 1:length(outList) - endTP = startTP + tpCounts(i) - 1; - cii = ciftiopen(strtrim(inList{i}), 'wb_command'); - cii.cdata = Out.data(:, startTP:endTP) + runMeans{i}; - ciftisave(cii, strtrim(outList{i}), 'wb_command'); - startTP = endTP + 1; - end +function ApplyWFProfumo(inputFile, outputFile, numWisharts) + numWisharts = str2double(numWisharts); + cii = ciftiopen(strtrim(inputFile), 'wb_command'); + Out = icaDim(cii.cdata, 0, 1, -1, numWisharts); + cii.cdata = Out.data(:, startTP:endTP); + ciftisave(cii, strtrim(outputFile), 'wb_command'); end \ No newline at end of file From 271f419587342554979f9b321ad0b49c0cd03e53 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 22 Jul 2026 09:06:50 -0500 Subject: [PATCH 56/74] for Wishart filtering, Add un-variance normalization and un-demeaning to PFMPipeline deconcatenation in multi-run fix and single-run fix case. also add variance normalization and demean concatenation in single-run fix case. --- PFM/.gitignore | 2 +- PFM/PFMPipeline.sh | 76 ++++++++++++++++++++++++------------ PFM/scripts/ApplyWFProfumo.m | 2 +- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/PFM/.gitignore b/PFM/.gitignore index 4adcc2a24..6e91d7ef0 100644 --- a/PFM/.gitignore +++ b/PFM/.gitignore @@ -1 +1 @@ -profumo_v2.sif +*.sif diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 37f173664..65d0c88e1 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -111,6 +111,8 @@ function stepNameToInd() log_Err_Abort "unrecognized step name: '$1'" } + + startInd=$(stepNameToInd "$startStep") stopAfterInd=$(stepNameToInd "$stopAfterStep") @@ -176,12 +178,11 @@ do do mkdir -p "${WFDir}/${Subject}" - if [[ "$ConcatName" != "" ]] + if [[ "$ConcatName" != "" ]] # multi_run data then # Use already concatenated file concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" vnScalar="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" - vnSeries="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dtseries.nii" concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$concatFile" ]] then @@ -193,56 +194,79 @@ do --matlab-run-mode="$MatlabMode" # un-variance normalize post-WF concatenated data - wb_command -cifti-math "TCS / VN" "${vnSeries}" -var TCS "${concatOutFile}" -var VN "${vnScalar}" -select 1 1 -repeat # un-variance normalize concatenated data - - # Split back into individual runs + wb_command -cifti-math "TCS / VN" "${concatOutFile}" -var TCS "${concatOutFile}" -var VN "${vnScalar}" -select 1 1 -repeat + + # Split back into individual runs and restore means cumTP=0 for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_mean.dscalar.nii" + meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" if [[ -f "$origFile" ]] then nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) startIdx=$((cumTP + 1)) endIdx=$((cumTP + nTP)) outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - wb_command -cifti-merge "$outFile" -direction ROW \ - -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" - - # un-demean the deconcatenated WF data by adding back the mean from the original run + wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + + # un-demean the deconcatenated post-WF data wb_command -cifti-math "TCS + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -select 1 1 -repeat cumTP=$endIdx - fi done fi - else - # No concat file then pass individual runs - inputList="" - outputList="" + else # single run data + # No concat file not supplied so create a temporary one for Wishart filtering + demeanVNarray=() + vnScalarArray=() for fMRIName in "${fMRINamesArray[@]}" do inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - if [[ -f "$inputFile" ]] - then - if [[ "$inputList" != "" ]]; then inputList+=","; outputList+=","; fi - inputList+="$inputFile" - outputList+="$outputFile" - fi + + # demean and variance normalize runs + wb_command -cifti-math "(TCS - MEAN) * VN" "$outputFile" -var TCS "$inputFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat + demeanVNarray+=("$outputFile") + vnScalarArray+=("$vnScalarFile") done + # concatenate the demeaned+VN files + concatOutFile="${WFDir}/${Subject}/CONCAT_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + wb_shortcuts -cifti-concatenate "${concatOutFile}" "${demeanVNarray[*]}" + + log_Msg "Applying Wishart filter for subject $Subject" "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ - --input="$inputList" \ - --output="$outputList" \ + --input="$concatOutFile" \ + --output="$concatOutFile" \ --num-wishart="$NumWishart" \ --matlab-run-mode="$MatlabMode" - - - + # deconcatenate the Wishart filtered data back into individual runs + # (each run has its own VN file, so un-VN with each separately) + cumTP=0 + for fMRIName in "${fMRINamesArray[@]}" + do + origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" + if [[ -f "$origFile" ]] + then + nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) + startIdx=$((cumTP + 1)) + endIdx=$((cumTP + nTP)) + outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + + # un-variance normalize and un-demean each post-WF run + wb_command -cifti-math "(TCS / VN) + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat + + cumTP=$endIdx + fi + done fi done fi diff --git a/PFM/scripts/ApplyWFProfumo.m b/PFM/scripts/ApplyWFProfumo.m index 65a07b5ff..30a3b4028 100644 --- a/PFM/scripts/ApplyWFProfumo.m +++ b/PFM/scripts/ApplyWFProfumo.m @@ -2,6 +2,6 @@ function ApplyWFProfumo(inputFile, outputFile, numWisharts) numWisharts = str2double(numWisharts); cii = ciftiopen(strtrim(inputFile), 'wb_command'); Out = icaDim(cii.cdata, 0, 1, -1, numWisharts); - cii.cdata = Out.data(:, startTP:endTP); + cii.cdata = Out.data; ciftisave(cii, strtrim(outputFile), 'wb_command'); end \ No newline at end of file From 5b48d6f6e3d252c042b93e8ce217f611a656730b Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 22 Jul 2026 11:45:59 -0500 Subject: [PATCH 57/74] fix multi-run deconcatenation, and comment out single run for now --- PFM/PFMPipeline.sh | 110 +++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 65d0c88e1..0de311f33 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -1,4 +1,4 @@ -#!/bin/bash +cd #!/bin/bash set -eu pipedirguessed=0 @@ -182,7 +182,7 @@ do then # Use already concatenated file concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - vnScalar="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + VNA="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$concatFile" ]] then @@ -192,16 +192,14 @@ do --output="$concatOutFile" \ --num-wishart="$NumWishart" \ --matlab-run-mode="$MatlabMode" - - # un-variance normalize post-WF concatenated data - wb_command -cifti-math "TCS / VN" "${concatOutFile}" -var TCS "${concatOutFile}" -var VN "${vnScalar}" -select 1 1 -repeat - + # Split back into individual runs and restore means cumTP=0 for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" + VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + MEAN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" if [[ -f "$origFile" ]] then nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) @@ -210,63 +208,69 @@ do outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" - # un-demean the deconcatenated post-WF data - wb_command -cifti-math "TCS + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -select 1 1 -repeat + # un-variance normalize and un-demean the deconcatenated post-WF data + wb_command -cifti-math "((TCS / VNA) * VN) + MEAN" ${cifti_out} \ + -var TCS ${cifti_out} \ + -var VNA ${VNA} -select 1 1 -repeat \ + -var VN ${VN} -select 1 1 -repeat \ + -var MEAN ${MEAN} -select 1 1 -repeat + cumTP=$endIdx fi done fi else # single run data + echo ToDO # No concat file not supplied so create a temporary one for Wishart filtering - demeanVNarray=() - vnScalarArray=() - for fMRIName in "${fMRINamesArray[@]}" - do - inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" - meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" - outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + # demeanVNarray=() + # vnScalarArray=() + # for fMRIName in "${fMRINamesArray[@]}" + # do + # inputFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + # vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + # meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" + # outputFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - # demean and variance normalize runs - wb_command -cifti-math "(TCS - MEAN) * VN" "$outputFile" -var TCS "$inputFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat - demeanVNarray+=("$outputFile") - vnScalarArray+=("$vnScalarFile") - done - - # concatenate the demeaned+VN files - concatOutFile="${WFDir}/${Subject}/CONCAT_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - wb_shortcuts -cifti-concatenate "${concatOutFile}" "${demeanVNarray[*]}" + # # demean and variance normalize runs + # wb_command -cifti-math "(TCS - MEAN) / VN" "$outputFile" -var TCS "$inputFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat + # demeanVNarray+=("$outputFile") + # vnScalarArray+=("$vnScalarFile") + # done + + # # concatenate the demeaned+VN files + # concatOutFile="${WFDir}/${Subject}/CONCAT_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + # wb_shortcuts -cifti-concatenate "${concatOutFile}" "${demeanVNarray[*]}" - log_Msg "Applying Wishart filter for subject $Subject" - "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ - --input="$concatOutFile" \ - --output="$concatOutFile" \ - --num-wishart="$NumWishart" \ - --matlab-run-mode="$MatlabMode" - - # deconcatenate the Wishart filtered data back into individual runs - # (each run has its own VN file, so un-VN with each separately) - cumTP=0 - for fMRIName in "${fMRINamesArray[@]}" - do - origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" - meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" - if [[ -f "$origFile" ]] - then - nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) - startIdx=$((cumTP + 1)) - endIdx=$((cumTP + nTP)) - outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + # log_Msg "Applying Wishart filter for subject $Subject" + # "$HCPPIPEDIR"/PFM/scripts/ApplyWFProfumo.sh \ + # --input="$concatOutFile" \ + # --output="$concatOutFile" \ + # --num-wishart="$NumWishart" \ + # --matlab-run-mode="$MatlabMode" + + # # deconcatenate the Wishart filtered data back into individual runs + # # (each run has its own VN file, so un-VN with each separately) + # cumTP=0 + # for fMRIName in "${fMRINamesArray[@]}" + # do + # origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" + # vnScalarFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + # meanFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" + # if [[ -f "$origFile" ]] + # then + # nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) + # startIdx=$((cumTP + 1)) + # endIdx=$((cumTP + nTP)) + # outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" + # wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" - # un-variance normalize and un-demean each post-WF run - wb_command -cifti-math "(TCS / VN) + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat + # # un-variance normalize and un-demean each post-WF run + # wb_command -cifti-math "(TCS / VN) + MEAN" "$outFile" -var TCS "$outFile" -var MEAN "$meanFile" -var VN "$vnScalarFile" -select 1 1 -repeat - cumTP=$endIdx - fi - done + # cumTP=$endIdx + # fi + # done fi done fi From aa474a0ca46d4d16fb4ef70b7856a115136115bb Mon Sep 17 00:00:00 2001 From: Andrea Yang Date: Tue, 4 Aug 2026 18:22:01 -0500 Subject: [PATCH 58/74] Update highpass filter and reference image in script --- Examples/Scripts/RunPFMpipelineFullRun.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Scripts/RunPFMpipelineFullRun.sh b/Examples/Scripts/RunPFMpipelineFullRun.sh index ff6194424..44ee1b93a 100755 --- a/Examples/Scripts/RunPFMpipelineFullRun.sh +++ b/Examples/Scripts/RunPFMpipelineFullRun.sh @@ -150,7 +150,7 @@ ConcatName="" subjectExpectedTimepoints="4800" # set temporal highpass full-width (2*sigma) used in preprocessing -HighPass="0" +HighPass="2000" #set fMRIResolution of data, like '2','1.60' or '2.40' fMRIResolution="2.0" @@ -161,7 +161,7 @@ PFMdim="76" PFMFolder=${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}_PFM_d${PFMdim} # Reference image for PROFUMO -RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_tclean_meanvn.dscalar.nii" +RefImage="${StudyFolder}/$GroupAverageName/MNINonLinear/Results/${OutputfMRIName}/${OutputfMRIName}_Atlas_MSMAll_hp${HighPass}_clean_rclean_tclean_meanvn.dscalar.nii" # set the file name component representing the preprocessing already done fMRIProcSTRING="hp${HighPass}_clean_rclean_tclean" From 9046688fc84bd466faf23603e87ef287ea7b3e8e Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 4 Aug 2026 11:49:11 -0500 Subject: [PATCH 59/74] Only re-use wishart filtered data if KeepWishart --- PFM/PFMPipeline.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 0de311f33..e256a47bb 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -169,8 +169,7 @@ do done done - if $wfComplete - then + if $wfComplete && $KeepWishartBool; then log_Msg "WF files already exist in ${WFDir}" else log_Msg "Running Wishart filtering with ${NumWishart} iterations" From 4a109c73241b6da3265affeebe9446eca50cad70 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 5 Aug 2026 08:43:02 -0500 Subject: [PATCH 60/74] fix VN file path --- PFM/PFMPipeline.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index e256a47bb..3fd18c9b9 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -169,7 +169,7 @@ do done done - if $wfComplete && $KeepWishartBool; then + if $wfComplete && [[ $KeepWishartBool == 1 ]]; then log_Msg "WF files already exist in ${WFDir}" else log_Msg "Running Wishart filtering with ${NumWishart} iterations" @@ -197,7 +197,7 @@ do for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING%%_*}_vn.dscalar.nii" # not terribly robust MEAN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" if [[ -f "$origFile" ]] then @@ -208,8 +208,8 @@ do wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" # un-variance normalize and un-demean the deconcatenated post-WF data - wb_command -cifti-math "((TCS / VNA) * VN) + MEAN" ${cifti_out} \ - -var TCS ${cifti_out} \ + wb_command -cifti-math "((TCS / VNA) * VN) + MEAN" ${outFile} \ + -var TCS ${outFile} \ -var VNA ${VNA} -select 1 1 -repeat \ -var VN ${VN} -select 1 1 -repeat \ -var MEAN ${MEAN} -select 1 1 -repeat From b1fd5f9ad080126705195f63e08b5a2f255e91c1 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 12 Aug 2026 16:32:06 -0500 Subject: [PATCH 61/74] remove post-deconcatenation variance restoration operation (as coppied from hcp_multli_run_fix), add variance normalization option that uses the clean_vn dscalar from the tICA pipeline. Add option for weighting data by vertex areas, using the mean area for volume grayordinate weights. --- PFM/PFMPipeline.sh | 54 +++++++++++++++++++++++++++++++------- PFM/scripts/PostPROFUMO.m | 17 +++++++++++- PFM/scripts/PostPROFUMO.sh | 5 +++- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 3fd18c9b9..001e23aea 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -54,6 +54,8 @@ opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initiali opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filter iterations for prefiltering (0 to skip)" '0' opts_AddOptional '--keep-wishart-files' 'KeepWishartFiles' 'YES or NO' "keep Wishart-filtered files after PROFUMO instead of deleting (default NO)" 'NO' +opts_AddOptional '--variance-normalization' 'VarNorm' 'YES or NO' "Variance normalize Wishart-filtered data before PROFUMO (default NO)" 'NO' +opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight Wishart-filtered data by vertex areas for PROFUMO (default NO)" 'NO' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -93,6 +95,9 @@ IFS='@' read -a fMRINamesArray <<<"$fMRINames" FixLegacyBiasBool=$(opts_StringToBool "$FixLegacyBias") KeepWishartBool=$(opts_StringToBool "$KeepWishartFiles") +VarNormBool=$(opts_StringToBool "$VarNorm") +VAweightBool=$(opts_StringToBool "$VAweight") + if ! [[ "$parLimit" == "-1" || "$parLimit" =~ [1-9][0-9]* ]] then log_Err_Abort "--parallel-limit must be a positive integer or -1, provided value: '$parLimit'" @@ -181,7 +186,7 @@ do then # Use already concatenated file concatFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - VNA="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + clean_VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" concatOutFile="${WFDir}/${Subject}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" if [[ -f "$concatFile" ]] then @@ -197,22 +202,50 @@ do for fMRIName in "${fMRINamesArray[@]}" do origFile="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}.dtseries.nii" - VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING%%_*}_vn.dscalar.nii" # not terribly robust - MEAN="${StudyFolder}/${Subject}/MNINonLinear/Results/${fMRIName}/${fMRIName}_Atlas_mean.dscalar.nii" if [[ -f "$origFile" ]] then nTP=$(wb_command -file-information "$origFile" -only-number-of-maps) startIdx=$((cumTP + 1)) endIdx=$((cumTP + nTP)) outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" - wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" + wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" # naive splitting - # un-variance normalize and un-demean the deconcatenated post-WF data - wb_command -cifti-math "((TCS / VNA) * VN) + MEAN" ${outFile} \ - -var TCS ${outFile} \ - -var VNA ${VNA} -select 1 1 -repeat \ - -var VN ${VN} -select 1 1 -repeat \ - -var MEAN ${MEAN} -select 1 1 -repeat + ## The "*_clean*" volume and CIFTI timeseries have no mean, i.e. they are intensity normalized. + ## However, timeseries files without "*_vn*"" retain run variances, i.e. they are not variance normalized. + ## To expound, for concatenated (non "_vn*") timeseries, the indiviudal run variances are divided out, data concatenated and the mean of run variances (VNA) multiplied back in. + ## Therefore, naive splitting of the concatenated timeseries will result the data retaining this run-average variance. + ## In addition, the tICA pipeline creates a concat "*_clean_vn*" dscalar file, this contains the pooled variance across runs, rather than average of run variances. + ## The variance normalization option below divides each run (after splitting) by the clean_VN map to yield variance normalized (post-wishart) single run timeseries. + if [[ "$VarNormBool" == 1 ]];then + wb_command -cifti-math "(TCS / clean_VN)" ${outFile} \ + -var TCS ${outFile} \ + -var clean_VN ${clean_VN} -select 1 1 -repeat + fi + + if [[ "$VAweightBool" == 1 ]];then + log_Msg "Weighting data by average vertex areas" + VA=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness_va.${LowResMesh}k_fs_LR.dscalar.nii + VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness_va.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii + ATLASroiL=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.L.atlasroi.${LowResMesh}k_fs_LR.shape.gii + ATLASroiR=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.R.atlasroi.${LowResMesh}k_fs_LR.shape.gii + if [[ ! -f "$VAgray" ]]; then + # create VA cifti with volume grayordinates filled with average of vertex areas for weighting + tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file + tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file + tempfiles_create "tmp_lab_XXXXXX.nii.gz" tmp_lab_file + tempfiles_create "tmp_Lva_XXXXXX.shape.gii" tmp_Lva_file + tempfiles_create "tmp_Rva_XXXXXX.shape.gii" tmp_Rva_file + wb_command -cifti-separate ${outFile} COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -label "$tmp_lab_file" + wb_command -cifti-separate ${VA} COLUMN -metric CORTEX_LEFT "$tmp_Lva_file" -metric CORTEX_RIGHT "$tmp_Rva_file" + mean_VA=$(wb_command -cifti-stats ${VA} -reduce MEAN) # $VA is a dscalar cifti already masked by ATLASroi + wb_command -volume-math "(ROI * $mean_VA)" "$tmp_roi_file" -var ROI "$tmp_roi_file" + wb_command -cifti-create-dense-scalar ${VAgray} -volume "$tmp_roi_file" "$tmp_lab_file" \ + -left-metric "$tmp_Lva_file" -roi-left $ATLASroiL -right-metric "$tmp_Rva_file" -roi-right $ATLASroiR + fi + wb_command -cifti-math "(TCS * VA)" ${outFile} \ + -var TCS ${outFile} \ + -var VA ${VAgray} -select 1 1 -repeat + fi cumTP=$endIdx fi @@ -296,6 +329,7 @@ do log_Msg "WF complete" fi + # Set up PROFUMO paths PFM_PATH="${PFMFolder}/Analysis.pfm" RESULTS_PATH="${PFMFolder}/Results.ppp" diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 5c5ccdd58..b8346f56c 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,4 +1,4 @@ -function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder,VAweightBool) % PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) % This function imports PROFUMO results and generates CIFTI-format time courses % and power spectra for each subject. The outputs are used for subsequent @@ -16,6 +16,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % LowResMesh - Mesh resolution (e.g., '10' for 10k) % TR - Repetition time in seconds % PFMFolder - Path to PROFUMO results folder +% VAweightBool - Boolean flag for vertex area weighting %% Parse string inputs and initialize Subjlist = strsplit(SubjListRaw, '@'); @@ -89,6 +90,20 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc ts.NtimepointsPerSubject = size(TCS, 1); PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + %% divide out vertex area weights + if VAweightBool + fprintf('Dividing out vertex area weights\n'); + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness_va.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + PFMTCSorig.cdata = PFMTCSorig.cdata ./ VAgray; + PFMSpectraorig.cdata = PFMSpectraorig.cdata ./ VAgray; + PFMTCS.cdata = PFMTCS.cdata ./ VAgray; + PFMSpectra.cdata = PFMSpectra.cdata ./ VAgray; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata ./ VAgray; + ciftisave(maps, mapFile, wbcommand); + end + %% Save individual-level results % Save original and amplitude-modulated time courses and spectra ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index 699c67c6e..ee7e95c7d 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -28,6 +28,7 @@ opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" opts_AddMandatory '--concat-name' 'ConcatName' 'string' "concatenated fMRI name if using multi-run data" opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' +opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas for PROFUMO (default NO)" 'NO' opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode 0 = compiled MATLAB @@ -35,6 +36,8 @@ opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_m 2 = Octave" "$g_matlab_default_mode" opts_ParseArguments "$@" +VAweightBool=$(opts_StringToBool "$VAweight") + if ((pipedirguessed)) then @@ -75,7 +78,7 @@ IFS='@' read -a fMRIList <<<"$fMRIListRaw" this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder") +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder" "$VAweightBool") case "$MatlabMode" in (0) From 98638924b6f46584228d8c3bb98d18b8f8657bb9 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 13 Aug 2026 11:12:47 -0500 Subject: [PATCH 62/74] Update PFM/PFMPipeline.sh Co-authored-by: glasserm --- PFM/PFMPipeline.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 001e23aea..ca90490c1 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -210,12 +210,7 @@ do outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" # naive splitting - ## The "*_clean*" volume and CIFTI timeseries have no mean, i.e. they are intensity normalized. - ## However, timeseries files without "*_vn*"" retain run variances, i.e. they are not variance normalized. - ## To expound, for concatenated (non "_vn*") timeseries, the indiviudal run variances are divided out, data concatenated and the mean of run variances (VNA) multiplied back in. - ## Therefore, naive splitting of the concatenated timeseries will result the data retaining this run-average variance. - ## In addition, the tICA pipeline creates a concat "*_clean_vn*" dscalar file, this contains the pooled variance across runs, rather than average of run variances. - ## The variance normalization option below divides each run (after splitting) by the clean_VN map to yield variance normalized (post-wishart) single run timeseries. + ## The concatenated timeseries are intensity normalized and differences in unstructured noise variance between runs and the means have been removed. For model-free analyses (i.e., not task-GLM) that prefer single runs, it is best to simply deconcatenate the runs. If variance normalization is desired, use the same _clean_vn file from the concatenated folder for each run, rather than the original _vn files, which may cause extreme values in areas of little or no signal and require more complex handling. if [[ "$VarNormBool" == 1 ]];then wb_command -cifti-math "(TCS / clean_VN)" ${outFile} \ -var TCS ${outFile} \ From 111dec71ffe32db78df022c916b71b77beb8640a Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 13 Aug 2026 12:18:11 -0500 Subject: [PATCH 63/74] Added RegString. Moved VAgray creation out into subject loop. --- PFM/PFMPipeline.sh | 39 +++++++++++++++++++++------------------ PFM/scripts/PostPROFUMO.m | 2 +- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index ca90490c1..a34185f6c 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -196,6 +196,26 @@ do --output="$concatOutFile" \ --num-wishart="$NumWishart" \ --matlab-run-mode="$MatlabMode" + + if [[ "$VAweightBool" == 1 ]]; then + # create VA cifti with volume grayordinates filled with average of vertex areas for weighting + VA=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va.${LowResMesh}k_fs_LR.dscalar.nii + VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii + ATLASroiL=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.L.atlasroi.${LowResMesh}k_fs_LR.shape.gii + ATLASroiR=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.R.atlasroi.${LowResMesh}k_fs_LR.shape.gii + tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file + tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file + tempfiles_create "tmp_lab_XXXXXX.nii.gz" tmp_lab_file + tempfiles_create "tmp_Lva_XXXXXX.shape.gii" tmp_Lva_file + tempfiles_create "tmp_Rva_XXXXXX.shape.gii" tmp_Rva_file + wb_command -cifti-separate ${outFile} COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -label "$tmp_lab_file" + wb_command -cifti-separate ${VA} COLUMN -metric CORTEX_LEFT "$tmp_Lva_file" -metric CORTEX_RIGHT "$tmp_Rva_file" + mean_VA=$(wb_command -cifti-stats ${VA} -reduce MEAN) # $VA is a dscalar cifti already masked by ATLASroi + wb_command -volume-math "(ROI * $mean_VA)" "$tmp_roi_file" -var ROI "$tmp_roi_file" + wb_command -cifti-create-dense-scalar ${VAgray} -volume "$tmp_roi_file" "$tmp_lab_file" \ + -left-metric "$tmp_Lva_file" -roi-left $ATLASroiL -right-metric "$tmp_Rva_file" -roi-right $ATLASroiR + fi + # Split back into individual runs and restore means cumTP=0 @@ -217,26 +237,9 @@ do -var clean_VN ${clean_VN} -select 1 1 -repeat fi + if [[ "$VAweightBool" == 1 ]];then log_Msg "Weighting data by average vertex areas" - VA=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness_va.${LowResMesh}k_fs_LR.dscalar.nii - VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness_va.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii - ATLASroiL=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.L.atlasroi.${LowResMesh}k_fs_LR.shape.gii - ATLASroiR=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.R.atlasroi.${LowResMesh}k_fs_LR.shape.gii - if [[ ! -f "$VAgray" ]]; then - # create VA cifti with volume grayordinates filled with average of vertex areas for weighting - tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file - tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file - tempfiles_create "tmp_lab_XXXXXX.nii.gz" tmp_lab_file - tempfiles_create "tmp_Lva_XXXXXX.shape.gii" tmp_Lva_file - tempfiles_create "tmp_Rva_XXXXXX.shape.gii" tmp_Rva_file - wb_command -cifti-separate ${outFile} COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -label "$tmp_lab_file" - wb_command -cifti-separate ${VA} COLUMN -metric CORTEX_LEFT "$tmp_Lva_file" -metric CORTEX_RIGHT "$tmp_Rva_file" - mean_VA=$(wb_command -cifti-stats ${VA} -reduce MEAN) # $VA is a dscalar cifti already masked by ATLASroi - wb_command -volume-math "(ROI * $mean_VA)" "$tmp_roi_file" -var ROI "$tmp_roi_file" - wb_command -cifti-create-dense-scalar ${VAgray} -volume "$tmp_roi_file" "$tmp_lab_file" \ - -left-metric "$tmp_Lva_file" -roi-left $ATLASroiL -right-metric "$tmp_Rva_file" -roi-right $ATLASroiR - fi wb_command -cifti-math "(TCS * VA)" ${outFile} \ -var TCS ${outFile} \ -var VA ${VAgray} -select 1 1 -repeat diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index b8346f56c..9b9d77a78 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -93,7 +93,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% divide out vertex area weights if VAweightBool fprintf('Dividing out vertex area weights\n'); - VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness_va.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); PFMTCSorig.cdata = PFMTCSorig.cdata ./ VAgray; PFMSpectraorig.cdata = PFMSpectraorig.cdata ./ VAgray; PFMTCS.cdata = PFMTCS.cdata ./ VAgray; From 9c3d9b0ea96868dc0a46b3b383b3924fc1cb2c13 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Thu, 13 Aug 2026 17:00:00 -0500 Subject: [PATCH 64/74] Update PFM/PFMPipeline.sh Co-authored-by: Tim Coalson --- PFM/PFMPipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index a34185f6c..520fabb59 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -214,7 +214,7 @@ do wb_command -volume-math "(ROI * $mean_VA)" "$tmp_roi_file" -var ROI "$tmp_roi_file" wb_command -cifti-create-dense-scalar ${VAgray} -volume "$tmp_roi_file" "$tmp_lab_file" \ -left-metric "$tmp_Lva_file" -roi-left $ATLASroiL -right-metric "$tmp_Rva_file" -roi-right $ATLASroiR - fi + fi # Split back into individual runs and restore means From bd78fe2f4fe1a36fbb09ce7dd10025ad958482c6 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 18 Aug 2026 10:15:36 -0500 Subject: [PATCH 65/74] Restore variance to maps after PROFUMO, bugfixes with VAweightBool handling, simplify va_norm.grayordinates creation. --- PFM/PFMPipeline.sh | 42 +++++++++++++++++++++----------------- PFM/scripts/PostPROFUMO.m | 22 +++++++++++++------- PFM/scripts/PostPROFUMO.sh | 3 ++- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 520fabb59..e1d6a1bb9 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -53,9 +53,11 @@ opts_AddOptional '--profumo-multi-start-iterations' 'MultiStartIterations' 'inte opts_AddOptional '--profumo-initial-maps' 'InitialMaps' 'path' "file to initialise the decomposition based on spatial maps" opts_AddOptional '--profumo-load-sequentially' 'LoadSequentially' 'YES or NO' "load data sequentially in PROFUMO (useful for memory management)" 'YES' opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filter iterations for prefiltering (0 to skip)" '0' + +# Post-Wishart filter specific parameters (only used if --num-wishart > 0) opts_AddOptional '--keep-wishart-files' 'KeepWishartFiles' 'YES or NO' "keep Wishart-filtered files after PROFUMO instead of deleting (default NO)" 'NO' -opts_AddOptional '--variance-normalization' 'VarNorm' 'YES or NO' "Variance normalize Wishart-filtered data before PROFUMO (default NO)" 'NO' -opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight Wishart-filtered data by vertex areas for PROFUMO (default NO)" 'NO' +opts_AddOptional '--variance-normalization' 'VarNorm' 'YES or NO' "Variance normalize data before PROFUMO (default NO)" 'NO' +opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas before PROFUMO (default NO)" 'NO' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' @@ -198,25 +200,15 @@ do --matlab-run-mode="$MatlabMode" if [[ "$VAweightBool" == 1 ]]; then - # create VA cifti with volume grayordinates filled with average of vertex areas for weighting - VA=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va.${LowResMesh}k_fs_LR.dscalar.nii - VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii - ATLASroiL=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.L.atlasroi.${LowResMesh}k_fs_LR.shape.gii - ATLASroiR=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.R.atlasroi.${LowResMesh}k_fs_LR.shape.gii + # create VA_norm cifti with volume grayordinates filled with ones areas for weighting + VAnorm=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii + VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file - tempfiles_create "tmp_lab_XXXXXX.nii.gz" tmp_lab_file - tempfiles_create "tmp_Lva_XXXXXX.shape.gii" tmp_Lva_file - tempfiles_create "tmp_Rva_XXXXXX.shape.gii" tmp_Rva_file - wb_command -cifti-separate ${outFile} COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -label "$tmp_lab_file" - wb_command -cifti-separate ${VA} COLUMN -metric CORTEX_LEFT "$tmp_Lva_file" -metric CORTEX_RIGHT "$tmp_Rva_file" - mean_VA=$(wb_command -cifti-stats ${VA} -reduce MEAN) # $VA is a dscalar cifti already masked by ATLASroi - wb_command -volume-math "(ROI * $mean_VA)" "$tmp_roi_file" -var ROI "$tmp_roi_file" - wb_command -cifti-create-dense-scalar ${VAgray} -volume "$tmp_roi_file" "$tmp_lab_file" \ - -left-metric "$tmp_Lva_file" -roi-left $ATLASroiL -right-metric "$tmp_Rva_file" -roi-right $ATLASroiR + wb_command -cifti-separate "${concatOutFile}" COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -crop + wb_command -cifti-create-dense-from-template "${concatOutFile}" "$VAgray" -cifti "$VAnorm" -volume-all "$tmp_roi_file" -from-cropped fi - - + # Split back into individual runs and restore means cumTP=0 for fMRIName in "${fMRINamesArray[@]}" @@ -230,7 +222,10 @@ do outFile="${WFDir}/${Subject}/${fMRIName}_Atlas${RegString}_${fMRIProcSTRING}_WF.dtseries.nii" wb_command -cifti-merge "$outFile" -direction ROW -cifti "$concatOutFile" -index "$startIdx" -up-to "$endIdx" # naive splitting - ## The concatenated timeseries are intensity normalized and differences in unstructured noise variance between runs and the means have been removed. For model-free analyses (i.e., not task-GLM) that prefer single runs, it is best to simply deconcatenate the runs. If variance normalization is desired, use the same _clean_vn file from the concatenated folder for each run, rather than the original _vn files, which may cause extreme values in areas of little or no signal and require more complex handling. + ## The concatenated timeseries are intensity normalized and differences in unstructured noise variance between runs and the means have been removed. + # For model-free analyses (i.e., not task-GLM) that prefer single runs, it is best to simply deconcatenate the runs. + # If variance normalization is desired, use the same _clean_vn file from the concatenated folder for each run, rather than the original _vn files, + # which may cause extreme values in areas of little or no signal and require more complex handling. if [[ "$VarNormBool" == 1 ]];then wb_command -cifti-math "(TCS / clean_VN)" ${outFile} \ -var TCS ${outFile} \ @@ -402,6 +397,12 @@ do RESULTS_PATH="${PFMFolder}/Results.ppp" REAL_REF_IMAGE=$(readlink -f "${RefImage}") + if [[ "${VarNormBool}" == 1 ]];then + clean_VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" + else + clean_VN='''' + fi + # Remove any existing Results.ppp directory # so postprocess_results.py writes fresh output to Results.ppp if [[ -d "${PFMFolder}/Results.ppp" ]] @@ -435,7 +436,10 @@ do --low-res-mesh="$LowResMesh" \ --profumo-tr="$TR" \ --pfm-folder="$PFMFolder" \ + --clean-vn-name="$clean_VN" \ + --weight-vertex-areas="$VAweight" \ --matlab-run-mode="$MatlabMode" + ;; (RSNRegression) log_Msg "Running RSNRegression step" diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 9b9d77a78..7f147edaa 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,4 +1,4 @@ -function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder,VAweightBool) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder,clean_VN_Name,VAweightBool) % PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) % This function imports PROFUMO results and generates CIFTI-format time courses % and power spectra for each subject. The outputs are used for subsequent @@ -16,12 +16,14 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % LowResMesh - Mesh resolution (e.g., '10' for 10k) % TR - Repetition time in seconds % PFMFolder - Path to PROFUMO results folder +% clean_VN_Name - Path to group variance normalization file for restoring variance (unused if empty) % VAweightBool - Boolean flag for vertex area weighting %% Parse string inputs and initialize Subjlist = strsplit(SubjListRaw, '@'); fMRINames = strsplit(fMRIListRaw, '@'); TR = str2double(TR); +VAweightBool = logical(str2double(VAweightBool)); wbcommand = 'wb_command'; %% Main loop: Process each subject @@ -81,7 +83,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Create time course and spectral CIFTI files % Generate CIFTI structure for time courses - PFMTCS = cifti_struct_create_sdseries(TCS'); + PFMTCS = cifti_struct_create_sdseries(TCS','step',TR); % Store power spectra ts.Nnodes = size(TCS, 2); @@ -90,14 +92,20 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc ts.NtimepointsPerSubject = size(TCS, 1); PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + %% restore variance + if ~isempty(clean_VN_Name) + fprintf('Restoring variance\n'); + clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata .* clean_VN; + ciftisave(maps, mapFile, wbcommand); + end + %% divide out vertex area weights if VAweightBool fprintf('Dividing out vertex area weights\n'); - VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand); - PFMTCSorig.cdata = PFMTCSorig.cdata ./ VAgray; - PFMSpectraorig.cdata = PFMSpectraorig.cdata ./ VAgray; - PFMTCS.cdata = PFMTCS.cdata ./ VAgray; - PFMSpectra.cdata = PFMSpectra.cdata ./ VAgray; + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; maps = ciftiopen(mapFile, wbcommand); maps.cdata = maps.cdata ./ VAgray; diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index ee7e95c7d..d19122d69 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -28,6 +28,7 @@ opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" opts_AddMandatory '--concat-name' 'ConcatName' 'string' "concatenated fMRI name if using multi-run data" opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' +opts_AddOptional '--clean-vn-name' 'clean_VN_Name' 'path' "Path to group variance normalization file for restoring variance (default: none)" '''' opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas for PROFUMO (default NO)" 'NO' opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode @@ -78,7 +79,7 @@ IFS='@' read -a fMRIList <<<"$fMRIListRaw" this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder" "$VAweightBool") +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder" "$clean_VN_Name" "$VAweightBool") case "$MatlabMode" in (0) From 3ba3bf54d7eb0f62a65074ca36356cd302f5e852 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 18 Aug 2026 10:29:11 -0500 Subject: [PATCH 66/74] standardize indentation --- PFM/PFMPipeline.sh | 16 ++++----- PFM/scripts/PostPROFUMO.m | 68 +++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index e1d6a1bb9..c0944fc73 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -227,17 +227,17 @@ do # If variance normalization is desired, use the same _clean_vn file from the concatenated folder for each run, rather than the original _vn files, # which may cause extreme values in areas of little or no signal and require more complex handling. if [[ "$VarNormBool" == 1 ]];then - wb_command -cifti-math "(TCS / clean_VN)" ${outFile} \ - -var TCS ${outFile} \ - -var clean_VN ${clean_VN} -select 1 1 -repeat + log_Msg "Normalizing variance" + wb_command -cifti-math "(TCS / clean_VN)" ${outFile} \ + -var TCS ${outFile} \ + -var clean_VN ${clean_VN} -select 1 1 -repeat fi - if [[ "$VAweightBool" == 1 ]];then - log_Msg "Weighting data by average vertex areas" - wb_command -cifti-math "(TCS * VA)" ${outFile} \ - -var TCS ${outFile} \ - -var VA ${VAgray} -select 1 1 -repeat + log_Msg "Weighting data by average vertex areas" + wb_command -cifti-math "(TCS * VA)" ${outFile} \ + -var TCS ${outFile} \ + -var VA ${VAgray} -select 1 1 -repeat fi cumTP=$endIdx diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 7f147edaa..2a86aa7ef 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -28,46 +28,46 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Main loop: Process each subject for s = 1:numel(Subjlist) - fprintf('Processing subject %d/%d: %s\n', s, numel(Subjlist), Subjlist{s}); + fprintf('Processing subject %d/%d: %s\n', s, numel(Subjlist), Subjlist{s}); - %% Identify available fMRI runs for this subject - % Determine which fMRI runs exist for this subject - % If ConcatName is specified, use concatenated version; otherwise check individual runs - subfMRINames = {}; - if ~strcmp(ConcatName, '') + %% Identify available fMRI runs for this subject + % Determine which fMRI runs exist for this subject + % If ConcatName is specified, use concatenated version; otherwise check individual runs + subfMRINames = {}; + if ~strcmp(ConcatName, '') % Multi-run data: check if concatenated dataset exists if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii'],'file') - c = 1; - for r = 1:numel(fMRINames) + c = 1; + for r = 1:numel(fMRINames) if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') - subfMRINames{c} = fMRINames{r}; - c = c + 1; + subfMRINames{c} = fMRINames{r}; + c = c + 1; end - end % for r = 1:numel(fMRINames) + end % for r = 1:numel(fMRINames) end - else + else % Single-run data: check which runs exist c = 1; for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') subfMRINames{c} = fMRINames{r}; c = c + 1; - end + end end % for r = 1:numel(fMRINames) - end + end - %% Process subject if valid runs found - if numel(subfMRINames) ~= 0 + %% Process subject if valid runs found + if numel(subfMRINames) ~= 0 %% Load and concatenate PFM time courses and amplitudes % Load PROFUMO outputs and amplitude-modulate time courses origTCS = []; % Original unmodulated time courses TCS = []; % Amplitude-modulated time courses for r = 1:numel(subfMRINames) - runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; + origTCS = [origTCS ; runTCS]; + TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; end % for r = 1:numel(subfMRINames) %% Create original time course and spectral CIFTI files @@ -94,22 +94,22 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% restore variance if ~isempty(clean_VN_Name) - fprintf('Restoring variance\n'); - clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; - mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; - maps = ciftiopen(mapFile, wbcommand); - maps.cdata = maps.cdata .* clean_VN; - ciftisave(maps, mapFile, wbcommand); + fprintf('Restoring variance\n'); + clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata .* clean_VN; + ciftisave(maps, mapFile, wbcommand); end %% divide out vertex area weights if VAweightBool - fprintf('Dividing out vertex area weights\n'); - VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; - mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; - maps = ciftiopen(mapFile, wbcommand); - maps.cdata = maps.cdata ./ VAgray; - ciftisave(maps, mapFile, wbcommand); + fprintf('Dividing out vertex area weights\n'); + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata ./ VAgray; + ciftisave(maps, mapFile, wbcommand); end %% Save individual-level results @@ -123,6 +123,6 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Copy individual PFM maps % Link PROFUMO spatial maps to subject's fsaverage space directory copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); - end % if numel(subfMRINames) ~= 0 + end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) end From 90f5b20745e286c595aa488128826cf39b5f9d44 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 18 Aug 2026 10:38:48 -0500 Subject: [PATCH 67/74] va_norm lives in /T1w --- PFM/PFMPipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index c0944fc73..45019d1e0 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -201,8 +201,8 @@ do if [[ "$VAweightBool" == 1 ]]; then # create VA_norm cifti with volume grayordinates filled with ones areas for weighting - VAnorm=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii - VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii + VAnorm=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii + VAgray=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file wb_command -cifti-separate "${concatOutFile}" COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -crop From 4d19505e3c5c7b707690c90b52e9e71e9dad5edb Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 18 Aug 2026 16:39:00 -0500 Subject: [PATCH 68/74] Feed PostPROFUMO a VarNormBool, instead of a file becuase clean_VN is subject-specific --- PFM/PFMPipeline.sh | 8 +------- PFM/scripts/PostPROFUMO.m | 10 ++++++---- PFM/scripts/PostPROFUMO.sh | 5 +++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 45019d1e0..a90140d15 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -397,12 +397,6 @@ do RESULTS_PATH="${PFMFolder}/Results.ppp" REAL_REF_IMAGE=$(readlink -f "${RefImage}") - if [[ "${VarNormBool}" == 1 ]];then - clean_VN="${StudyFolder}/${Subject}/MNINonLinear/Results/${ConcatName}/${ConcatName}_Atlas${RegString}_${fMRIProcSTRING}_vn.dscalar.nii" - else - clean_VN='''' - fi - # Remove any existing Results.ppp directory # so postprocess_results.py writes fresh output to Results.ppp if [[ -d "${PFMFolder}/Results.ppp" ]] @@ -436,7 +430,7 @@ do --low-res-mesh="$LowResMesh" \ --profumo-tr="$TR" \ --pfm-folder="$PFMFolder" \ - --clean-vn-name="$clean_VN" \ + --variance-normalize="$VarNorm" \ --weight-vertex-areas="$VAweight" \ --matlab-run-mode="$MatlabMode" diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 2a86aa7ef..5f4bca91b 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -1,4 +1,4 @@ -function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder,clean_VN_Name,VAweightBool) +function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder,VarNormBool,VAweightBool) % PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProcSTRING, OutputfMRIName, OutputSTRING, RegString, LowResMesh, TR, PFMFolder) % This function imports PROFUMO results and generates CIFTI-format time courses % and power spectra for each subject. The outputs are used for subsequent @@ -16,13 +16,14 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % LowResMesh - Mesh resolution (e.g., '10' for 10k) % TR - Repetition time in seconds % PFMFolder - Path to PROFUMO results folder -% clean_VN_Name - Path to group variance normalization file for restoring variance (unused if empty) +% VarNormBool - Boolean flag for variance normalization % VAweightBool - Boolean flag for vertex area weighting %% Parse string inputs and initialize Subjlist = strsplit(SubjListRaw, '@'); fMRINames = strsplit(fMRIListRaw, '@'); TR = str2double(TR); +VarNormBool = logical(str2double(VarNormBool)); VAweightBool = logical(str2double(VAweightBool)); wbcommand = 'wb_command'; @@ -93,8 +94,9 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); %% restore variance - if ~isempty(clean_VN_Name) + if VarNormBool fprintf('Restoring variance\n'); + clean_VN_Name = [StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName RegString fMRIProcSTRING '_vn.dscalar.nii']; clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; maps = ciftiopen(mapFile, wbcommand); @@ -105,7 +107,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% divide out vertex area weights if VAweightBool fprintf('Dividing out vertex area weights\n'); - VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; maps = ciftiopen(mapFile, wbcommand); maps.cdata = maps.cdata ./ VAgray; diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index d19122d69..d43cb1851 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -28,7 +28,7 @@ opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" opts_AddMandatory '--concat-name' 'ConcatName' 'string' "concatenated fMRI name if using multi-run data" opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' -opts_AddOptional '--clean-vn-name' 'clean_VN_Name' 'path' "Path to group variance normalization file for restoring variance (default: none)" '''' +opts_AddOptional '--variance-normalize' 'VarNorm' 'YES or NO' "Variance normalize data before PROFUMO (default NO)" 'NO' opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas for PROFUMO (default NO)" 'NO' opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode @@ -37,6 +37,7 @@ opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_m 2 = Octave" "$g_matlab_default_mode" opts_ParseArguments "$@" +VarNormBool=$(opts_StringToBool "$VarNorm") VAweightBool=$(opts_StringToBool "$VAweight") @@ -79,7 +80,7 @@ IFS='@' read -a fMRIList <<<"$fMRIListRaw" this_script_dir=$(dirname "$0") #matlab function arguments converted to strings -matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder" "$clean_VN_Name" "$VAweightBool") +matlab_argarray=("$StudyFolder" "$SubjListRaw" "$fMRIListRaw" "$ConcatName" "$fMRIProcSTRING" "$OutputfMRIName" "$OutputSTRING" "$RegString" "$LowResMesh" "$TR" "$PFMFolder" "$VarNormBool" "$VAweightBool") case "$MatlabMode" in (0) From ae4f2af1672631ba47a780fc499e384176e88ffc Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 19 Aug 2026 15:08:28 -0500 Subject: [PATCH 69/74] Set Variance Normalization and Vertex area weighting defaults to YES. --- PFM/PFMPipeline.sh | 6 +++--- PFM/scripts/PostPROFUMO.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index a90140d15..27bfe427a 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -1,4 +1,4 @@ -cd #!/bin/bash +#!/bin/bash set -eu pipedirguessed=0 @@ -56,8 +56,8 @@ opts_AddOptional '--num-wishart' 'NumWishart' 'integer' "number of Wishart filte # Post-Wishart filter specific parameters (only used if --num-wishart > 0) opts_AddOptional '--keep-wishart-files' 'KeepWishartFiles' 'YES or NO' "keep Wishart-filtered files after PROFUMO instead of deleting (default NO)" 'NO' -opts_AddOptional '--variance-normalization' 'VarNorm' 'YES or NO' "Variance normalize data before PROFUMO (default NO)" 'NO' -opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas before PROFUMO (default NO)" 'NO' +opts_AddOptional '--variance-normalization' 'VarNorm' 'YES or NO' "Variance normalize data before PROFUMO (default YES)" 'YES' +opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas before PROFUMO (default YES)" 'YES' #optional parameters opts_AddOptional '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution, like '32' for 32k_fs_LR" '32' diff --git a/PFM/scripts/PostPROFUMO.sh b/PFM/scripts/PostPROFUMO.sh index d43cb1851..5c29bf014 100755 --- a/PFM/scripts/PostPROFUMO.sh +++ b/PFM/scripts/PostPROFUMO.sh @@ -28,8 +28,8 @@ opts_AddMandatory '--low-res-mesh' 'LowResMesh' 'string' "mesh resolution" opts_AddMandatory '--pfm-folder' 'PFMFolder' 'path' "path to PFM results folder" opts_AddMandatory '--concat-name' 'ConcatName' 'string' "concatenated fMRI name if using multi-run data" opts_AddMandatory '--profumo-tr' 'TR' "repetition time for PROFUMO analysis" '0.72' -opts_AddOptional '--variance-normalize' 'VarNorm' 'YES or NO' "Variance normalize data before PROFUMO (default NO)" 'NO' -opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Weight data by vertex areas for PROFUMO (default NO)" 'NO' +opts_AddOptional '--variance-normalize' 'VarNorm' 'YES or NO' "Data were variance normalized before PROFUMO, so restore varaince (default YES)" 'YES' +opts_AddOptional '--weight-vertex-areas' 'VAweight' 'YES or NO' "Data were weighted by vertex areas before PROFUMO, so restore unweighted state (default YES)" 'YES' opts_AddOptional '--matlab-run-mode' 'MatlabMode' '0, 1, or 2' "defaults to $g_matlab_default_mode 0 = compiled MATLAB From 9e66de157afb026568ea6de8563de7de8c291e30 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Wed, 19 Aug 2026 15:18:04 -0500 Subject: [PATCH 70/74] refactor to fix indentation and rearrange code blocks for readability --- PFM/scripts/PostPROFUMO.m | 155 +++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 5f4bca91b..34ffb32a5 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -36,95 +36,96 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % If ConcatName is specified, use concatenated version; otherwise check individual runs subfMRINames = {}; if ~strcmp(ConcatName, '') - % Multi-run data: check if concatenated dataset exists - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii'],'file') - c = 1; - for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') - subfMRINames{c} = fMRINames{r}; - c = c + 1; + % Multi-run data: check if concatenated dataset exists + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName fMRIProcSTRING '.dtseries.nii'],'file') + c = 1; + for r = 1:numel(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') + subfMRINames{c} = fMRINames{r}; + c = c + 1; + end + end % for r = 1:numel(fMRINames) end - end % for r = 1:numel(fMRINames) - end else - % Single-run data: check which runs exist - c = 1; - for r = 1:numel(fMRINames) - if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') - subfMRINames{c} = fMRINames{r}; - c = c + 1; - end - end % for r = 1:numel(fMRINames) + % Single-run data: check which runs exist + c = 1; + for r = 1:numel(fMRINames) + if exist([StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' fMRINames{r} '/' fMRINames{r} fMRIProcSTRING '.dtseries.nii'],'file') + subfMRINames{c} = fMRINames{r}; + c = c + 1; + end + end % for r = 1:numel(fMRINames) end %% Process subject if valid runs found if numel(subfMRINames) ~= 0 - %% Load and concatenate PFM time courses and amplitudes - % Load PROFUMO outputs and amplitude-modulate time courses - origTCS = []; % Original unmodulated time courses - TCS = []; % Amplitude-modulated time courses - for r = 1:numel(subfMRINames) - runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + %% Load and concatenate PFM time courses and amplitudes + % Load PROFUMO outputs and amplitude-modulate time courses + origTCS = []; % Original unmodulated time courses + TCS = []; % Amplitude-modulated time courses + for r = 1:numel(subfMRINames) + runTCS = load([PFMFolder '/Results.ppp/TimeCourses/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); + runAmp = load([PFMFolder '/Results.ppp/Amplitudes/sub-' Subjlist{s} '_run-' subfMRINames{r} '.csv']); - origTCS = [origTCS ; runTCS]; - TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; - end % for r = 1:numel(subfMRINames) - - %% Create original time course and spectral CIFTI files - % Generate CIFTI structure for unmodulated time courses - PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); - - % Store power spectra - ts.Nnodes = size(origTCS, 2); - ts.Nsubjects = 1; - ts.ts = origTCS; - ts.NtimepointsPerSubject = size(origTCS, 1); - PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - - %% Create time course and spectral CIFTI files - % Generate CIFTI structure for time courses - PFMTCS = cifti_struct_create_sdseries(TCS','step',TR); - - % Store power spectra - ts.Nnodes = size(TCS, 2); - ts.Nsubjects = 1; - ts.ts = TCS; - ts.NtimepointsPerSubject = size(TCS, 1); - PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + origTCS = [origTCS ; runTCS]; + TCS = [TCS ; runTCS .* repmat(runAmp', size(runTCS, 1), 1)]; + end % for r = 1:numel(subfMRINames) + + %% Create original time course and spectral CIFTI files + % Generate CIFTI structure for unmodulated time courses + PFMTCSorig = cifti_struct_create_sdseries(origTCS','step',TR); + + % Store power spectra + ts.Nnodes = size(origTCS, 2); + ts.Nsubjects = 1; + ts.ts = origTCS; + ts.NtimepointsPerSubject = size(origTCS, 1); + PFMSpectraorig = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); + + %% Create time course and spectral CIFTI files + % Generate CIFTI structure for time courses + PFMTCS = cifti_struct_create_sdseries(TCS','step',TR); + + % Store power spectra + ts.Nnodes = size(TCS, 2); + ts.Nsubjects = 1; + ts.ts = TCS; + ts.NtimepointsPerSubject = size(TCS, 1); + PFMSpectra = cifti_struct_create_sdseries(nets_spectra_sp(ts)','step',1/TR); - %% restore variance - if VarNormBool - fprintf('Restoring variance\n'); - clean_VN_Name = [StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName RegString fMRIProcSTRING '_vn.dscalar.nii']; - clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; - mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; - maps = ciftiopen(mapFile, wbcommand); - maps.cdata = maps.cdata .* clean_VN; - ciftisave(maps, mapFile, wbcommand); - end + %% Save non-map results + % Save original and amplitude-modulated time courses and spectra + ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - %% divide out vertex area weights - if VAweightBool - fprintf('Dividing out vertex area weights\n'); - VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; - mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; - maps = ciftiopen(mapFile, wbcommand); - maps.cdata = maps.cdata ./ VAgray; - ciftisave(maps, mapFile, wbcommand); - end + ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - %% Save individual-level results - % Save original and amplitude-modulated time courses and spectra - ciftisave(PFMTCSorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectraorig, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra_orig.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + %% Handle maps + % restore variance + if VarNormBool + fprintf('Restoring variance\n'); + clean_VN_Name = [StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName RegString fMRIProcSTRING '_vn.dscalar.nii']; + clean_VN = ciftiopen(clean_VN_Name, wbcommand).cdata; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata .* clean_VN; + ciftisave(maps, mapFile, wbcommand); + end - ciftisave(PFMTCS, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_ts.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); - ciftisave(PFMSpectra, [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_spectra.' LowResMesh 'k_fs_LR.sdseries.nii'], wbcommand); + % divide out vertex area weights + if VAweightBool + fprintf('Dividing out vertex area weights\n'); + VAgray = ciftiopen([StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.grayordinates.' LowResMesh 'k_fs_LR.dscalar.nii'], wbcommand).cdata; + mapFile = [PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii']; + maps = ciftiopen(mapFile, wbcommand); + maps.cdata = maps.cdata ./ VAgray; + ciftisave(maps, mapFile, wbcommand); + end + + % Copy PFM maps from PFM folder to subject's MNINonLinear/fsaverage_LR space directory + copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); - %% Copy individual PFM maps - % Link PROFUMO spatial maps to subject's fsaverage space directory - copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) end From d419332febb0759fd74101dffa316dea0c2eb849 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 31 Aug 2026 14:47:01 -0500 Subject: [PATCH 71/74] use va_norm from MNINonLinear --- PFM/PFMPipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 27bfe427a..95d54ab46 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -201,8 +201,8 @@ do if [[ "$VAweightBool" == 1 ]]; then # create VA_norm cifti with volume grayordinates filled with ones areas for weighting - VAnorm=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii - VAgray=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii + VAnorm=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii + VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file wb_command -cifti-separate "${concatOutFile}" COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -crop From 9a9962a3ec306b00f7235122695500dd36f54408 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Mon, 31 Aug 2026 16:08:15 -0500 Subject: [PATCH 72/74] Make va_norm with grayordinates a temporary file, like in RSNRegression. --- PFM/PFMPipeline.sh | 10 +++++----- PFM/scripts/PostPROFUMO.m | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 95d54ab46..9fbb5a475 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -200,13 +200,13 @@ do --matlab-run-mode="$MatlabMode" if [[ "$VAweightBool" == 1 ]]; then - # create VA_norm cifti with volume grayordinates filled with ones areas for weighting - VAnorm=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii - VAgray=${StudyFolder}/${Subject}/MNINonLinear/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.grayordinates.${LowResMesh}k_fs_LR.dscalar.nii + # create temporary VA_norm cifti with volume grayordinates filled with ones areas for weighting + VAnorm=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii + tempfiles_create "tmp_VAgray_XXXXXX.nii.gz" tmp_VAgray_file tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file wb_command -cifti-separate "${concatOutFile}" COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -crop - wb_command -cifti-create-dense-from-template "${concatOutFile}" "$VAgray" -cifti "$VAnorm" -volume-all "$tmp_roi_file" -from-cropped + wb_command -cifti-create-dense-from-template "${concatOutFile}" "$tmp_VAgray_file" -cifti "$VAnorm" -volume-all "$tmp_roi_file" -from-cropped fi # Split back into individual runs and restore means @@ -237,7 +237,7 @@ do log_Msg "Weighting data by average vertex areas" wb_command -cifti-math "(TCS * VA)" ${outFile} \ -var TCS ${outFile} \ - -var VA ${VAgray} -select 1 1 -repeat + -var VA ${tmp_VAgray_file} -select 1 1 -repeat fi cumTP=$endIdx diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index 34ffb32a5..bd31cda29 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -59,6 +59,17 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc %% Process subject if valid runs found if numel(subfMRINames) ~= 0 + + if VAweightBool + % create temporary VA_norm cifti with volume grayordinates filled with ones areas for weighting + VAnorm = [StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.' LowResMesh 'k_fs_LR.dscalar.nii']; + tmp_VAgray_file = [tempname '.nii.gz']; + tmp_jnk_file = [tempname '.nii.gz']; + tmp_roi_file = [tempname '.nii.gz']; + system(sprintf('%s -cifti-separate "%s" COLUMN -volume-all "%s" -roi "%s" -crop', wbcommand, concatOutFile, tmp_jnk_file, tmp_roi_file)); + system(sprintf('%s -cifti-create-dense-from-template "%s" "%s" -cifti "%s" -volume-all "%s" -from-cropped', wbcommand, concatOutFile, tmp_VAgray_file, VAnorm, tmp_roi_file)); + end + %% Load and concatenate PFM time courses and amplitudes % Load PROFUMO outputs and amplitude-modulate time courses origTCS = []; % Original unmodulated time courses @@ -126,6 +137,12 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc % Copy PFM maps from PFM folder to subject's MNINonLinear/fsaverage_LR space directory copyfile([PFMFolder '/Results.ppp/Maps/sub-' Subjlist{s} '.dscalar.nii'], [StudyFolder '/' Subjlist{s} '/MNINonLinear/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.' OutputSTRING RegString '_origmaps.' LowResMesh 'k_fs_LR.dscalar.nii']); + % clean up temporary files + if VAweightBool + delete(tmp_VAgray_file); + delete(tmp_roi_file); + delete(tmp_jnk_file); + end end % if numel(subfMRINames) ~= 0 end % for s = 1:numel(Subjlist) end From e233cffeb0a39383b1e45a24234a1f8fcd31af23 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 1 Sep 2026 12:46:05 -0500 Subject: [PATCH 73/74] fix tempfile extension --- PFM/PFMPipeline.sh | 2 +- PFM/scripts/PostPROFUMO.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PFM/PFMPipeline.sh b/PFM/PFMPipeline.sh index 9fbb5a475..4c4700ccf 100755 --- a/PFM/PFMPipeline.sh +++ b/PFM/PFMPipeline.sh @@ -202,7 +202,7 @@ do if [[ "$VAweightBool" == 1 ]]; then # create temporary VA_norm cifti with volume grayordinates filled with ones areas for weighting VAnorm=${StudyFolder}/${Subject}/T1w/fsaverage_LR${LowResMesh}k/${Subject}.midthickness${RegString}_va_norm.${LowResMesh}k_fs_LR.dscalar.nii - tempfiles_create "tmp_VAgray_XXXXXX.nii.gz" tmp_VAgray_file + tempfiles_create "tmp_VAgray_XXXXXX.dscalar.nii" tmp_VAgray_file tempfiles_create "tmp_jnk_XXXXXX.nii.gz" tmp_jnk_file tempfiles_create "tmp_roi_XXXXXX.nii.gz" tmp_roi_file wb_command -cifti-separate "${concatOutFile}" COLUMN -volume-all "$tmp_jnk_file" -roi "$tmp_roi_file" -crop diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index bd31cda29..a1d3e7da3 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -63,7 +63,7 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc if VAweightBool % create temporary VA_norm cifti with volume grayordinates filled with ones areas for weighting VAnorm = [StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.' LowResMesh 'k_fs_LR.dscalar.nii']; - tmp_VAgray_file = [tempname '.nii.gz']; + tmp_VAgray_file = [tempname '.dscalar.nii']; tmp_jnk_file = [tempname '.nii.gz']; tmp_roi_file = [tempname '.nii.gz']; system(sprintf('%s -cifti-separate "%s" COLUMN -volume-all "%s" -roi "%s" -crop', wbcommand, concatOutFile, tmp_jnk_file, tmp_roi_file)); From 9b882712d7791adda03a0a0d0d969fe194948115 Mon Sep 17 00:00:00 2001 From: Burke Rosen Date: Tue, 1 Sep 2026 16:28:48 -0500 Subject: [PATCH 74/74] fix cifti template for creating VAgray --- PFM/scripts/PostPROFUMO.m | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PFM/scripts/PostPROFUMO.m b/PFM/scripts/PostPROFUMO.m index a1d3e7da3..346a34700 100755 --- a/PFM/scripts/PostPROFUMO.m +++ b/PFM/scripts/PostPROFUMO.m @@ -62,12 +62,13 @@ function PostPROFUMO(StudyFolder, SubjListRaw, fMRIListRaw, ConcatName, fMRIProc if VAweightBool % create temporary VA_norm cifti with volume grayordinates filled with ones areas for weighting + ciftiTemplate = [StudyFolder '/' Subjlist{s} '/MNINonLinear/Results/' ConcatName '/' ConcatName RegString fMRIProcSTRING '_vn.dscalar.nii']; % use clean_VN as cifti template VAnorm = [StudyFolder '/' Subjlist{s} '/T1w/fsaverage_LR' LowResMesh 'k/' Subjlist{s} '.midthickness' RegString '_va_norm.' LowResMesh 'k_fs_LR.dscalar.nii']; tmp_VAgray_file = [tempname '.dscalar.nii']; tmp_jnk_file = [tempname '.nii.gz']; tmp_roi_file = [tempname '.nii.gz']; - system(sprintf('%s -cifti-separate "%s" COLUMN -volume-all "%s" -roi "%s" -crop', wbcommand, concatOutFile, tmp_jnk_file, tmp_roi_file)); - system(sprintf('%s -cifti-create-dense-from-template "%s" "%s" -cifti "%s" -volume-all "%s" -from-cropped', wbcommand, concatOutFile, tmp_VAgray_file, VAnorm, tmp_roi_file)); + system(sprintf('%s -cifti-separate "%s" COLUMN -volume-all "%s" -roi "%s" -crop', wbcommand, ciftiTemplate, tmp_jnk_file, tmp_roi_file)); + system(sprintf('%s -cifti-create-dense-from-template "%s" "%s" -cifti "%s" -volume-all "%s" -from-cropped', wbcommand, ciftiTemplate, tmp_VAgray_file, VAnorm, tmp_roi_file)); end %% Load and concatenate PFM time courses and amplitudes