diff --git a/rip_docker.sh b/rip_docker.sh index ccc0319..34952db 100755 --- a/rip_docker.sh +++ b/rip_docker.sh @@ -13,28 +13,53 @@ if [ "$#" -ne 2 ]; then exit -1 fi -# Flags determined by examining docker-wine with: -# --as-me -# --workdir -# --xvfb -# --name -# https://github.com/scottyhardy/docker-wine/blob/master/docker-wine -docker run \ - -it \ - --rm \ - --volume=${2}:/data \ - --env=USER_NAME=${USER} \ - --env=USER_UID=$(id -u ${USER}) \ - --env=USER_GID=$(id -g ${USER}) \ - --env=USER_HOME=${HOME} \ - --workdir=/home/${USER} \ - --env=USE_XVFB=yes \ - --env=XVFB_SERVER=:95 \ - --env=XVFB_SCREEN=0 \ - --env=XVFB_RESOLUTION=320x240x8 \ - --env=DISPLAY=:95 \ - --hostname=bruker-ripper \ - --name=bruker-ripper \ - --shm-size=1g \ - --env=TZ=America/Los_Angeles \ - ${1} +## Support running in parallel: +# https://stackoverflow.com/questions/30332137/xvfb-run-unreliable-when-multiple-instances-invoked-in-parallel + +# allow settings to be updated via environment +: "${xvfb_lockdir:=$HOME/.xvfb-locks}" +: "${xvfb_display_min:=99}" +: "${xvfb_display_max:=599}" + +# assuming only one user will use this, let's put the locks in our own home directory +# avoids vulnerability to symlink attacks. +mkdir -p -- "$xvfb_lockdir" || exit + +i=$xvfb_display_min # minimum display number +while (( i < xvfb_display_max )); do + if [ -f "/tmp/.X$i-lock" ]; then # still avoid an obvious open display + (( ++i )); continue + fi + exec 5>"$xvfb_lockdir/$i" || continue # open a lockfile + if flock -x -n 5; then # try to lock it + # if locked, run + + # Flags determined by examining docker-wine with: + # --as-me + # --workdir + # --xvfb + # --name + # https://github.com/scottyhardy/docker-wine/blob/master/docker-wine + exec docker run \ + -i \ + --rm \ + --volume=${2}:/data \ + --env=USER_NAME=${USER} \ + --env=USER_UID=$(id -u ${USER}) \ + --env=USER_GID=$(id -g ${USER}) \ + --env=USER_HOME=${HOME} \ + --workdir=/home/${USER} \ + --env=USE_XVFB=yes \ + --env=XVFB_SERVER=:$i \ + --env=XVFB_SCREEN=0 \ + --env=XVFB_RESOLUTION=320x240x8 \ + --env=DISPLAY=:$i \ + --hostname=bruker-ripper \ + --name=bruker-ripper-$i \ + --shm-size=1g \ + --env=TZ=America/Los_Angeles \ + ${1} + fi + (( i++ )) +done + diff --git a/two-photon/process.py b/two-photon/process.py index 25b8c3a..cb04e46 100644 --- a/two-photon/process.py +++ b/two-photon/process.py @@ -121,7 +121,7 @@ def get_dirname_hdf5(sess_name, rec_name): prev_data = get_dirname_hdf5(sn, rn) / 'data' / 'data.h5' data_files.append(prev_data) data_files.append(fname_hdf5) - run_suite2p(data_files, dirname_output, mdata) + run_suite2p(data_files, dirname_output, mdata, args.ops_suite2p) if args.backup_output: backup(dirname_output, dirname_backup / 'output') @@ -156,8 +156,7 @@ def preprocess(basename_input, dirname_output, fname_csv, fname_uncorrected, fna stim_channel_name, settle_time): """Main method for running processing of TIFF files into HDF5.""" size = mdata['size'] - - df_voltage = pd.read_csv(fname_csv, index_col='Time(ms)', skipinitialspace=True) + df_voltage = pd.read_csv(fname_csv, skipinitialspace=True) logger.info('Read voltage recordings from: %s, preview:\n%s', fname_csv, df_voltage.head()) fname_frame_start = dirname_output / 'frame_start.h5' frame_start = artefacts.get_frame_start(df_voltage, fname_frame_start) @@ -241,13 +240,19 @@ def run_cmd(cmd, expected_returncode, shell=False): logger.info('Output:\n%s', result.stdout.decode('utf-8')) -def run_suite2p(hdf5_list, dirname_output, mdata): +def run_suite2p(hdf5_list, dirname_output, mdata, ops_file=None): z_planes = mdata['size']['z_planes'] fs_param = 1. / (mdata['period'] * z_planes) # Load suite2p only right before use, as it has a long load time. + import suite2p from suite2p import run_s2p - default_ops = run_s2p.default_ops() + if ops_file is None: + ops = suite2p.default_ops() + else: + # from: + # https://github.com/MouseLand/suite2p/blob/4b6c3a95b53e5581dbab1feb26d67878db866068/suite2p/gui/rungui.py#L472 + ops = np.load(ops_file, allow_pickle=True).item() params = { 'input_format': 'h5', 'data_path': [str(f.parent) for f in hdf5_list], @@ -264,7 +269,7 @@ def run_suite2p(hdf5_list, dirname_output, mdata): logger.info('Running suite2p on files:\n%s\n%s', '\n'.join(str(f) for f in hdf5_list), params) with open(dirname_output / 'recording_order.json', 'w') as fout: json.dump([str(e) for e in hdf5_list], fout, indent=4) - run_s2p.run_s2p(ops=default_ops, db=params) + run_s2p(ops=ops, db=params) def parse_args(): @@ -295,6 +300,8 @@ def parse_args(): help='Top level directory for SLM setup data') group.add_argument('--output_dir', type=pathlib.Path, help='Top level directory of data processing') + group.add_argument('--suite2p-ops', type=pathlib.Path, help='.ops file for suite2p', default=None) + group.add_argument('--recording', type=str, help=('Name of a recording, given as a slash separated id of SESSION/RECORDING/OPTIONAL_PREFIX ' diff --git a/two-photon/rip.py b/two-photon/rip.py index 4d2ef1b..47b435b 100644 --- a/two-photon/rip.py +++ b/two-photon/rip.py @@ -15,8 +15,8 @@ # Ripping process does not end cleanly, so the filesystem is polled to detect the # processing finishing. The following variables relate to the timing of that polling # process. -RIP_TOTAL_WAIT_SECS = 3600 # Total time to wait for ripping before killing it. -RIP_EXTRA_WAIT_SECS = 10 # Extra time to wait after ripping is detected to be done. +RIP_TOTAL_WAIT_SECS = 60*60*10 # Total time to wait for ripping before killing it. +RIP_EXTRA_WAIT_SECS = 60*60*10 # Extra time to wait after ripping is detected to be done. RIP_POLL_SECS = 10 # Time to wait between polling the filesystem. @@ -35,6 +35,7 @@ def determine_ripper(data_dir, ripper_dir): version = root.attrib['version'] # Prairie View versions are given in the form A.B.C.D. + # TODO: allow match when minor version mismatches, eg fall back to `version = '5.5.1.1'` match = re.match(r'^(?P\d+\.\d+)\.\d+\.\d+$', version) if not match: raise RippingError('Could not parse version (expected A.B.C.D): %s' % version) diff --git a/two-photon/transform.py b/two-photon/transform.py index 72917c7..1670d41 100644 --- a/two-photon/transform.py +++ b/two-photon/transform.py @@ -45,7 +45,7 @@ def convert(data, fname_data, df_artefacts=None, fname_uncorrected=None): logger.info('Writing corrected data to %s', fname_data) with h5py.File(fname_uncorrected, 'r') as hfile: - arr = da.from_array(hfile[HDF5_KEY]) + arr = da.from_array(hfile[HDF5_KEY], chunks=('auto', -1, -1, -1)) # Depth of 1 in the first coordinate means to bring in the frames before and after # the chunk -- needed for doing diffs. depth = (1, 0, 0, 0)