diff --git a/CHANGELOG.rst b/CHANGELOG.rst index db07ead..aca027f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,10 +6,13 @@ Change Log This document records the main changes to the sdss_access code. -Unreleased +3.0.11 (06-03-2026) ---------- - PR `84` - Add `Path.mos_target_num` method. - PR `85` - Replace `pkg_resources.parse_version` with `packaging.version.parse` in `conf.py`. +- PR `87` - Fix `extract` method for MOS paths +- PR `88` - Support `*` wildcard in MOS paths +- PR `90` - Doc edits 3.0.10 (07-10-2025) ------------------- diff --git a/docs/sphinx/intro.rst b/docs/sphinx/intro.rst index b5871db..85872c3 100644 --- a/docs/sphinx/intro.rst +++ b/docs/sphinx/intro.rst @@ -38,10 +38,10 @@ The full paths are concrete paths. # import the path from sdss_access import Path - mypath = Path(release='dr17') + path = Path(release='dr17') # generate a file system path - mypath.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') + path.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') '/Users/Brian/Work/sdss/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' Note that this only generates a path. The file may not actually exist locally. If you want to generate a URL path to @@ -49,7 +49,7 @@ the file on the SAS at Utah, you can use `Path.url <.BasePath.url>`. :: # generate a http path to the file - mypath.url('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') + path.url('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') 'https://data.sdss.org/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' You can also pass in the full path directly as a string in cases. In those cases, the first argument passed in must @@ -57,8 +57,8 @@ be an empty string. :: # pass in the full path directly to path.url - full = mypath.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') - mypath.url('', full=full) + full = path.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') + path.url('', full=full) 'https://data.sdss.org/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' Path Names @@ -72,23 +72,23 @@ are defined inside the SDSS ``tree`` product, within a `[PATHS]` section in the or `data/dr15.cfg`. Within ``sdss_access``, all paths are available as a dictionary, ``path.templates``:: from sdss_access.path import Path - mypath = Path(release='dr17') + path = Path(release='dr17') # show the dictionary of available paths - mypath.templates + path.templates To look up what path names are available, you can use `Path.lookup_names <.BasePath.lookup_names>`. :: # look up the available path names - mypath.lookup_names() + path.lookup_names() ['BOSSLyaDR_cat', ..., 'mangacube', ..., 'xdqso_index'] To look up what keywords are needed for a given path, you can use `Path.lookup_keys <.BasePath.lookup_keys>`. :: # look up the keyword arguments needed to define a MaNGA cube path - mypath.lookup_keys('mangacube') + path.lookup_keys('mangacube') ['plate', 'drpver', 'ifu', 'wave'] The full list of paths can also be found :ref:`here `. To create a new path, see @@ -100,11 +100,11 @@ the ``remote`` keyword argument :: # check for local path existence - mypath.exists('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') + path.exists('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') True # check for remote path existence on the SAS - mypath.exists('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG', remote=True) + path.exists('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG', remote=True) True Required Keywords @@ -115,13 +115,13 @@ are required. Not specifying all the keywords will result in an error raised. :: - >>> mypath = Path(release='dr17') + >>> path = Path(release='dr17') >>> # see the required keys - >>> mypath.lookup_keys('mangacube') + >>> path.lookup_keys('mangacube') ['plate', 'drpver', 'wave', 'ifu'] - >>> mypath.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901') + >>> path.full('mangacube', drpver='v3_1_1', plate='8485', ifu='1901') KeyError: "Missing required keyword arguments: ['wave']" Environment Paths @@ -134,8 +134,8 @@ paths relevant to that environment. >>> # load the default environment / paths >>> from sdss_access.path import Path - >>> mypath = Path() - >>> mypath + >>> path = Path() + >>> path >> # load the SDSS-V environment and paths >>> from sdss_access.path import Path - >>> mypath = Path(release='sdsswork') - >>> mypath + >>> path = Path(release='sdsswork') + >>> path >> # switch to the environment for public data release DR17 - >>> mypath = Path(release='DR17') - >>> mypath + >>> path = Path(release='DR17') + >>> path >> # load the SDSS-V environment but preserve your original os.environ - >>> mypath = Path(release='sdsswork', preserve_envvars=True) + >>> path = Path(release='sdsswork', preserve_envvars=True) Alternatively, you can preserve a subset of enviroment variables from your original ``os.environ`` by passing in a list of environment variables. :: >>> # preserve only a single environment variable - >>> mypath = Path(release='sdsswork', preserve_envvars=['ROBOSTRATEGY_DATA']) + >>> path = Path(release='sdsswork', preserve_envvars=['ROBOSTRATEGY_DATA']) If you wish to permanently preserve your locally set environment variables, you can set the ``preserve_envvars`` parameter to ``true`` in a custom tree YAML configuration file located at ``~/.config/sdss/sdss_access.yml``. For example @@ -195,8 +195,8 @@ must match the SAS directory structure, and have the relevant environment variab >>> filepath = '/Users/Brian/Work/sdss/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' >>> # extract the keywords - >>> mypath = Path(release='dr17') - >>> mypath.extract('mangacube', filepath) + >>> path = Path(release='dr17') + >>> path.extract('mangacube', filepath) {'drpver': 'v3_1_1', 'plate': '8485', 'ifu': '1901', 'wave': 'LOG'} @@ -211,7 +211,6 @@ home directory, as a new ``sas`` directory. for more information. sdss_access has four classes designed to facilitate access to SAS data. -Most users, will only need to use the `.Access` class. - **Access** - class that automatically decides between `.RsyncAccess` and `.CurlAccess` based on the operating system. - **HttpAccess** - uses the `urllib` package to download data using a direct http request @@ -219,43 +218,45 @@ Most users, will only need to use the `.Access` class. - **CurlAccess** - uses `curl` to download data. This is the only available method for use on Windows machines. Note that all remote access classes, after instantiation, must call the `Access.remote <.BaseAccess.remote>` method before -adding paths to ensure successful downloading of data. +adding paths to ensure successful downloading of data. - -Using the `.Access` class. Depending on your operating system, ``posix`` or not, Access will either create itself using -`.RsyncAccess` or `.CurlAccess`, and behave as either object. Via `.Acccess`, Windows machines will always use `.CurlAccess`, +Most users, will only need to use the `.Access` class. Depending on your operating system, +``posix`` or not, Access will either create itself using `.RsyncAccess` or `.CurlAccess`, and +behave as either object. Via `.Acccess`, Windows machines will always use `.CurlAccess`, while Linux or Macs will automatically utilize `.RsyncAccess`. + +Using the `.Access` class. :: # import the access class from sdss_access import Access - myaccess = Access(release='DR17') + access = Access(release='DR17') # the access mode is automatically set to rsync. - print(myaccess) + print(access) >>> # the class now behaves exactly like RsyncAccess. # download a MaNGA cube - myaccess.remote() + access.remote() # See the above section on "Path Names" for how to find the required keywords for mangacube (i.e. drpver, plate, ifu) - myaccess.add('mangacube', drpver='v3_1_1', plate='8485', ifu='1901') - myaccess.set_stream() - myaccess.commit() + access.add('mangacube', drpver='v3_1_1', plate='8485', ifu='1901') + access.set_stream() + access.commit() # download a mos_target_sdss_id_flat fits file - myaccess = Access(release="DR20") - myaccess.remote() - myaccess.add("mos_target_sds_id_flat", v_targ=V_TARG, ftype="fits", num="*") - myaccess.set_stream() - myaccess.commit() + access = Access(release="DR20") + access.remote() + access.add("mos_target_sds_id_flat", v_targ=V_TARG, ftype="fits", num="*") + access.set_stream() + access.commit() # download a mos_target_sdss_id_flat parquet file - myaccess = Access(release="DR20") - myaccess.remote() - myaccess.add("mos_target_sdss_id_flat", v_targ=V_TARG, ftype="parquet", num="*") - myaccess.set_stream() - myaccess.commit() + access = Access(release="DR20") + access.remote() + access.add("mos_target_sdss_id_flat", v_targ=V_TARG, ftype="parquet", num="*") + access.set_stream() + access.commit() In all cases, successful ``sdss_access`` downloads will return a code of 0. Any other number indicates that a problem @@ -264,19 +265,20 @@ files within the temporary directory. As noted above, most users will only need to use the `.Access` class. Use the `.HttpAccess`, `.RsyncAccess`, and `.CurlAccess` classes only if you have some specific reason to use them instead of the `.Access` class. -Note that the syntax for `.HttpAccess` is different from the syntax for `.Access`, `.RsyncAccess`, and `.CurlAccess`. `.Access` only chooses between `.RsyncAccess` and `.CurlAccess`. `.Access` does not select `.HttpAccess`. `.HttpAccess` is a different download mechanism. +Note that the syntax for `.HttpAccess` is different from the syntax for `.Access`, `.RsyncAccess`, and `.CurlAccess`. +`.HttpAccess` uses a different protocol for downloading files than `rsync` or `curl`. Using the `.HttpAccess` class. :: from sdss_access import HttpAccess - my_http_access = HttpAccess(release='DR17', verbose=True) + http_access = HttpAccess(release='DR17', verbose=True) # set to use remote - my_http_access.remote() + http_access.remote() # get the file - my_http_access.get('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') + http_access.get('mangacube', drpver='v3_1_1', plate='8485', ifu='1901', wave='LOG') Using the `.RsyncAccess` class. `.RsyncAccess` is generally much faster then `.HttpAccess` as it spreads multiple file downloads across multiple continuous rsync download streams. @@ -285,20 +287,20 @@ file downloads across multiple continuous rsync download streams. # import the rsync class from sdss_access import RsyncAccess - myrsync = RsyncAccess(release='DR17') + rsync = RsyncAccess(release='DR17') # sets a remote mode to the real SAS - myrsync.remote() + rsync.remote() # add all the file(s) you want to download # let's download all DR17 MaNGA cubes for plate 8485 - myrsync.add('mangacube', drpver='v3_1_1', plate='8485', ifu='*', wave='LOG') + rsync.add('mangacube', drpver='v3_1_1', plate='8485', ifu='*', wave='LOG') # set the stream tasks - myrsync.set_stream() + rsync.set_stream() # start the download(s) - myrsync.commit() + rsync.commit() Using the `.CurlAccess` class. `.CurlAccess` behaves exactly the same way as `.RsyncAccess`. After importing and instantiating a `.CurlAccess` object, all methods and behavior are the same as in the `.RsyncAccess` class. @@ -306,7 +308,7 @@ instantiating a `.CurlAccess` object, all methods and behavior are the same as i # import the curl class from sdss_access import CurlAccess - mycurl = CurlAccess(release='DR17') + curl = CurlAccess(release='DR17') @@ -321,16 +323,16 @@ The ``input_type`` keyword specifies the type of path input. :: from sdss_access import RsyncAccess - myrsync = RsyncAccess(release='DR17') - myrsync.remote() + rsync = RsyncAccess(release='DR17') + rsync.remote() # add a url to the stream for download url = 'rsync://dtn.sdss.org/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' - myrsync.add_file(f, input_type='url') + rsync.add_file(f, input_type='url') # add a file to the stream for download path = '/Users/Brian/Work/sdss/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1902-LOGCUBE.fits.gz' - myrsync.add_file(path, input_type='filepath') + rsync.add_file(path, input_type='filepath') Following Symlinks ^^^^^^^^^^^^^^^^^^ @@ -341,13 +343,13 @@ can disable this behaviour by setting the ``follow_symlinks`` flag to False. :: from sdss_access import RsyncAccess - myrsync = RsyncAccess(release='DR17') - myrsync.remote() - myrsync.add('mangacube', drpver='v3_1_1', plate='8485', ifu='*', wave='LOG') - myrsync.set_stream() + rsync = RsyncAccess(release='DR17') + rsync.remote() + rsync.add('mangacube', drpver='v3_1_1', plate='8485', ifu='*', wave='LOG') + rsync.set_stream() # disable follow_symlinks - myrsync.commit(follow_symlinks=False) + rsync.commit(follow_symlinks=False) Accessing SDSS-V Products @@ -359,8 +361,8 @@ When specifying ``release="sdss5"``, you may notice the new server location, e.g :: >>> from sdss_access import Access - >>> myaccess = Access() - >>> myaccess + >>> access = Access() + >>> access As with SDSS-IV, ``sdss_access`` requires valid authentication to download @@ -373,11 +375,11 @@ the robostrategy completeness files for SDSS-V. :: from sdss_access import Access - myaccess = Access() - myaccess.remote() - myaccess.add('rsCompleteness', observatory='apo', plan='epsilon-2-core-*') - myaccess.set_stream() - myaccess.commit() + access = Access() + access.remote() + access.add('rsCompleteness', observatory='apo', plan='epsilon-2-core-*') + access.set_stream() + access.commit() .. note:: As of ``version >= 3.0.0``, and ``tree >= 4.0.0`` the default config of "sdsswork" is for SDSS-V @@ -396,18 +398,18 @@ the ``public`` keyword. # import the path and set it to use the DR17 release from sdss_access.path import Path - mypath = Path(release='DR17') + path = Path(release='DR17') # check if a public path - mypath.public + path.public True # generate a file system path - mypath.full('mangacube', drpver='v3_1_1', plate=8485, ifu=1901, wave='LOG') + path.full('mangacube', drpver='v3_1_1', plate=8485, ifu=1901, wave='LOG') '/Users/Brian/Work/sdss/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1901-LOGCUBE.fits.gz' # setup rsync access to download public data from DR17 - myrsync = RsyncAccess(public=True, release='DR17') + rsync = RsyncAccess(public=True, release='DR17') .. _sdss-access-svn: @@ -425,11 +427,11 @@ it uses the local path definition, and for urls, it uses the correct ``svn.sdss. from sdss_access.path import Path # load the paths for DR17 - mypath = Path(release='DR17') - mypath.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') + path = Path(release='DR17') + path.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') '/Users/Brian/Work/sdss/data/manga/mangapreim/v2_9/data/D0084XX/8405/preimage-1-42007_irg.jpg' - mypath.url('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') + path.url('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') 'https://svn.sdss.org/public/data/manga/mangapreim/tags/v2_9/data/D0084XX/8405/preimage-1-42007_irg.jpg' As always, paths generated by ``tree`` and ``sdss_access`` use the directory structure as it exists on the SDSS @@ -450,20 +452,20 @@ the ``force_module`` keyword. :: # load the paths for DR17 - mypath = Path(release='DR17') - mypath.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') + path = Path(release='DR17') + path.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') '/Users/Brian/Work/sdss/data/manga/mangapreim/v2_9/data/D0084XX/8405/preimage-1-42007_irg.jpg' # Override the path to use my local module - mypath.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007', force_module=True) + path.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007', force_module=True) '/Users/Brian/Work/sdss/data/manga/mangapreim/trunk/data/D0084XX/8405/preimage-1-42007_irg.jpg' If you want to always override paths with any local modules found, you can set the ``force_modules`` keyword on ``Path`` instantiation. :: - mypath = Path(release='DR17', force_modules=True) - mypath.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') + path = Path(release='DR17', force_modules=True) + path.full('mangapreimg', designid=8405, designgrp='D0084XX', mangaid='1-42007') '/Users/Brian/Work/sdss/data/manga/mangapreim/trunk/data/D0084XX/8405/preimage-1-42007_irg.jpg' You can also set the ``force_modules`` parameter in your custom config file, ``~/.config/sdss/sdss_access.yml`` to diff --git a/docs/sphinx/path_defs.rst b/docs/sphinx/path_defs.rst index e3ad296..764fe0e 100644 --- a/docs/sphinx/path_defs.rst +++ b/docs/sphinx/path_defs.rst @@ -17,6 +17,15 @@ SDSSWORK :prog: sdsswork :templates: +.. _dr20: + +DR20 +---- + +.. datamodel:: sdss_access.path.path:Path + :prog: DR20 + :templates: + .. _dr19: DR19 @@ -150,4 +159,13 @@ IPL3 .. datamodel:: sdss_access.path.path:Path :prog: IPL3 + :templates: + +.. _ipl4: + +IPL4 +---- + +.. datamodel:: sdss_access.path.path:Path + :prog: IPL4 :templates: \ No newline at end of file diff --git a/docs/sphinx/path_evolution.rst b/docs/sphinx/path_evolution.rst index 557117b..5ed1d88 100644 --- a/docs/sphinx/path_evolution.rst +++ b/docs/sphinx/path_evolution.rst @@ -10,6 +10,10 @@ releases (DRs) and internal SDSS data releases, e.g. MaNGA Product Launches (MPL Public Data Releases -------------------- +.. changelog:: sdss_access.path.changelog:compute_changelog + :prog: changes + :drs: dr20, dr19 + .. changelog:: sdss_access.path.changelog:compute_changelog :prog: changes :drs: dr19, dr18 @@ -60,6 +64,10 @@ Internal Data Releases Internal Product Launches (IPLs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. changelog:: sdss_access.path.changelog:compute_changelog + :prog: changes + :drs: ipl4, ipl3 + .. changelog:: sdss_access.path.changelog:compute_changelog :prog: changes :drs: ipl3, ipl2 diff --git a/docs/sphinx/paths.rst b/docs/sphinx/paths.rst index 87331cb..7df26ed 100644 --- a/docs/sphinx/paths.rst +++ b/docs/sphinx/paths.rst @@ -14,7 +14,8 @@ Current Working Paths Public Data Release Paths -* :ref:`DR19 ` - paths defined for public data release 19 +* :ref:`DR20 ` - paths defined for public data release 20 +* :ref:`DR19 ` * :ref:`DR18 ` * :ref:`DR17 ` * :ref:`DR16 ` @@ -32,6 +33,7 @@ Internal SDSS Release Paths * :ref:`IPL-1 ` - Internal Product Launch 1 * :ref:`IPL-2 ` - Internal Product Launch 2 * :ref:`IPL-3 ` - Internal Product Launch 3 +* :ref:`IPL-4 ` - Internal Product Launch 4 Latest Path Evolution