Skip to content

gridding updates #248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smwoodman wants to merge 3 commits into
base: main
Choose a base branch
from
Open

gridding updates #248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0704ba4
update gridded
smwoodman Jul 7, 2026
1341940
add distance over ground processing role
smwoodman Jul 8, 2026
f2de423
use consistent variable name
smwoodman Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 36 additions & 11 deletions pyglider/ncprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,9 @@ def make_gridfiles(
dz : float, default = 1
Vertical grid spacing in meters.

starttime : str, default = '1970-01-01'
Start time for gridding. Data before this time is ignored.

maskfunction : callable or None, optional
Function applied to the dataset before gridding,
usually to choose what data will be set to NaN based on quality flags.
Expand All @@ -250,7 +253,10 @@ def make_gridfiles(
with subsequent files overwriting previous files.

Note:
By default, the arithmetic mean is used to bin all variables, except for those with
By default, the arithmetic mean is used to bin all variables, except for
distance_over_ground (never gridded), some profile variables
(profile_start_time, profile_end_time, and profile_direction,
which are gridded using min, max, and mode, respectively), or variables with
an average_method attribute inherited from the timeseries. This attribute is specified
in the YAML configuration file when the timeseries is created. For example, if a variable
has average_method: geometric mean, the geometric mean is used when gridding that variable.
Expand All @@ -272,6 +278,8 @@ def make_gridfiles(
profile_index_varname = utils._resolve_role(ds, varnames, 'profile_index')
lat_varname = utils._resolve_role(ds, varnames, 'latitude')
lon_varname = utils._resolve_role(ds, varnames, 'longitude')
profile_direction_varname = utils._resolve_role(ds, varnames, 'profile_direction')
dog_varname = utils._resolve_role(ds, varnames, 'distance_over_ground')

if maskfunction is not None:
ds = maskfunction(ds)
Expand Down Expand Up @@ -323,19 +331,26 @@ def make_gridfiles(
'comment': 'center of depth bins',
}

# Bin by profile index, for the mean time, lat, and lon values for each profile
# Bin by profile index, for the mean time, lat, and lon values for each profile
ds['time_1970'] = xr.DataArray(
ds.time.values.astype(np.float64), dims=['time'], attrs={}
)

for td in ('time_1970', lon_varname, lat_varname):
for td in ('time_1970', lon_varname, lat_varname, profile_direction_varname):

good = np.where(~np.isnan(ds[td]) & (ds[profile_index_varname] % 1 == 0))[0]
if len(good) > 1:
if td == profile_direction_varname:
method = (lambda x: stats.mode(x, keepdims=True, nan_policy='omit')[0][0])
ds[td].attrs["average_method"] = "mode"
else:
method = 'mean'
ds[td].attrs["average_method"] = "arithmetic mean"

dat, xedges, binnumber = stats.binned_statistic(
ds[profile_index_varname].values[good],
ds[td].values[good],
statistic='mean',
statistic=method,

@smwoodman smwoodman Jul 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Used method to be consistent with the for k in ds.keys(): block, even though the for td, bin_stat in profile_lookup.items(): uses bin_stat as the varname instead.

Update: I also switched from bin_stat to method in the for td, bin_stat in profile_lookup.items(): block, to be consistent throughout the function.

bins=[profile_bins],
)
if td == 'time_1970':
Expand All @@ -350,41 +365,51 @@ def make_gridfiles(
# Bin by profile index, for the profile start (min) and end (max) times
profile_lookup = {'profile_time_start': "min", 'profile_time_end': "max"}
good = np.where(~np.isnan(ds['time']) & (ds[profile_index_varname] % 1 == 0))[0]
for td, bin_stat in profile_lookup.items():
_log.debug(f'td, bin_stat {td}, {bin_stat}')
for td, method in profile_lookup.items():
_log.debug(f'td, method {td}, {method}')
attrs = profile_meta.get(td, {})
attrs['average_method'] = method

dat, xedges, binnumber = stats.binned_statistic(
ds[profile_index_varname].values[good],
ds['time_1970'].values[good],
statistic=bin_stat,
statistic=method,
bins=[profile_bins],
)
dat = dat.astype('timedelta64[ns]') + np.datetime64('1970-01-01T00:00:00')
_log.info(f'{td} {len(dat)}')
dsout[td] = ((xdimname), dat, profile_meta.get(td, {}))
dsout[td] = ((xdimname), dat, attrs)

ds = ds.drop_vars('time_1970')
_log.info(f'Done times!')
_log.info('Done times!')


skip_vars = {'time', lat_varname, lon_varname, depth_varname, profile_index_varname}
skip_vars = {
'time', lat_varname, lon_varname, depth_varname, profile_index_varname,
profile_direction_varname, dog_varname
}
for k in ds.keys():
if k in skip_vars or 'time' in k:
continue
if not np.issubdtype(ds[k].dtype, np.number):
continue
if 'average_method' in ds[k].attrs.keys() and ds[k].attrs['average_method'] == 'none':
_log.debug('Skipping %s because average_method is none', k)
continue
_log.info('Gridding %s', k)
good = np.where(~np.isnan(ds[k]) & (ds[profile_index_varname] % 1 == 0))[0]
if len(good) <= 0:
continue
if 'QC_protocol' in ds[k].attrs.values():
method = np.nanmax
else:
if 'average_method' in ds[k].attrs.values():
if 'average_method' in ds[k].attrs.keys():
method = ds[k].attrs['average_method']
if method == 'geometric mean':
method = stats.gmean
else:
method = 'mean'
ds[k].attrs['average_method'] = 'arithmetic mean'

dat, xedges, yedges, binnumber = stats.binned_statistic_2d(
ds[profile_index_varname].values[good],
Expand Down
4 changes: 2 additions & 2 deletions pyglider/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,12 +868,12 @@ def _get_varnames(deployment):
the relevant derived variable; do not need ``processing_role`` if
``processing_method`` entries name their inputs explicitly:
``temperature``, ``conductivity``, ``salinity``, ``profile_direction``,
``oxygen_concentration``.
``oxygen_concentration``, ``distance_over_ground``.
"""
known_roles = {
'time', 'latitude', 'longitude', 'pressure', 'temperature',
'conductivity', 'salinity', 'depth', 'profile_index',
'profile_direction', 'oxygen_concentration',
'profile_direction', 'oxygen_concentration', "distance_over_ground",
}
ncvar = deployment.get('netcdf_variables', {})
varnames = {}
Expand Down
1 change: 1 addition & 0 deletions tests/example-data/example-seaexplorer/deploymentRealtime_og10.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,7 @@ netcdf_variables:
distance_over_ground:
latitude: LATITUDE
longitude: LONGITUDE
processing_role: distance_over_ground
long_name: Distance over ground flown since mission start
units: km

Expand Down
Binary file modified tests/example-data/example-slocum/L0-gridfiles/dfo-rosie713-20190615_grid.nc
View file
Open in desktop
Binary file not shown.
1 change: 1 addition & 0 deletions tests/example-data/example-slocum/deploymentRealtime_og10.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ netcdf_variables:
distance_over_ground:
latitude: LATITUDE
longitude: LONGITUDE
processing_role: distance_over_ground
long_name: Distance over ground flown since mission start
units: km

Expand Down
Binary file modified tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc
View file
Open in desktop
Binary file not shown.
52 changes: 29 additions & 23 deletions tests/expected/example-seaexplorer/L0-gridfiles-og10/dfo-eva035-20190718.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,26 @@ dimensions:
time = 22 ;
DEPTH = 1100 ;
variables:
double PROFILE_DIRECTION(time) ;
PROFILE_DIRECTION:_FillValue = NaN ;
PROFILE_DIRECTION:long_name = "Vertical direction of profile" ;
PROFILE_DIRECTION:units = "1" ;
PROFILE_DIRECTION:comment = "1 = descending, -1 = ascending, 0 = not in a profile" ;
PROFILE_DIRECTION:sources = "time PRES" ;
PROFILE_DIRECTION:method = "get_profiles_new" ;
PROFILE_DIRECTION:average_method = "mode" ;
PROFILE_DIRECTION:coverage_content_type = "physicalMeasurement" ;
PROFILE_DIRECTION:coordinates = "LATITUDE LONGITUDE profile" ;
double profile_time_start(time) ;
profile_time_start:_FillValue = NaN ;
profile_time_start:average_method = "min" ;
profile_time_start:coverage_content_type = "physicalMeasurement" ;
profile_time_start:coordinates = "LATITUDE LONGITUDE profile" ;
profile_time_start:units = "seconds since 1970-01-01T00:00:00+00:00" ;
profile_time_start:calendar = "gregorian" ;
double profile_time_end(time) ;
profile_time_end:_FillValue = NaN ;
profile_time_end:average_method = "max" ;
profile_time_end:coverage_content_type = "physicalMeasurement" ;
profile_time_end:coordinates = "LATITUDE LONGITUDE profile" ;
profile_time_end:units = "seconds since 1970-01-01T00:00:00+00:00" ;
Expand All @@ -33,6 +45,7 @@ variables:
CNDC:comment = " " ;
CNDC:platform = "platform" ;
CNDC:ancillary_variables = " " ;
CNDC:average_method = "arithmetic mean" ;
CNDC:coverage_content_type = "physicalMeasurement" ;
CNDC:coordinates = "LATITUDE LONGITUDE profile" ;
double TEMP(DEPTH, time) ;
Expand All @@ -53,6 +66,7 @@ variables:
TEMP:comment = " " ;
TEMP:platform = "platform" ;
TEMP:ancillary_variables = " " ;
TEMP:average_method = "arithmetic mean" ;
TEMP:coverage_content_type = "physicalMeasurement" ;
TEMP:coordinates = "LATITUDE LONGITUDE profile" ;
double PRES(DEPTH, time) ;
Expand All @@ -75,6 +89,7 @@ variables:
PRES:comment = " " ;
PRES:platform = "platform" ;
PRES:ancillary_variables = " " ;
PRES:average_method = "arithmetic mean" ;
PRES:coverage_content_type = "physicalMeasurement" ;
PRES:coordinates = "LATITUDE LONGITUDE profile" ;
double HEADING(DEPTH, time) ;
Expand All @@ -90,6 +105,7 @@ variables:
HEADING:platform = "platform" ;
HEADING:resolution = " " ;
HEADING:ancillary_variables = " " ;
HEADING:average_method = "arithmetic mean" ;
HEADING:coverage_content_type = "physicalMeasurement" ;
HEADING:coordinates = "LATITUDE LONGITUDE profile" ;
double PITCH(DEPTH, time) ;
Expand All @@ -105,6 +121,7 @@ variables:
PITCH:platform = "platform" ;
PITCH:resolution = " " ;
PITCH:ancillary_variables = " " ;
PITCH:average_method = "arithmetic mean" ;
PITCH:coverage_content_type = "physicalMeasurement" ;
PITCH:coordinates = "LATITUDE LONGITUDE profile" ;
double ROLL(DEPTH, time) ;
Expand All @@ -120,6 +137,7 @@ variables:
ROLL:platform = "platform" ;
ROLL:resolution = " " ;
ROLL:ancillary_variables = " " ;
ROLL:average_method = "arithmetic mean" ;
ROLL:coverage_content_type = "physicalMeasurement" ;
ROLL:coordinates = "LATITUDE LONGITUDE profile" ;
double CHLA(DEPTH, time) ;
Expand All @@ -136,6 +154,7 @@ variables:
CHLA:platform = "platform" ;
CHLA:resolution = " " ;
CHLA:ancillary_variables = " " ;
CHLA:average_method = "arithmetic mean" ;
CHLA:coverage_content_type = "physicalMeasurement" ;
CHLA:coordinates = "LATITUDE LONGITUDE profile" ;
double CDOM(DEPTH, time) ;
Expand All @@ -151,6 +170,7 @@ variables:
CDOM:platform = "platform" ;
CDOM:resolution = " " ;
CDOM:ancillary_variables = " " ;
CDOM:average_method = "arithmetic mean" ;
CDOM:coverage_content_type = "physicalMeasurement" ;
CDOM:coordinates = "LATITUDE LONGITUDE profile" ;
double BBP700(DEPTH, time) ;
Expand All @@ -166,31 +186,9 @@ variables:
BBP700:platform = "platform" ;
BBP700:resolution = " " ;
BBP700:ancillary_variables = " " ;
BBP700:average_method = "arithmetic mean" ;
BBP700:coverage_content_type = "physicalMeasurement" ;
BBP700:coordinates = "LATITUDE LONGITUDE profile" ;
double PROFILE_DIRECTION(DEPTH, time) ;
PROFILE_DIRECTION:_FillValue = NaN ;
PROFILE_DIRECTION:long_name = "Vertical direction of profile" ;
PROFILE_DIRECTION:units = "1" ;
PROFILE_DIRECTION:comment = "1 = descending, -1 = ascending, 0 = not in a profile" ;
PROFILE_DIRECTION:sources = "time PRES" ;
PROFILE_DIRECTION:method = "get_profiles_new" ;
PROFILE_DIRECTION:coverage_content_type = "physicalMeasurement" ;
PROFILE_DIRECTION:coordinates = "LATITUDE LONGITUDE profile" ;
double DISTANCE_OVER_GROUND(DEPTH, time) ;
DISTANCE_OVER_GROUND:_FillValue = NaN ;
DISTANCE_OVER_GROUND:long_name = "Distance over ground flown since mission start" ;
DISTANCE_OVER_GROUND:units = "km" ;
DISTANCE_OVER_GROUND:method = "distance_over_ground" ;
DISTANCE_OVER_GROUND:sources = "LATITUDE LONGITUDE" ;
DISTANCE_OVER_GROUND:comment = "Computed by distance_over_ground from latitude=LATITUDE, longitude=LONGITUDE" ;
DISTANCE_OVER_GROUND:accuracy = " " ;
DISTANCE_OVER_GROUND:precision = " " ;
DISTANCE_OVER_GROUND:platform = "platform" ;
DISTANCE_OVER_GROUND:resolution = " " ;
DISTANCE_OVER_GROUND:ancillary_variables = " " ;
DISTANCE_OVER_GROUND:coverage_content_type = "physicalMeasurement" ;
DISTANCE_OVER_GROUND:coordinates = "LATITUDE LONGITUDE profile" ;
double PSAL(DEPTH, time) ;
PSAL:_FillValue = NaN ;
PSAL:long_name = "Sea water practical salinity" ;
Expand All @@ -209,6 +207,7 @@ variables:
PSAL:sources = "CNDC TEMP PRES" ;
PSAL:platform = "platform" ;
PSAL:ancillary_variables = " " ;
PSAL:average_method = "arithmetic mean" ;
PSAL:coverage_content_type = "physicalMeasurement" ;
PSAL:coordinates = "LATITUDE LONGITUDE profile" ;
double THETA(DEPTH, time) ;
Expand All @@ -226,6 +225,7 @@ variables:
THETA:comment = "Computed by potential_temperature from salinity=PSAL, temperature=TEMP, pressure=PRES" ;
THETA:platform = "platform" ;
THETA:ancillary_variables = " " ;
THETA:average_method = "arithmetic mean" ;
THETA:coverage_content_type = "physicalMeasurement" ;
THETA:coordinates = "LATITUDE LONGITUDE profile" ;
double SIGTHETA(DEPTH, time) ;
Expand All @@ -243,6 +243,7 @@ variables:
SIGTHETA:comment = "Computed by potential_density_sigma0 from salinity=PSAL, temperature=TEMP, pressure=PRES, latitude=LATITUDE, longitude=LONGITUDE" ;
SIGTHETA:platform = "platform" ;
SIGTHETA:ancillary_variables = " " ;
SIGTHETA:average_method = "arithmetic mean" ;
SIGTHETA:coverage_content_type = "physicalMeasurement" ;
SIGTHETA:coordinates = "LATITUDE LONGITUDE profile" ;
double DENSITY(DEPTH, time) ;
Expand All @@ -262,6 +263,7 @@ variables:
DENSITY:platform = "platform" ;
DENSITY:resolution = " " ;
DENSITY:ancillary_variables = " " ;
DENSITY:average_method = "arithmetic mean" ;
DENSITY:coverage_content_type = "physicalMeasurement" ;
DENSITY:coordinates = "LATITUDE LONGITUDE profile" ;
double LATITUDE_GPS(DEPTH, time) ;
Expand All @@ -277,6 +279,7 @@ variables:
LATITUDE_GPS:platform = "platform" ;
LATITUDE_GPS:resolution = " " ;
LATITUDE_GPS:ancillary_variables = " " ;
LATITUDE_GPS:average_method = "arithmetic mean" ;
LATITUDE_GPS:coverage_content_type = "physicalMeasurement" ;
LATITUDE_GPS:coordinates = "LATITUDE LONGITUDE profile" ;
double LONGITUDE_GPS(DEPTH, time) ;
Expand All @@ -292,6 +295,7 @@ variables:
LONGITUDE_GPS:platform = "platform" ;
LONGITUDE_GPS:resolution = " " ;
LONGITUDE_GPS:ancillary_variables = " " ;
LONGITUDE_GPS:average_method = "arithmetic mean" ;
LONGITUDE_GPS:coverage_content_type = "physicalMeasurement" ;
LONGITUDE_GPS:coordinates = "LATITUDE LONGITUDE profile" ;
int mission_number ;
Expand Down Expand Up @@ -340,6 +344,7 @@ variables:
LONGITUDE:platform = "platform" ;
LONGITUDE:resolution = " " ;
LONGITUDE:ancillary_variables = " " ;
LONGITUDE:average_method = "arithmetic mean" ;
double LATITUDE(time) ;
LATITUDE:_FillValue = NaN ;
LATITUDE:source = "NAV_LATITUDE" ;
Expand All @@ -360,6 +365,7 @@ variables:
LATITUDE:platform = "platform" ;
LATITUDE:resolution = " " ;
LATITUDE:ancillary_variables = " " ;
LATITUDE:average_method = "arithmetic mean" ;

// global attributes:
:Conventions = "CF-1.8" ;
Expand Down
Binary file modified tests/expected/example-seaexplorer/L0-gridfiles-og10/dfo-eva035-20190718.nc
View file
Open in desktop
Binary file not shown.
Loading
Loading