Skip to content

gridding updates #248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smwoodman wants to merge 3 commits into
base: main
Choose a base branch
from
Open

gridding updates #248

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0704ba4
update gridded
smwoodman Jul 7, 2026
1341940
add distance over ground processing role
smwoodman Jul 8, 2026
f2de423
use consistent variable name
smwoodman Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 33 additions & 8 deletions pyglider/ncprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,9 @@ def make_gridfiles(
dz : float, default = 1
Vertical grid spacing in meters.

starttime : str, default = '1970-01-01'
Start time for gridding. Data before this time is ignored.

maskfunction : callable or None, optional
Function applied to the dataset before gridding,
usually to choose what data will be set to NaN based on quality flags.
Expand All @@ -250,7 +253,10 @@ def make_gridfiles(
with subsequent files overwriting previous files.

Note:
By default, the arithmetic mean is used to bin all variables, except for those with
By default, the arithmetic mean is used to bin all variables, except for
distance_over_ground (never gridded), some profile variables
(profile_start_time, profile_end_time, and profile_direction,
which are gridded using min, max, and mode, respectively), or variables with
an average_method attribute inherited from the timeseries. This attribute is specified
in the YAML configuration file when the timeseries is created. For example, if a variable
has average_method: geometric mean, the geometric mean is used when gridding that variable.
Expand All @@ -272,6 +278,7 @@ def make_gridfiles(
profile_index_varname = utils._resolve_role(ds, varnames, 'profile_index')
lat_varname = utils._resolve_role(ds, varnames, 'latitude')
lon_varname = utils._resolve_role(ds, varnames, 'longitude')
profile_direction_varname = utils._resolve_role(ds, varnames, 'profile_direction')

if maskfunction is not None:
ds = maskfunction(ds)
Expand Down Expand Up @@ -323,19 +330,26 @@ def make_gridfiles(
'comment': 'center of depth bins',
}

# Bin by profile index, for the mean time, lat, and lon values for each profile
# Bin by profile index, for the mean time, lat, and lon values for each profile
ds['time_1970'] = xr.DataArray(
ds.time.values.astype(np.float64), dims=['time'], attrs={}
)

for td in ('time_1970', lon_varname, lat_varname):
for td in ('time_1970', lon_varname, lat_varname, profile_direction_varname):

good = np.where(~np.isnan(ds[td]) & (ds[profile_index_varname] % 1 == 0))[0]
if len(good) > 1:
if td == profile_direction_varname:
method = (lambda x: stats.mode(x, keepdims=True, nan_policy='omit')[0][0])
ds[td].attrs["average_method"] = "mode"
else:
method = 'mean'
ds[td].attrs["average_method"] = "arithmetic mean"

dat, xedges, binnumber = stats.binned_statistic(
ds[profile_index_varname].values[good],
ds[td].values[good],
statistic='mean',
statistic=method,

@smwoodman smwoodman Jul 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Used method to be consistent with the for k in ds.keys(): block, even though the for td, bin_stat in profile_lookup.items(): uses bin_stat as the varname instead.

Update: I also switched from bin_stat to method in the for td, bin_stat in profile_lookup.items(): block, to be consistent throughout the function.

bins=[profile_bins],
)
if td == 'time_1970':
Expand All @@ -352,6 +366,9 @@ def make_gridfiles(
good = np.where(~np.isnan(ds['time']) & (ds[profile_index_varname] % 1 == 0))[0]
for td, bin_stat in profile_lookup.items():
_log.debug(f'td, bin_stat {td}, {bin_stat}')
attrs = profile_meta.get(td, {})
attrs['average_method'] = bin_stat

dat, xedges, binnumber = stats.binned_statistic(
ds[profile_index_varname].values[good],
ds['time_1970'].values[good],
Expand All @@ -360,31 +377,39 @@ def make_gridfiles(
)
dat = dat.astype('timedelta64[ns]') + np.datetime64('1970-01-01T00:00:00')
_log.info(f'{td} {len(dat)}')
dsout[td] = ((xdimname), dat, profile_meta.get(td, {}))
dsout[td] = ((xdimname), dat, attrs)

ds = ds.drop_vars('time_1970')
_log.info(f'Done times!')
_log.info('Done times!')


skip_vars = {'time', lat_varname, lon_varname, depth_varname, profile_index_varname}
skip_vars = {
'time', lat_varname, lon_varname, depth_varname, profile_index_varname,
"distance_over_ground", "DISTANCE_OVER_GROUND",

@smwoodman smwoodman Jul 7, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to do this in some way other than adding both 'distance_over_ground' and 'DISTANCE_OVER_GROUND' to skip_vars (eg, using utils._resolve_role), but couldn't figure out how

@jklymak jklymak Jul 8, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think if we look at the OG1 files we will see an entry for DISTANCE_OVER_GROUND

  DISTANCE_OVER_GROUND:
    processing_method:
      distance_over_ground:
        latitude: LATITUDE
        longitude: LONGITUDE
    long_name:     Distance over ground flown since mission start
    units:         km

I think this should gain a processing_role: distance_over_ground and then the "default" distance over ground should also get that processing role.

@smwoodman smwoodman Jul 8, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. I added processing_role: distance_over_ground to the example slocum/seaexplorer OG1 yml files, and updated the gridding code to use utils._resolve_role for dog.

and then the "default" distance over ground should also get that processing role.

I think this means to add 'distance_over_ground' to the 'known_roles' block in _get_varnames, so I added it there (original shown below). But please redirect me as needed

def _get_varnames(deployment):
...
    known_roles = {
        'time', 'latitude', 'longitude', 'pressure', 'temperature',
        'conductivity', 'salinity', 'depth', 'profile_index',
        'profile_direction', 'oxygen_concentration',
    }

profile_direction_varname,
}
for k in ds.keys():
if k in skip_vars or 'time' in k:
continue
if not np.issubdtype(ds[k].dtype, np.number):
continue
if 'average_method' in ds[k].attrs.keys() and ds[k].attrs['average_method'] == 'none':
_log.debug('Skipping %s because average_method is none', k)
continue
_log.info('Gridding %s', k)
good = np.where(~np.isnan(ds[k]) & (ds[profile_index_varname] % 1 == 0))[0]
if len(good) <= 0:
continue
if 'QC_protocol' in ds[k].attrs.values():
method = np.nanmax
else:
if 'average_method' in ds[k].attrs.values():
if 'average_method' in ds[k].attrs.keys():
method = ds[k].attrs['average_method']
if method == 'geometric mean':
method = stats.gmean
else:
method = 'mean'
ds[k].attrs['average_method'] = 'arithmetic mean'

dat, xedges, yedges, binnumber = stats.binned_statistic_2d(
ds[profile_index_varname].values[good],
Expand Down
Binary file modified tests/example-data/example-slocum/L0-gridfiles/dfo-rosie713-20190615_grid.nc
View file
Open in desktop
Binary file not shown.
Binary file modified tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc
View file
Open in desktop
Binary file not shown.
52 changes: 29 additions & 23 deletions tests/expected/example-seaexplorer/L0-gridfiles-og10/dfo-eva035-20190718.cdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,26 @@ dimensions:
time = 22 ;
DEPTH = 1100 ;
variables:
double PROFILE_DIRECTION(time) ;
PROFILE_DIRECTION:_FillValue = NaN ;
PROFILE_DIRECTION:long_name = "Vertical direction of profile" ;
PROFILE_DIRECTION:units = "1" ;
PROFILE_DIRECTION:comment = "1 = descending, -1 = ascending, 0 = not in a profile" ;
PROFILE_DIRECTION:sources = "time PRES" ;
PROFILE_DIRECTION:method = "get_profiles_new" ;
PROFILE_DIRECTION:average_method = "mode" ;
PROFILE_DIRECTION:coverage_content_type = "physicalMeasurement" ;
PROFILE_DIRECTION:coordinates = "LATITUDE LONGITUDE profile" ;
double profile_time_start(time) ;
profile_time_start:_FillValue = NaN ;
profile_time_start:average_method = "min" ;
profile_time_start:coverage_content_type = "physicalMeasurement" ;
profile_time_start:coordinates = "LATITUDE LONGITUDE profile" ;
profile_time_start:units = "seconds since 1970-01-01T00:00:00+00:00" ;
profile_time_start:calendar = "gregorian" ;
double profile_time_end(time) ;
profile_time_end:_FillValue = NaN ;
profile_time_end:average_method = "max" ;
profile_time_end:coverage_content_type = "physicalMeasurement" ;
profile_time_end:coordinates = "LATITUDE LONGITUDE profile" ;
profile_time_end:units = "seconds since 1970-01-01T00:00:00+00:00" ;
Expand All @@ -33,6 +45,7 @@ variables:
CNDC:comment = " " ;
CNDC:platform = "platform" ;
CNDC:ancillary_variables = " " ;
CNDC:average_method = "arithmetic mean" ;
CNDC:coverage_content_type = "physicalMeasurement" ;
CNDC:coordinates = "LATITUDE LONGITUDE profile" ;
double TEMP(DEPTH, time) ;
Expand All @@ -53,6 +66,7 @@ variables:
TEMP:comment = " " ;
TEMP:platform = "platform" ;
TEMP:ancillary_variables = " " ;
TEMP:average_method = "arithmetic mean" ;
TEMP:coverage_content_type = "physicalMeasurement" ;
TEMP:coordinates = "LATITUDE LONGITUDE profile" ;
double PRES(DEPTH, time) ;
Expand All @@ -75,6 +89,7 @@ variables:
PRES:comment = " " ;
PRES:platform = "platform" ;
PRES:ancillary_variables = " " ;
PRES:average_method = "arithmetic mean" ;
PRES:coverage_content_type = "physicalMeasurement" ;
PRES:coordinates = "LATITUDE LONGITUDE profile" ;
double HEADING(DEPTH, time) ;
Expand All @@ -90,6 +105,7 @@ variables:
HEADING:platform = "platform" ;
HEADING:resolution = " " ;
HEADING:ancillary_variables = " " ;
HEADING:average_method = "arithmetic mean" ;
HEADING:coverage_content_type = "physicalMeasurement" ;
HEADING:coordinates = "LATITUDE LONGITUDE profile" ;
double PITCH(DEPTH, time) ;
Expand All @@ -105,6 +121,7 @@ variables:
PITCH:platform = "platform" ;
PITCH:resolution = " " ;
PITCH:ancillary_variables = " " ;
PITCH:average_method = "arithmetic mean" ;
PITCH:coverage_content_type = "physicalMeasurement" ;
PITCH:coordinates = "LATITUDE LONGITUDE profile" ;
double ROLL(DEPTH, time) ;
Expand All @@ -120,6 +137,7 @@ variables:
ROLL:platform = "platform" ;
ROLL:resolution = " " ;
ROLL:ancillary_variables = " " ;
ROLL:average_method = "arithmetic mean" ;
ROLL:coverage_content_type = "physicalMeasurement" ;
ROLL:coordinates = "LATITUDE LONGITUDE profile" ;
double CHLA(DEPTH, time) ;
Expand All @@ -136,6 +154,7 @@ variables:
CHLA:platform = "platform" ;
CHLA:resolution = " " ;
CHLA:ancillary_variables = " " ;
CHLA:average_method = "arithmetic mean" ;
CHLA:coverage_content_type = "physicalMeasurement" ;
CHLA:coordinates = "LATITUDE LONGITUDE profile" ;
double CDOM(DEPTH, time) ;
Expand All @@ -151,6 +170,7 @@ variables:
CDOM:platform = "platform" ;
CDOM:resolution = " " ;
CDOM:ancillary_variables = " " ;
CDOM:average_method = "arithmetic mean" ;
CDOM:coverage_content_type = "physicalMeasurement" ;
CDOM:coordinates = "LATITUDE LONGITUDE profile" ;
double BBP700(DEPTH, time) ;
Expand All @@ -166,31 +186,9 @@ variables:
BBP700:platform = "platform" ;
BBP700:resolution = " " ;
BBP700:ancillary_variables = " " ;
BBP700:average_method = "arithmetic mean" ;
BBP700:coverage_content_type = "physicalMeasurement" ;
BBP700:coordinates = "LATITUDE LONGITUDE profile" ;
double PROFILE_DIRECTION(DEPTH, time) ;
PROFILE_DIRECTION:_FillValue = NaN ;
PROFILE_DIRECTION:long_name = "Vertical direction of profile" ;
PROFILE_DIRECTION:units = "1" ;
PROFILE_DIRECTION:comment = "1 = descending, -1 = ascending, 0 = not in a profile" ;
PROFILE_DIRECTION:sources = "time PRES" ;
PROFILE_DIRECTION:method = "get_profiles_new" ;
PROFILE_DIRECTION:coverage_content_type = "physicalMeasurement" ;
PROFILE_DIRECTION:coordinates = "LATITUDE LONGITUDE profile" ;
double DISTANCE_OVER_GROUND(DEPTH, time) ;
DISTANCE_OVER_GROUND:_FillValue = NaN ;
DISTANCE_OVER_GROUND:long_name = "Distance over ground flown since mission start" ;
DISTANCE_OVER_GROUND:units = "km" ;
DISTANCE_OVER_GROUND:method = "distance_over_ground" ;
DISTANCE_OVER_GROUND:sources = "LATITUDE LONGITUDE" ;
DISTANCE_OVER_GROUND:comment = "Computed by distance_over_ground from latitude=LATITUDE, longitude=LONGITUDE" ;
DISTANCE_OVER_GROUND:accuracy = " " ;
DISTANCE_OVER_GROUND:precision = " " ;
DISTANCE_OVER_GROUND:platform = "platform" ;
DISTANCE_OVER_GROUND:resolution = " " ;
DISTANCE_OVER_GROUND:ancillary_variables = " " ;
DISTANCE_OVER_GROUND:coverage_content_type = "physicalMeasurement" ;
DISTANCE_OVER_GROUND:coordinates = "LATITUDE LONGITUDE profile" ;
double PSAL(DEPTH, time) ;
PSAL:_FillValue = NaN ;
PSAL:long_name = "Sea water practical salinity" ;
Expand All @@ -209,6 +207,7 @@ variables:
PSAL:sources = "CNDC TEMP PRES" ;
PSAL:platform = "platform" ;
PSAL:ancillary_variables = " " ;
PSAL:average_method = "arithmetic mean" ;
PSAL:coverage_content_type = "physicalMeasurement" ;
PSAL:coordinates = "LATITUDE LONGITUDE profile" ;
double THETA(DEPTH, time) ;
Expand All @@ -226,6 +225,7 @@ variables:
THETA:comment = "Computed by potential_temperature from salinity=PSAL, temperature=TEMP, pressure=PRES" ;
THETA:platform = "platform" ;
THETA:ancillary_variables = " " ;
THETA:average_method = "arithmetic mean" ;
THETA:coverage_content_type = "physicalMeasurement" ;
THETA:coordinates = "LATITUDE LONGITUDE profile" ;
double SIGTHETA(DEPTH, time) ;
Expand All @@ -243,6 +243,7 @@ variables:
SIGTHETA:comment = "Computed by potential_density_sigma0 from salinity=PSAL, temperature=TEMP, pressure=PRES, latitude=LATITUDE, longitude=LONGITUDE" ;
SIGTHETA:platform = "platform" ;
SIGTHETA:ancillary_variables = " " ;
SIGTHETA:average_method = "arithmetic mean" ;
SIGTHETA:coverage_content_type = "physicalMeasurement" ;
SIGTHETA:coordinates = "LATITUDE LONGITUDE profile" ;
double DENSITY(DEPTH, time) ;
Expand All @@ -262,6 +263,7 @@ variables:
DENSITY:platform = "platform" ;
DENSITY:resolution = " " ;
DENSITY:ancillary_variables = " " ;
DENSITY:average_method = "arithmetic mean" ;
DENSITY:coverage_content_type = "physicalMeasurement" ;
DENSITY:coordinates = "LATITUDE LONGITUDE profile" ;
double LATITUDE_GPS(DEPTH, time) ;
Expand All @@ -277,6 +279,7 @@ variables:
LATITUDE_GPS:platform = "platform" ;
LATITUDE_GPS:resolution = " " ;
LATITUDE_GPS:ancillary_variables = " " ;
LATITUDE_GPS:average_method = "arithmetic mean" ;
LATITUDE_GPS:coverage_content_type = "physicalMeasurement" ;
LATITUDE_GPS:coordinates = "LATITUDE LONGITUDE profile" ;
double LONGITUDE_GPS(DEPTH, time) ;
Expand All @@ -292,6 +295,7 @@ variables:
LONGITUDE_GPS:platform = "platform" ;
LONGITUDE_GPS:resolution = " " ;
LONGITUDE_GPS:ancillary_variables = " " ;
LONGITUDE_GPS:average_method = "arithmetic mean" ;
LONGITUDE_GPS:coverage_content_type = "physicalMeasurement" ;
LONGITUDE_GPS:coordinates = "LATITUDE LONGITUDE profile" ;
int mission_number ;
Expand Down Expand Up @@ -340,6 +344,7 @@ variables:
LONGITUDE:platform = "platform" ;
LONGITUDE:resolution = " " ;
LONGITUDE:ancillary_variables = " " ;
LONGITUDE:average_method = "arithmetic mean" ;
double LATITUDE(time) ;
LATITUDE:_FillValue = NaN ;
LATITUDE:source = "NAV_LATITUDE" ;
Expand All @@ -360,6 +365,7 @@ variables:
LATITUDE:platform = "platform" ;
LATITUDE:resolution = " " ;
LATITUDE:ancillary_variables = " " ;
LATITUDE:average_method = "arithmetic mean" ;

// global attributes:
:Conventions = "CF-1.8" ;
Expand Down
Binary file modified tests/expected/example-seaexplorer/L0-gridfiles-og10/dfo-eva035-20190718.nc
View file
Open in desktop
Binary file not shown.
Loading
Loading