From bab02a5ce1e3f5b14f00ea5fd9e59bf039a32bd0 Mon Sep 17 00:00:00 2001 From: Joao Souza Date: Thu, 23 Apr 2026 11:26:49 -0600 Subject: [PATCH 1/4] Refactor ObsADT::Change georef to include only QCed observations Modify the way the reference ADT is calculated to include only the observations that passed the QC steps defined in the IODA yaml. This avoids contaminating the ADT that is assimilate with any wrog data that is filtered out in the QC. --- src/ufo/operators/marine/adt/ObsADT.cc | 61 ++++++++++++++++++-------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/src/ufo/operators/marine/adt/ObsADT.cc b/src/ufo/operators/marine/adt/ObsADT.cc index 78019bc3c..7c5921a72 100644 --- a/src/ufo/operators/marine/adt/ObsADT.cc +++ b/src/ufo/operators/marine/adt/ObsADT.cc @@ -61,38 +61,63 @@ void ObsADT::simulateObs(const GeoVaLs & geovals, ioda::ObsVector & ovec, oops::Log::trace() << "ObsADT::simulateObs start" << std::endl; const double missing = util::missingValue(); - - // get geovals - std::vector vec(ovec.nlocs()); - geovals.getAtLevel(vec, oops::Variable{"sea_surface_height_above_geoid"}, 0); - - // get obs + const size_t nlocs = ovec.nlocs(); + + // ----------------------- + // 1. GeoVaLs + // ----------------------- + std::vector vec(nlocs); + geovals.getAtLevel(vec, + oops::Variable{"sea_surface_height_above_geoid"}, 0); + + // ----------------------- + // 2. Observations + // ----------------------- std::vector obs; - odb_.get_db("ObsValue", ovec.varnames().variables()[operatorVarIndex_], obs); + odb_.get_db("ObsValue", + ovec.varnames().variables()[operatorVarIndex_], + obs); + + // ----------------------- + // 3. Compute global offset (QC-filtered) + // ----------------------- + double offset = 0.0; - // calculate global offsets - double offset = 0; auto accumVal = odb_.distribution()->createAccumulator(); auto accumCnt = odb_.distribution()->createAccumulator(); - for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { - // TODO(travis) also remove obs that have failed QC before this point? - // Not doing it now to keep answers from changing with this PR. - if (obs[jloc] != missing && vec[jloc] != missing) { + + for (size_t jloc = 0; jloc < nlocs; ++jloc) { + // ✅ CORRECT QC ACCESS FOR ObsDataVector + const int qc = qc_flags[operatorVarIndex_][jloc]; + + if (qc == 0 && + obs[jloc] != missing && + vec[jloc] != missing && + std::isfinite(obs[jloc]) && + std::isfinite(vec[jloc])) { + accumVal->addTerm(jloc, vec[jloc] - obs[jloc]); accumCnt->addTerm(jloc, 1); } } - auto count = accumCnt->computeResult(); + + const int count = accumCnt->computeResult(); + if (count > 0) { offset = accumVal->computeResult() / count; } - // oops::Log::debug() << "ObsADT::simulateObs offset: " << offset << std::endl; - // subtract offset from geoval - for (size_t jloc = 0; jloc < ovec.nlocs(); ++jloc) { + // ----------------------- + // 4. Apply offset (no QC filtering here!) + // ----------------------- + for (size_t jloc = 0; jloc < nlocs; ++jloc) { const size_t idx = jloc * ovec.nvars() + operatorVarIndex_; + ovec[idx] = vec[jloc]; - if (ovec[idx] != missing) ovec[idx] -= offset; + + if (ovec[idx] != missing && std::isfinite(ovec[idx])) { + ovec[idx] -= offset; + } } oops::Log::trace() << "ObsADT::simulateObs done" << std::endl; From 87f78c0d167dcc157e4e073cf4788ff635d51d15 Mon Sep 17 00:00:00 2001 From: Joao Souza Date: Tue, 2 Jun 2026 12:07:47 -0400 Subject: [PATCH 2/4] Revert "Merge branch 'develop' into patch-2" This reverts commit a858a187a62b26057dee46213d5e65037e3ca3d3, reversing changes made to bab02a5ce1e3f5b14f00ea5fd9e59bf039a32bd0. --- src/mains/RunCRTM.h | 5 + src/mains/RunPreProcess.h | 6 +- src/ufo/CMakeLists.txt | 2 +- src/ufo/ObsFilter.cc | 2 +- src/ufo/ObsFilter.h | 8 +- src/ufo/ObsFilterBase.cc | 2 +- src/ufo/ObsFilterBase.h | 19 +- src/ufo/ObsFilters.cc | 135 ++++--- src/ufo/ObsFilters.h | 12 +- src/ufo/ObsTraits.h | 6 +- src/ufo/errors/ObsErrorBase.h | 2 - src/ufo/errors/ObsErrorCrossVarCov.cc | 75 +--- src/ufo/errors/ObsErrorCrossVarCov.h | 10 +- src/ufo/errors/ObsErrorDiagonal.cc | 2 +- src/ufo/errors/ObsErrorDiagonalInvGamma.cc | 2 +- src/ufo/errors/ObsErrorDiffusion.cc | 161 ++------- src/ufo/errors/ObsErrorDiffusion.h | 21 -- src/ufo/errors/ObsErrorParametersBase.h | 5 - src/ufo/errors/ObsErrorReconditioner.cc | 6 +- src/ufo/errors/ObsErrorWithinGroupCov.cc | 141 +------- src/ufo/errors/ObsErrorWithinGroupCov.h | 17 +- src/ufo/filters/AcceptList.cc | 4 +- src/ufo/filters/AcceptList.h | 4 +- src/ufo/filters/AverageObsToGeoValLevels.cc | 6 +- src/ufo/filters/AverageObsToGeoValLevels.h | 4 +- src/ufo/filters/BackgroundCheck.cc | 20 +- src/ufo/filters/BackgroundCheck.h | 4 +- src/ufo/filters/BayesianBackgroundCheck.cc | 8 +- src/ufo/filters/BayesianBackgroundCheck.h | 4 +- src/ufo/filters/BayesianBackgroundQCFlags.cc | 4 +- src/ufo/filters/BayesianBackgroundQCFlags.h | 4 +- src/ufo/filters/BlackList.cc | 4 +- src/ufo/filters/BlackList.h | 4 +- src/ufo/filters/CMakeLists.txt | 4 - .../filters/ConventionalProfileProcessing.cc | 6 +- .../filters/ConventionalProfileProcessing.h | 4 +- .../CopyFlagsFromExtendedToOriginalSpace.cc | 6 +- .../CopyFlagsFromExtendedToOriginalSpace.h | 4 +- src/ufo/filters/CreateDiagnosticFlags.cc | 6 +- src/ufo/filters/CreateDiagnosticFlags.h | 4 +- src/ufo/filters/DifferenceCheck.cc | 4 +- src/ufo/filters/DifferenceCheck.h | 4 +- src/ufo/filters/EnsembleStatistics.cc | 6 +- src/ufo/filters/EnsembleStatistics.h | 4 +- src/ufo/filters/FilterBase.cc | 16 +- src/ufo/filters/FilterBase.h | 16 +- src/ufo/filters/FinalCheck.cc | 13 +- src/ufo/filters/FinalCheck.h | 4 +- src/ufo/filters/Gaussian_Thinning.cc | 10 +- src/ufo/filters/Gaussian_Thinning.h | 4 +- src/ufo/filters/GeoVaLsWriter.cc | 2 +- src/ufo/filters/GeoVaLsWriter.h | 4 +- src/ufo/filters/HistoryCheck.cc | 16 +- src/ufo/filters/HistoryCheck.h | 8 +- src/ufo/filters/ImpactHeightCheck.cc | 6 +- src/ufo/filters/ImpactHeightCheck.h | 6 +- src/ufo/filters/MWCLWCheck.cc | 4 +- src/ufo/filters/MWCLWCheck.h | 4 +- src/ufo/filters/MetOfficeBuddyCheck.cc | 8 +- src/ufo/filters/MetOfficeBuddyCheck.h | 4 +- src/ufo/filters/MetOfficeDuplicateCheck.cc | 4 +- src/ufo/filters/MetOfficeDuplicateCheck.h | 4 +- .../MetOfficePressureConsistencyCheck.cc | 6 +- .../MetOfficePressureConsistencyCheck.h | 4 +- src/ufo/filters/ModelBestFitPressure.cc | 4 +- src/ufo/filters/ModelBestFitPressure.h | 4 +- src/ufo/filters/ModelObThreshold.cc | 4 +- src/ufo/filters/ModelObThreshold.h | 4 +- src/ufo/filters/ObsAccessor.cc | 3 - src/ufo/filters/ObsBoundsCheck.cc | 6 +- src/ufo/filters/ObsBoundsCheck.h | 4 +- src/ufo/filters/ObsDerivativeCheck.cc | 4 +- src/ufo/filters/ObsDerivativeCheck.h | 4 +- src/ufo/filters/ObsDiagnosticsWriter.cc | 4 +- src/ufo/filters/ObsDiagnosticsWriter.h | 4 +- src/ufo/filters/ObsDomainCheck.cc | 4 +- src/ufo/filters/ObsDomainCheck.h | 4 +- src/ufo/filters/ObsDomainErrCheck.cc | 17 +- src/ufo/filters/ObsDomainErrCheck.h | 4 +- src/ufo/filters/ObsPolygonCheck.cc | 4 +- src/ufo/filters/ObsPolygonCheck.h | 4 +- src/ufo/filters/ObsProcessorBase.cc | 10 +- src/ufo/filters/ObsProcessorBase.h | 8 +- .../filters/ObsRefractivityGradientCheck.cc | 8 +- .../filters/ObsRefractivityGradientCheck.h | 4 +- .../filters/OceanVerticalStabilityCheck.cc | 6 +- src/ufo/filters/OceanVerticalStabilityCheck.h | 4 +- src/ufo/filters/ParameterSubstitution.cc | 4 +- src/ufo/filters/ParameterSubstitution.h | 4 +- src/ufo/filters/PerformAction.cc | 4 +- src/ufo/filters/PerformAction.h | 4 +- src/ufo/filters/PoissonDiskThinning.cc | 38 +- src/ufo/filters/PoissonDiskThinning.h | 4 +- src/ufo/filters/PreQC.cc | 4 +- src/ufo/filters/PreQC.h | 4 +- src/ufo/filters/PrintFilterData.cc | 6 +- src/ufo/filters/PrintFilterData.h | 4 +- .../ProbabilityGrossErrorWholeReport.cc | 4 +- .../ProbabilityGrossErrorWholeReport.h | 4 +- src/ufo/filters/ProcessAMVQI.cc | 4 +- src/ufo/filters/ProcessAMVQI.h | 4 +- .../filters/ProfileAverageObsToModLevels.cc | 10 +- .../filters/ProfileAverageObsToModLevels.h | 4 +- src/ufo/filters/ProfileBackgroundCheck.cc | 18 +- src/ufo/filters/ProfileBackgroundCheck.h | 4 +- src/ufo/filters/ProfileFewObsCheck.cc | 8 +- src/ufo/filters/ProfileFewObsCheck.h | 4 +- src/ufo/filters/ProfileMaxDifferenceCheck.cc | 4 +- src/ufo/filters/ProfileMaxDifferenceCheck.h | 4 +- src/ufo/filters/ProfileUnFlagObsCheck.cc | 16 +- src/ufo/filters/ProfileUnFlagObsCheck.h | 4 +- src/ufo/filters/QCmanager.cc | 25 +- src/ufo/filters/QCmanager.h | 6 +- src/ufo/filters/SatName.cc | 4 +- src/ufo/filters/SatName.h | 4 +- src/ufo/filters/SatwindInversionCorrection.cc | 6 +- src/ufo/filters/SatwindInversionCorrection.h | 4 +- src/ufo/filters/SpikeAndStepCheck.cc | 6 +- src/ufo/filters/SpikeAndStepCheck.h | 4 +- src/ufo/filters/StuckCheck.cc | 4 +- src/ufo/filters/StuckCheck.h | 4 +- src/ufo/filters/SuperOb.cc | 8 +- src/ufo/filters/SuperOb.h | 4 +- .../SuperRefractionCheckImpactParameter.cc | 8 +- .../SuperRefractionCheckImpactParameter.h | 4 +- src/ufo/filters/SuperRefractionCheckNBAM.cc | 4 +- src/ufo/filters/SuperRefractionCheckNBAM.h | 4 +- src/ufo/filters/TemporalThinning.cc | 8 +- src/ufo/filters/TemporalThinning.h | 4 +- src/ufo/filters/Thinning.cc | 4 +- src/ufo/filters/Thinning.h | 4 +- src/ufo/filters/TrackCheck.cc | 6 +- src/ufo/filters/TrackCheck.h | 4 +- src/ufo/filters/TrackCheckShip.cc | 8 +- src/ufo/filters/TrackCheckShip.h | 4 +- src/ufo/filters/VariableAssignment.cc | 8 +- src/ufo/filters/VariableAssignment.h | 4 +- src/ufo/filters/VariableTransforms.cc | 4 +- src/ufo/filters/VariableTransforms.h | 4 +- src/ufo/filters/Variables.cc | 2 +- src/ufo/filters/actions/SetFlag.cc | 2 +- .../gnssroonedvarcheck/GNSSROOneDVarCheck.cc | 10 +- .../gnssroonedvarcheck/GNSSROOneDVarCheck.h | 4 +- .../ufo_gnssroonedvarcheck_mod.f90 | 18 +- .../obsfunctions/CircularDifference.cc | 118 ------- .../filters/obsfunctions/CircularDifference.h | 113 ------ .../CloudDetectMinResidualAVHRR.cc | 10 +- .../obsfunctions/CloudDetectMinResidualIR.cc | 15 + .../obsfunctions/HydrometeorCheckAMSUA.cc | 4 +- .../obsfunctions/HydrometeorCheckAMSUAclr.cc | 4 +- .../obsfunctions/HydrometeorCheckATMS.cc | 6 +- .../filters/obsfunctions/ObsErrorBoundMW.cc | 11 +- .../ObsErrorFactorPressureCheck.cc | 3 +- .../ObsErrorFactorSituDependMW.cc | 10 +- .../ObsErrorFactorSurfJacobianRad.cc | 2 +- .../obsfunctions/ObsErrorFactorTopoRad.cc | 4 +- src/ufo/filters/obsfunctions/SolarZenith.cc | 4 +- src/ufo/filters/obsfunctions/TimeBinner.cc | 334 ------------------ src/ufo/filters/obsfunctions/TimeBinner.h | 247 ------------- .../RefractivityOneDVarCheck.cc | 10 +- .../RefractivityOneDVarCheck.h | 4 +- .../rttovonedvarcheck/RTTOVOneDVarCheck.cc | 8 +- .../rttovonedvarcheck/RTTOVOneDVarCheck.h | 4 +- .../ufo_rttovonedvarcheck_ob_mod.f90 | 6 +- .../ufo_rttovonedvarcheck_setup_mod.f90 | 8 +- src/ufo/fov/calc_fov_conical.f90 | 6 +- src/ufo/fov/calc_fov_crosstrk.f90 | 6 +- src/ufo/obslocalization/ObsHorLocGC99.h | 16 - src/ufo/obslocalization/ObsHorLocSOAR.h | 16 - src/ufo/obslocalization/ObsHorLocalization.h | 24 -- src/ufo/obslocalization/ObsLocalization.h | 10 - src/ufo/obslocalization/ObsLocalizationBase.h | 12 - src/ufo/operators/crtm/ObsRadianceCRTM.cc | 2 +- .../crtm/ObsRadianceCRTM.interface.h | 4 +- src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc | 4 +- .../crtm/ObsRadianceCRTMTLAD.interface.h | 6 +- .../ufo_gnssro_bendmetoffice_tlad_mod.F90 | 8 +- .../gnssro/QC/BackgroundCheckRONBAM.cc | 6 +- .../gnssro/QC/BackgroundCheckRONBAM.h | 4 +- src/ufo/operators/gnssro/QC/ROobserror.cc | 14 +- src/ufo/operators/gnssro/QC/ROobserror.h | 4 +- src/ufo/operators/oasim/ObsRadianceOASIM.cc | 3 +- .../rttov/CPP/v14/rttovcpp_interface.cc | 8 +- .../rttov/Fortran/ObsRadianceRTTOV.cc | 4 +- .../Fortran/ObsRadianceRTTOV.interface.h | 8 +- .../rttov/Fortran/ObsRadianceRTTOVTLAD.cc | 7 +- .../rttov/Fortran/ObsRadianceRTTOVTLAD.h | 1 - .../Fortran/ObsRadianceRTTOVTLAD.interface.h | 10 +- src/ufo/profile/ProfileAveragePressure.cc | 6 +- .../profile/ProfileAverageRelativeHumidity.cc | 6 +- src/ufo/profile/ProfileAverageTemperature.cc | 6 +- src/ufo/profile/ProfileAverageWindSpeed.cc | 6 +- src/ufo/profile/ProfileCheckRH.cc | 1 + src/ufo/utils/SaveBiasCoeffs.cc | 4 +- src/ufo/utils/SaveBiasCovariance.cc | 4 +- src/ufo/utils/dataextractor/DataExtractor.h | 6 - .../DataExtractorNetCDFBackend.h | 2 +- .../Cal_AdjustedHeightCoordinate.cc | 4 +- .../Cal_AdjustedHeightCoordinate.h | 6 +- .../Cal_HeightFromPressure.cc | 4 +- .../Cal_HeightFromPressure.h | 4 +- src/ufo/variabletransforms/Cal_Humidity.cc | 12 +- src/ufo/variabletransforms/Cal_Humidity.h | 12 +- .../Cal_IceThicknessFromFreeboard.cc | 4 +- .../Cal_IceThicknessFromFreeboard.h | 4 +- src/ufo/variabletransforms/Cal_Logarithm.cc | 4 +- src/ufo/variabletransforms/Cal_Logarithm.h | 4 +- src/ufo/variabletransforms/Cal_PStar.cc | 4 +- src/ufo/variabletransforms/Cal_PStar.h | 4 +- .../variabletransforms/Cal_PotentialTFromT.cc | 4 +- .../variabletransforms/Cal_PotentialTFromT.h | 4 +- .../Cal_PressureFromHeight.cc | 8 +- .../Cal_PressureFromHeight.h | 8 +- .../Cal_ProfileHorizontalDrift.cc | 4 +- .../Cal_ProfileHorizontalDrift.h | 4 +- .../Cal_QNHtoQFEpressure.cc | 4 +- .../variabletransforms/Cal_QNHtoQFEpressure.h | 4 +- .../Cal_RadarBeamGeometry.cc | 4 +- .../Cal_RadarBeamGeometry.h | 4 +- .../Cal_RemapScanPosition.cc | 4 +- .../Cal_RemapScanPosition.h | 4 +- .../Cal_SatBrightnessTempFromRad.cc | 4 +- .../Cal_SatBrightnessTempFromRad.h | 4 +- .../Cal_SatRadianceFromPCScores.cc | 6 +- .../Cal_SatRadianceFromPCScores.h | 6 +- .../Cal_SatRadianceFromScaledRadiance.cc | 8 +- .../Cal_SatRadianceFromScaledRadiance.h | 6 +- .../Cal_SatZenithAngleCorrection.cc | 4 +- .../Cal_SatZenithAngleCorrection.h | 4 +- .../Cal_SurfaceWindScalingCombined.cc | 4 +- .../Cal_SurfaceWindScalingCombined.h | 6 +- .../Cal_SurfaceWindScalingHeight.cc | 4 +- .../Cal_SurfaceWindScalingHeight.h | 4 +- .../Cal_SurfaceWindScalingPressure.cc | 4 +- .../Cal_SurfaceWindScalingPressure.h | 6 +- src/ufo/variabletransforms/Cal_Wind.cc | 8 +- src/ufo/variabletransforms/Cal_Wind.h | 12 +- .../OceanConversions/OceanDensity.cc | 4 +- .../OceanConversions/OceanDensity.h | 4 +- .../OceanConversions/OceanDepthToPressure.cc | 4 +- .../OceanConversions/OceanDepthToPressure.h | 4 +- ...ceanPracticalSalinityToAbsoluteSalinity.cc | 4 +- ...OceanPracticalSalinityToAbsoluteSalinity.h | 6 +- .../OceanTempToConservativeTemp.cc | 4 +- .../OceanTempToConservativeTemp.h | 4 +- .../OceanConversions/OceanTempToTheta.cc | 4 +- .../OceanConversions/OceanTempToTheta.h | 4 +- src/ufo/variabletransforms/TransformBase.cc | 10 +- src/ufo/variabletransforms/TransformBase.h | 47 +-- test/CMakeLists.txt | 18 +- test/mains/TestObsLocalization.cc | 15 - .../instrumentTests/amsua/CMakeLists.txt | 10 + .../amsua/amsua_n18_gfs_HofX_bc.yaml | 44 +++ .../unit_tests/errors/CMakeLists.txt | 12 +- .../unit_tests/errors/diracDiffusion.yaml | 4 - .../errors/obserrorcrossvarcorr.yaml | 16 +- .../errors/obserrorcrossvarcov.yaml | 10 +- .../unit_tests/errors/obserrordiffusion.yaml | 4 - .../errors/obserrorwithingroupcorr.yaml | 20 -- .../errors/reconditioning_withinGroup.yaml | 27 -- .../filters/obsfunctions/CMakeLists.txt | 34 +- .../function_assignvalueequalchannels.yaml | 2 +- .../function_circulardifference.yaml | 47 --- .../obsfunctions/function_timebinner.yaml | 208 ----------- .../obslocalization/obs_localization.yaml | 78 ---- .../unit_tests/predictors/CMakeLists.txt | 9 - .../predictors/bias_linear_op_tlad.yaml | 50 --- .../unit_tests/errors/diracDiffusion.ref | 8 +- test/ufo/ConventionalProfileProcessing.h | 14 +- test/ufo/GaussianThinning.h | 10 +- test/ufo/GeoVaLs.h | 2 +- test/ufo/HistoryCheck.h | 10 +- test/ufo/MetOfficeBuddyCheck.h | 6 +- test/ufo/MetOfficeBuddyPairFinder.h | 2 + test/ufo/ObsErrorAssign.h | 12 +- test/ufo/ObsFilters.h | 67 ++-- test/ufo/ObsLocalization.h | 112 ------ test/ufo/ParallelPoissonDiskThinning.h | 12 +- test/ufo/PoissonDiskThinning.h | 10 +- test/ufo/Reconditioning.h | 30 +- test/ufo/SampleAndReduceOverFieldOfView.h | 1 + test/ufo/SampledLocations.h | 2 + test/ufo/StuckCheck.h | 10 +- test/ufo/TemporalThinning.h | 10 +- test/ufo/TrackCheck.h | 10 +- test/ufo/TrackCheckShip.h | 10 +- test/ufo/TrackCheckShipMainLoop.h | 8 +- 287 files changed, 1012 insertions(+), 2793 deletions(-) delete mode 100644 src/ufo/filters/obsfunctions/CircularDifference.cc delete mode 100644 src/ufo/filters/obsfunctions/CircularDifference.h delete mode 100644 src/ufo/filters/obsfunctions/TimeBinner.cc delete mode 100644 src/ufo/filters/obsfunctions/TimeBinner.h delete mode 100644 test/mains/TestObsLocalization.cc create mode 100644 test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml delete mode 100644 test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml delete mode 100644 test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml delete mode 100644 test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml delete mode 100644 test/testinput/unit_tests/obslocalization/obs_localization.yaml delete mode 100644 test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml delete mode 100644 test/ufo/ObsLocalization.h diff --git a/src/mains/RunCRTM.h b/src/mains/RunCRTM.h index bce6d4e45..9b2e38f61 100644 --- a/src/mains/RunCRTM.h +++ b/src/mains/RunCRTM.h @@ -76,6 +76,11 @@ template class RunCRTM : public oops::Application { ObsDiags_ diag(obsdb[jj], hop.locations(), diagvars); hop.simulateObs(gval, hofx, ybias, qcflags, bias, diag); + + const double zz = hofx.rms(); + const double xx = conf[jj].getDouble("rms ref"); + const double tol = conf[jj].getDouble("tolerance"); +// BOOST_CHECK_CLOSE(xx, zz, tol); } return 0; diff --git a/src/mains/RunPreProcess.h b/src/mains/RunPreProcess.h index 955e6e4f1..ad085ddcd 100644 --- a/src/mains/RunPreProcess.h +++ b/src/mains/RunPreProcess.h @@ -57,8 +57,10 @@ class RunPreProcess : public oops::Application { util::TimeWindow(params.timeWindow.value()), this->getComm()); - ioda::ObsDataVector obserr_(obsdb, obsdb.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags_(obsdb, obsdb.obsvariables()); + std::shared_ptr> obserr_ = + std::make_shared>(obsdb, obsdb.obsvariables(), "ObsError"); + std::shared_ptr> qcflags_ = + std::make_shared>(obsdb, obsdb.obsvariables()); ufo::ObsFilters filters(obsdb, params.filtersParams.toConfiguration(), diff --git a/src/ufo/CMakeLists.txt b/src/ufo/CMakeLists.txt index 6d5e90d5a..2a9ab69f5 100644 --- a/src/ufo/CMakeLists.txt +++ b/src/ufo/CMakeLists.txt @@ -195,7 +195,7 @@ if(geos-aero_FOUND) endif() if(gsw_FOUND) - target_link_libraries(ufo PUBLIC gsw ${gsw_LIBRARIES}) + target_link_libraries(ufo PUBLIC gsw) target_compile_definitions(ufo PUBLIC GSW_FOUND) endif() diff --git a/src/ufo/ObsFilter.cc b/src/ufo/ObsFilter.cc index aa0ef22c1..e963c17ef 100644 --- a/src/ufo/ObsFilter.cc +++ b/src/ufo/ObsFilter.cc @@ -21,7 +21,7 @@ namespace ufo { ObsFilter::ObsFilter(ioda::ObsSpace & os, const ObsFilterParametersBase & parameters, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr) + ObsDataPtr_ flags, ObsDataPtr_ obserr) : ofilt_(), filterName_("ObsFilter::"+parameters.filter.value().value()) { oops::Log::trace() << "ObsFilter::ObsFilter starting" << std::endl; diff --git a/src/ufo/ObsFilter.h b/src/ufo/ObsFilter.h index 844c6f9f5..898443855 100644 --- a/src/ufo/ObsFilter.h +++ b/src/ufo/ObsFilter.h @@ -30,6 +30,8 @@ class ObsDiagnostics; /// \brief A filter processing observations. class ObsFilter : public util::Printable, private util::ObjectCounter { + template using ObsDataPtr_ = std::shared_ptr >; + public: static const std::string classname() {return "ufo::ObsFilter";} @@ -39,13 +41,13 @@ class ObsFilter : public util::Printable, /// Space containing the observations to process. /// \param params /// The filter's configuration parameters. - /// \param flags + /// \param qcflags /// Quality control flags. They may be modified by the filter. - /// \param obserr + /// \param obserrors /// Estimates of the standard deviations of observation errors. They may be modified by the /// filter. ObsFilter(ioda::ObsSpace &obsspace, const ObsFilterParametersBase ¶ms, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr); + ObsDataPtr_ qcflags, ObsDataPtr_ obserrors); ObsFilter(const ObsFilter &) = delete; ObsFilter(ObsFilter &&) = default; ObsFilter& operator=(const ObsFilter &) = delete; diff --git a/src/ufo/ObsFilterBase.cc b/src/ufo/ObsFilterBase.cc index 1f797072c..767b093f6 100644 --- a/src/ufo/ObsFilterBase.cc +++ b/src/ufo/ObsFilterBase.cc @@ -33,7 +33,7 @@ FilterFactory::FilterFactory(const std::string & name) { std::unique_ptr FilterFactory::create(ioda::ObsSpace & os, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr) { + ObsDataPtr_ flags, ObsDataPtr_ obserr) { oops::Log::trace() << "FilterFactory::create starting" << std::endl; const std::string &id = params.filter.value().value(); typename std::map::iterator diff --git a/src/ufo/ObsFilterBase.h b/src/ufo/ObsFilterBase.h index bdc716fc3..b18609826 100644 --- a/src/ufo/ObsFilterBase.h +++ b/src/ufo/ObsFilterBase.h @@ -59,7 +59,7 @@ enum class FilterStage {AUTO, /// In the former case, they should provide a constructor with the following signature: /// /// ObsFilter(ObsSpace_ &, const eckit::Configuration &, -/// ioda::ObsDataVector &, ioda::ObsDataVector &); +/// ObsDataPtr_, ObsDataPtr_); /// /// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase /// holding the settings of the filter in question. The implementation of the ObsFilter interface @@ -67,7 +67,7 @@ enum class FilterStage {AUTO, /// the following signature: /// /// ObsFilter(ObsSpace_ &, const Parameters_ &, -/// ioda::ObsDataVector &, ioda::ObsDataVector &); +/// ObsDataPtr_, ObsDataPtr_); // TODO(someone): check whether we can just keep Parameters here and remove the // Configuration option. class ObsFilterBase : public util::Printable, @@ -136,6 +136,8 @@ class ObsFilterParametersWrapper : public oops::Parameters { /// ObsFilter factory class FilterFactory { + template using ObsDataPtr_ = std::shared_ptr >; + public: /// \brief Create and return a new observation filter. /// @@ -144,8 +146,8 @@ class FilterFactory { /// otherwise an exception will be thrown. static std::unique_ptr create(ioda::ObsSpace &, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + ObsDataPtr_ flags = ObsDataPtr_(), + ObsDataPtr_ obserr = ObsDataPtr_()); /// \brief Create and return an instance of the subclass of ObsFilterParametersBase /// storing parameters of observation filters of the specified type. @@ -167,8 +169,7 @@ class FilterFactory { private: virtual std::unique_ptr make(ioda::ObsSpace &, const ObsFilterParametersBase &, - ioda::ObsDataVector &, - ioda::ObsDataVector &) = 0; + ObsDataPtr_, ObsDataPtr_) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -187,10 +188,12 @@ class FilterMaker : public FilterFactory { typedef oops::TParameters_IfAvailableElseFallbackType_t Parameters_; + template using ObsDataPtr_ = std::shared_ptr >; + std::unique_ptr make(ioda::ObsSpace & os, const ObsFilterParametersBase & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) override { + ObsDataPtr_ flags, + ObsDataPtr_ obserr) override { const auto &stronglyTypedParams = dynamic_cast(params); return std::make_unique(os, oops::parametersOrConfiguration::value>(stronglyTypedParams), diff --git a/src/ufo/ObsFilters.cc b/src/ufo/ObsFilters.cc index 1aaa7b6d9..e5d23fd72 100644 --- a/src/ufo/ObsFilters.cc +++ b/src/ufo/ObsFilters.cc @@ -22,11 +22,10 @@ namespace ufo { ObsFilters::ObsFilters(ioda::ObsSpace & os, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration) - : obsspace_(os), qcmanager_(nullptr), autoFilters_(), preFilters_(), priorFilters_(), - postFilters_(), geovars_(), diagvars_(), qcflags_(flags), obserr_(obserr), + : obsspace_(os), autoFilters_(), preFilters_(), priorFilters_(), postFilters_(), + geovars_(), diagvars_(), qcflags_(qcflags), obserrfilter_(obserr), iteration_(iteration) { oops::Log::trace() << "ObsFilters::ObsFilters starting" << std::endl; @@ -52,21 +51,22 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, // If at least one filter has been configured, prepend the QC manager and // append the Final Check to the list of filters. - atLeastOneFilterConfigured_ = - atLeastOneAutoFilterConfigured || atLeastOnePrePriorPostFilterConfigured; + const bool atLeastOneFilterConfigured = + atLeastOneAutoFilterConfigured || atLeastOnePrePriorPostFilterConfigured; - // Prepare QC handling and statistics if any filters are present - if (atLeastOneFilterConfigured_) { +// Prepare QC handling and statistics if any filters are present + if (atLeastOneFilterConfigured) { eckit::LocalConfiguration conf; conf.set("filter", "QCmanager"); ObsFilterParametersWrapper filterParams; filterParams.validateAndDeserialize(conf); - qcmanager_ = std::make_unique(os, filterParams.filterParameters, - qcflags_, obserr_); + if (atLeastOneAutoFilterConfigured) + autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); + else + preFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); } - // Create the filters, only at 0-th iteration, or at iterations specified in - // "apply at iterations" +// Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" if (atLeastOneAutoFilterConfigured) appendToFiltersList(autoFiltersParams, autoFilters_); if (atLeastOnePrePriorPostFilterConfigured) { @@ -75,18 +75,16 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, appendToFiltersList(postFiltersParams, postFilters_); } - // Create the final filter run at the end of the pipeline - if (atLeastOneFilterConfigured_) { +// Create the final filter run at the end of the pipeline + if (atLeastOneFilterConfigured) { eckit::LocalConfiguration conf; conf.set("filter", "Final Check"); ObsFilterParametersWrapper filterParams; filterParams.validateAndDeserialize(conf); if (atLeastOneAutoFilterConfigured) - autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, - obserr_); + autoFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); else - postFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, - obserr_); + postFilters_.emplace_back(os, filterParams.filterParameters, qcflags_, obserrfilter_); } oops::Log::trace() << "ObsFilters::ObsFilters done" << std::endl; @@ -94,20 +92,18 @@ ObsFilters::ObsFilters(ioda::ObsSpace & os, // ----------------------------------------------------------------------------- -void ObsFilters::appendToFiltersList(const FilterParams_& filtersParams, - std::vector& filters) { - for (const ObsFilterParametersWrapper& filterParams : filtersParams) { +void ObsFilters::appendToFiltersList(const FilterParams_ & filtersParams, + std::vector & filters) { + for (const ObsFilterParametersWrapper & filterParams : filtersParams) { // Only create filters for the 0-th iteration by default bool apply = (iteration_ == 0); // If "apply at iterations" is set, check if this is the right iteration if (filterParams.applyAtIterations.value() != boost::none) { - const std::set iters = - oops::parseIntSet(*filterParams.applyAtIterations.value()); + const std::set iters = oops::parseIntSet(*filterParams.applyAtIterations.value()); apply = oops::contains(iters, iteration_); } if (apply) { - filters.emplace_back(obsspace_, filterParams.filterParameters, - qcflags_, obserr_); + filters.emplace_back(obsspace_, filterParams.filterParameters, qcflags_, obserrfilter_); geovars_ += filters.back().requiredVars(); diagvars_ += filters.back().requiredHdiagnostics(); } @@ -117,78 +113,71 @@ void ObsFilters::appendToFiltersList(const FilterParams_& filtersParams, // ----------------------------------------------------------------------------- void ObsFilters::preProcess() { - if (atLeastOneFilterConfigured_) { - if (autoFilters_.size() > 0) { - qcmanager_->preProcess(); // No operation, placeholder - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.preProcess(); - } - } else { - qcmanager_->preProcess(); // No operation, placeholder - for (ObsFilter& filter : preFilters_) { - filter.checkFilterData(FilterStage::PRE); - filter.preProcess(); - } + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.preProcess(); + } + } else { + for (ObsFilter & filter : preFilters_) { + filter.checkFilterData(FilterStage::PRE); + filter.preProcess(); } } } // ----------------------------------------------------------------------------- -void ObsFilters::priorFilter(const GeoVaLs& gv) { - if (atLeastOneFilterConfigured_) { - qcmanager_->priorFilter(gv); // No operation, placeholder - if (autoFilters_.size() > 0) { - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.priorFilter(gv); - } - } else { - qcmanager_->priorFilter(gv); // No operation, placeholder - for (ObsFilter& filter : priorFilters_) { - filter.checkFilterData(FilterStage::PRIOR); - filter.priorFilter(gv); - } +void ObsFilters::priorFilter(const GeoVaLs & gv) { + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.priorFilter(gv); + } + } else { + for (ObsFilter & filter : priorFilters_) { + filter.checkFilterData(FilterStage::PRIOR); + filter.priorFilter(gv); } } } // ----------------------------------------------------------------------------- -void ObsFilters::postFilter(const GeoVaLs& gv, const ioda::ObsVector& hofx, - const ioda::ObsVector& bias, - const ObsDiagnostics& diags) { - if (atLeastOneFilterConfigured_) { - if (autoFilters_.size() > 0) { - qcmanager_->postFilter(gv, hofx, bias, diags); - for (ObsFilter& filter : autoFilters_) { - filter.checkFilterData(FilterStage::AUTO); - filter.postFilter(gv, hofx, bias, diags); - } - } else { - qcmanager_->postFilter(gv, hofx, bias, diags); - for (ObsFilter& filter : postFilters_) { - filter.checkFilterData(FilterStage::POST); - filter.postFilter(gv, hofx, bias, diags); - } +void ObsFilters::postFilter(const GeoVaLs & gv, + const ioda::ObsVector & hofx, + const ioda::ObsVector & bias, + const ObsDiagnostics & diags) { + if (autoFilters_.size() > 0) { + for (ObsFilter & filter : autoFilters_) { + filter.checkFilterData(FilterStage::AUTO); + filter.postFilter(gv, hofx, bias, diags); + } + } else { + for (ObsFilter & filter : postFilters_) { + filter.checkFilterData(FilterStage::POST); + filter.postFilter(gv, hofx, bias, diags); } } } // ----------------------------------------------------------------------------- -void ObsFilters::print(std::ostream& os) const { +void ObsFilters::print(std::ostream & os) const { if (autoFilters_.size() > 0) { os << "ObsFilters: " << autoFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& filter : autoFilters_) os << filter << std::endl; + for (const ObsFilter & filter : autoFilters_) + os << filter << std::endl; } else { os << "Pre filters: " << preFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& preFilter : preFilters_) os << preFilter << std::endl; + for (const ObsFilter & preFilter : preFilters_) + os << preFilter << std::endl; os << "Prior filters: " << priorFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& priorFilter : priorFilters_) os << priorFilter << std::endl; + for (const ObsFilter & priorFilter : priorFilters_) + os << priorFilter << std::endl; os << "Post filters: " << postFilters_.size() << " elements:" << std::endl; - for (const ObsFilter& postFilter : postFilters_) os << postFilter << std::endl; + for (const ObsFilter & postFilter : postFilters_) + os << postFilter << std::endl; } } diff --git a/src/ufo/ObsFilters.h b/src/ufo/ObsFilters.h index 35e48e6a6..c7d8db939 100644 --- a/src/ufo/ObsFilters.h +++ b/src/ufo/ObsFilters.h @@ -12,7 +12,6 @@ #include -#include "ioda/ObsDataVector.h" #include "oops/base/ObsVariables.h" #include "oops/base/Variables.h" @@ -66,6 +65,7 @@ class ObsFiltersParameters : public oops::Parameters { class ObsFilters : public util::Printable, private boost::noncopyable { typedef std::vector FilterParams_; + template using ObsDataPtr_ = std::shared_ptr >; public: /// Initialize all filters for \p obspace, from parameters, using @@ -74,7 +74,7 @@ class ObsFilters : public util::Printable, /// assimilation. ObsFilters(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector & flags, ioda::ObsDataVector & obserr, + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration = 0); void preProcess(); @@ -95,9 +95,6 @@ class ObsFilters : public util::Printable, std::vector & filters); ioda::ObsSpace & obsspace_; - /// QCmanager filter to handle QC flags and collect statistics, - /// whenever at least one filter is configured. - std::unique_ptr qcmanager_; // List of filters for which the stage (pre/prior/post) will be determined automatically. std::vector autoFilters_; // List of filters which have been designated to run at the pre stage. @@ -108,9 +105,8 @@ class ObsFilters : public util::Printable, std::vector postFilters_; oops::Variables geovars_; oops::ObsVariables diagvars_; - ioda::ObsDataVector & qcflags_; - ioda::ObsDataVector & obserr_; - bool atLeastOneFilterConfigured_ = false; + ObsDataPtr_ qcflags_; + ObsDataPtr_ obserrfilter_; const int iteration_; }; diff --git a/src/ufo/ObsTraits.h b/src/ufo/ObsTraits.h index 84c2ab857..9b66e4a08 100644 --- a/src/ufo/ObsTraits.h +++ b/src/ufo/ObsTraits.h @@ -1,8 +1,8 @@ /* * (C) Copyright 2017-2020 UCAR - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #ifndef UFO_OBSTRAITS_H_ @@ -11,7 +11,6 @@ #include #include "ioda/ObsDataVector.h" -#include "ioda/ObsIterator.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" @@ -40,7 +39,6 @@ struct ObsTraits { template using ObsDataVector = ioda::ObsDataVector; typedef ufo::ObsError ObsError; typedef ufo::ObsFilters ObsFilter; - typedef ioda::ObsIterator GeometryIterator; typedef ufo::ObsOperator ObsOperator; typedef ufo::LinearObsOperator LinearObsOperator; diff --git a/src/ufo/errors/ObsErrorBase.h b/src/ufo/errors/ObsErrorBase.h index b08619b33..531649d3b 100644 --- a/src/ufo/errors/ObsErrorBase.h +++ b/src/ufo/errors/ObsErrorBase.h @@ -93,8 +93,6 @@ class ObsErrorParametersWrapper : public oops::Parameters { /// is deserialized. oops::PolymorphicParameter errorParameters{"covariance model", "diagonal", this}; - - const ObsErrorParametersBase & errorParams() const {return this->errorParameters;} }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorCrossVarCov.cc b/src/ufo/errors/ObsErrorCrossVarCov.cc index f24eaece1..a97126f3c 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.cc +++ b/src/ufo/errors/ObsErrorCrossVarCov.cc @@ -34,7 +34,7 @@ ObsErrorCrossVarCov::ObsErrorCrossVarCov(const Parameters_ & params, varcorrelations_(Eigen::MatrixXd::Identity(stddev_.nvars(), stddev_.nvars())), reconditioner_(nullptr) { - oops::Log::trace() << "ObsErrorCrossVarCov::ObsErrorCrossVarCov starting" << std::endl; + // deserialize configuration into ObsErrorCrossVarCovParameters // Create reconditioner reconditioner_.reset(new ObsErrorReconditioner(params_.reconditioning.value())); // Open and read error correlations from the hdf5 file @@ -107,7 +107,6 @@ ObsErrorCrossVarCov::ObsErrorCrossVarCov(const Parameters_ & params, << " channels/variables in file: " << params.inputFile.value() << ". To see which channels, turn on OOPS_TRACE\n" << std::endl; } - oops::Log::trace() << "ObsErrorCrossVarCov::ObsErrorCrossVarCov finished" << std::endl; } // ----------------------------------------------------------------------------- @@ -310,80 +309,22 @@ void ObsErrorCrossVarCov::inverseMultiply(ioda::ObsVector & dy) const { // ----------------------------------------------------------------------------- void ObsErrorCrossVarCov::localize(ioda::ObsVector & locvector) const { - oops::Log::trace() << "ufo::ObsErrorCrossVarCov::localize start" << std::endl; - - const double missing = util::missingValue(); - - ASSERT_MSG(locvector.size() == stddev_.size(), - "ObsErrorCrossVarCov::localize: " - "Localization vector dimension must match dimension of obs space."); - std::vector localstddev; - const size_t nlocs = locvector.nlocs(); - const size_t nvars = locvector.nvars(); - local_jvars_.clear(); - local_nobs_.clear(); - for (size_t jloc = 0; jloc < nlocs; ++jloc) { - size_t nused = 0; - for (size_t jvar = 0; jvar < nvars; ++jvar) { - size_t jj = jloc*nvars + jvar; - if (locvector[jj] != missing && locvector[jj] <= 0) { - throw eckit::BadValue("Localization weights must be positive. Use " - "oops::util::missingValue() to indicate " - "an observation with a weight of zero."); - } - if (locvector[jj] != missing && stddev_[jj] != missing) { - localstddev.push_back(stddev_[jj] * std::pow(locvector[jj], -0.5)); - local_jvars_.push_back(jvar); - ++nused; - } - } - if (nused > 0) local_nobs_.push_back(nused); - } - local_stddev_ = Eigen::Map(localstddev.data(), localstddev.size()); + throw eckit::BadParameter("Trying to localize a correlated R matrix, this is " + "not yet implemented."); } int ObsErrorCrossVarCov::localDim() const { - return local_stddev_.size(); + throw eckit::BadParameter("Trying to localize a correlated R matrix, this is " + "not yet implemented."); } Eigen::MatrixXf ObsErrorCrossVarCov::localInverseMultiply(const Eigen::MatrixXf & zz) const { - ASSERT_MSG(zz.cols() == localDim(), - "ObsErrorCrossVarCov::localInverseMultiply: " - "Input vector dimension must match local R matrix dimension"); - Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); - size_t j_obs = 0; - // loop over obs locations in local volume - for (int pt_nobs : local_nobs_) { - // construct pt std dev and correlations for obs at this point - Eigen::VectorXf pt_stddev = local_stddev_.segment(j_obs, pt_nobs).cast(); - Eigen::MatrixXf pt_corr = Eigen::MatrixXf::Identity(pt_nobs, pt_nobs); - for (size_t ipt = 0; ipt < pt_nobs; ++ipt) { - for (size_t jpt = 0; jpt < ipt; ++jpt) { - // only fill lower triangle as that's all we need for LLT - pt_corr(ipt, jpt) = static_cast(varcorrelations_(local_jvars_[j_obs+ipt], - local_jvars_[j_obs+jpt])); - } - } - // Cholesky decomposition to allow inverse multiplication - Eigen::LLT LLT(pt_corr); - for (size_t ii = 0; ii < zz.rows(); ++ii) { - Eigen::VectorXf row = zz(ii, Eigen::seq(j_obs, j_obs+pt_nobs-1)).transpose(); - // D^{-1/2} * row - row.array() /= pt_stddev.array(); - // C^{-1} * D^{-1/2} * row - LLT.solveInPlace(row); - // D^{-1/2} * C^{-1} * D^{-1/2} * row - row.array() /= pt_stddev.array(); - // store in output - zzRinv(ii, Eigen::seq(j_obs, j_obs+pt_nobs-1)) = row.transpose(); - } - j_obs += pt_nobs; - } - return zzRinv; + throw eckit::BadParameter("Localisation not implemented for correlated R matrices"); } Eigen::VectorXd ObsErrorCrossVarCov::local_invVarR() const { - throw eckit::BadParameter("ObsErrorCrossVarCov::local_invVarR not implemented."); + throw eckit::BadParameter("Local inverse variance not implemented for " + "correlated R matrices"); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorCrossVarCov.h b/src/ufo/errors/ObsErrorCrossVarCov.h index a7c8c5663..4d18033e8 100644 --- a/src/ufo/errors/ObsErrorCrossVarCov.h +++ b/src/ufo/errors/ObsErrorCrossVarCov.h @@ -13,7 +13,6 @@ #include #include -#include #include "ioda/ObsVector.h" @@ -39,6 +38,9 @@ class ObsErrorCrossVarCovParameters : public ObsErrorParametersBase { /// Input file containing correlations or covariances. If covariances are /// specified, they will be converted to correlations. oops::RequiredParameter inputFile{"input file", this}; + + oops::Parameter reconditioning{"reconditioning", + ObsErrorReconditionerParameters(), this}; }; // ----------------------------------------------------------------------------- /// \brief Observation error covariance matrix with cross-variable @@ -110,12 +112,6 @@ class ObsErrorCrossVarCov : public ObsErrorBase { Parameters_ params_; /// Observation error standard deviations ioda::ObsVector stddev_; - /// Localised observation error standard deviations - mutable Eigen::VectorXd local_stddev_; - /// Variable indices of local obs, used to construct local correlations - mutable std::vector local_jvars_; - /// Number of local obs at each location, used to construct local correlations - mutable std::vector local_nobs_; /// Variables for which correlations are defined (same as ObsSpace::obsvariables()) const oops::ObsVariables vars_; /// Correlations between variables diff --git a/src/ufo/errors/ObsErrorDiagonal.cc b/src/ufo/errors/ObsErrorDiagonal.cc index 86906d993..429f1dad5 100644 --- a/src/ufo/errors/ObsErrorDiagonal.cc +++ b/src/ufo/errors/ObsErrorDiagonal.cc @@ -92,7 +92,7 @@ void ObsErrorDiagonal::localize(ioda::ObsVector & locvector) const { Eigen::MatrixXf ObsErrorDiagonal::localInverseMultiply(const Eigen::MatrixXf & zz) const { Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); for (int ii = 0; ii < zz.rows(); ++ii) { - zzRinv(ii, Eigen::placeholders::all) = zz(ii, Eigen::placeholders::all) + zzRinv(ii, Eigen::all) = zz(ii, Eigen::all) .cwiseProduct(local_inverseVariance_.cast().transpose()); } return zzRinv; diff --git a/src/ufo/errors/ObsErrorDiagonalInvGamma.cc b/src/ufo/errors/ObsErrorDiagonalInvGamma.cc index 77da67f7f..7ff75d25e 100644 --- a/src/ufo/errors/ObsErrorDiagonalInvGamma.cc +++ b/src/ufo/errors/ObsErrorDiagonalInvGamma.cc @@ -94,7 +94,7 @@ void ObsErrorDiagonalInvGamma::localize(ioda::ObsVector & locvector) const { Eigen::MatrixXf ObsErrorDiagonalInvGamma::localInverseMultiply(const Eigen::MatrixXf & zz) const { Eigen::MatrixXf zzRinv(zz.rows(), zz.cols()); for (int ii = 0; ii < zz.rows(); ++ii) { - zzRinv(ii, Eigen::placeholders::all) = zz(ii, Eigen::placeholders::all) + zzRinv(ii, Eigen::all) = zz(ii, Eigen::all) .cwiseProduct(local_inverseVariance_.cast().transpose()); } return zzRinv; diff --git a/src/ufo/errors/ObsErrorDiffusion.cc b/src/ufo/errors/ObsErrorDiffusion.cc index 7dcc65afc..4e9a5ebeb 100644 --- a/src/ufo/errors/ObsErrorDiffusion.cc +++ b/src/ufo/errors/ObsErrorDiffusion.cc @@ -10,12 +10,9 @@ #include #include -#include "atlas/grid/Grid.h" #include "atlas/mesh/Mesh.h" #include "atlas/mesh/MeshBuilder.h" #include "atlas/output/Gmsh.h" -#include "atlas/util/Geometry.h" -#include "atlas/util/KDTree.h" #include "eckit/exception/Exceptions.h" #include "eckit/mpi/Comm.h" @@ -50,25 +47,23 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, inverseVariance_ = stddev_; inverseVariance_ *= stddev_; inverseVariance_.invert(); - oops::Log::trace() << "ObsErrorDiffusion::ObsErrorDiffusion constructor start" << std::endl; + oops::Log::trace() << "ObsErrorDiffusion:ObsErrorDiffusion constructed nobs = " + << stddev_.nobs() << std::endl; // Set up diffusion grid as atlas functionspace - const atlas::idx_t nlocs = stddev_.nlocs(); + size_t nlocs = stddev_.nlocs(); std::vector lons(nlocs); std::vector lats(nlocs); std::vector gridXY(2*nlocs); - std::vector obsnodes(nlocs); // Get obs locations from obs space odb.get_db("MetaData", "longitude", lons); odb.get_db("MetaData", "latitude", lats); // fill vector in form: [ lon_0, lat_0, lon_1, lat_1, ... ] - for (atlas::idx_t i = 0; i < nlocs; ++i) { + for (size_t i = 0; i < nlocs; ++i) { gridXY[2*i] = lons[i]; gridXY[2*i+1] = lats[i]; - obsnodes[i] = atlas::PointLonLat(lons[i], lats[i]); - obsnodes[i].normalise(); } eckit::LocalConfiguration fspaceConfig; @@ -84,64 +79,22 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, atlas::Mesh mesh{}; atlas::FieldSet fset{}; -//------------------------------------------------------------------------------ -// If control grid parameters are provided, -// use them to create a coarser grid for the diffusion operator -// -// NOTE: not setting control grid as default as the grid spacing and -// remove within parameters should be explicitly set, -// depending on the observation network geometry, -// to ensure the control grid is created as intended -//------------------------------------------------------------------------------ - if (params_.controlGrid.value()) { - const int gridSpacing = params_.controlGrid.value()->gridSpacing.value(); - const double removeWithin = params_.controlGrid.value()->removeWithin.value(); - - // returns merged obs + remaining control grid points - std::vector mergedPoints = - createControlGrid(obsnodes, removeWithin, gridSpacing); - // store offset before updating gridXY - const atlas::idx_t nMerged = mergedPoints.size(); - obsOffset_ = nMerged - nlocs; - - // update gridXY with merged points for mesh creation - gridXY.resize(2 * mergedPoints.size()); - for (atlas::idx_t k = 0; k < nMerged; k++) { - gridXY[2*k] = mergedPoints[k].lon(); - gridXY[2*k+1] = mergedPoints[k].lat(); - } - fspaceConfig.set("grid.xy", gridXY); - } // Uses Delauney triangulation for mesh generation util::setupFunctionSpace(comm_, fspaceConfig, grid, partitioner, mesh, fspace, fset); - geom_.reset(new oops::GeometryData(fspace, fset, true, comm_)); - diffusionGeom_ = oops::Diffusion::calculateDerivedGeom(std::as_const(*geom_)); - diffusion_.reset(new oops::Diffusion(std::as_const(*geom_), diffusionGeom_)); - - const atlas::idx_t npts = geom_->functionSpace().size(); - // Save mesh and point-type marker field for visualization with gmsh + // save output for viewing with gmsh if (params_.outputDiffusionMesh.value()) { - // Create marker field: 0 = control grid point, 1 = obs point - atlas::Field markerField = geom_->functionSpace().createField( - atlas::option::levels(1) | atlas::option::name("pointType")); - auto v_marker = atlas::array::make_view(markerField); - v_marker.assign(0.0); - for (atlas::idx_t i = 0; i < nlocs; ++i) { - v_marker(obsOffset_ + i, 0) = 1.0; // set marker for obspoints to 1 - } - fset.add(markerField); // Add to fieldset before writing - // Write mesh with the marker field - const std::string filename = "diffusion_mesh.msh"; + std::string filename = "diffusion_mesh.msh"; atlas::output::Gmsh gmsh(filename, atlas::util::Config("coordinates", "xyz") | atlas::util::Config("ghost", true)); // enables viewing halos per task gmsh.write(mesh); - gmsh.write(fset, geom_->functionSpace()); - oops::Log::info() << "ObsErrorDiffusion: Saved diffusion mesh to " - << filename << std::endl; } + geom_.reset(new oops::GeometryData(fspace, fset, true, comm_)); + diffusionGeom_ = oops::Diffusion::calculateDerivedGeom(std::as_const(*geom_)); + diffusion_.reset(new oops::Diffusion(std::as_const(*geom_), diffusionGeom_)); + // ------------------------------------------------------------ // Create horizontal length scales for normalization // ------------------------------------------------------------ @@ -151,10 +104,6 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, auto v_hzScales = atlas::array::make_view(hzScales); const double val = params_.lscale; v_hzScales.assign(val); - // Set the scales for control points to 0 so they don't influence the diffusion of obs points - for (atlas::idx_t i = 0; i < obsOffset_; ++i) { - v_hzScales(i, 0) = 0; - } atlas::FieldSet scales; scales.add(hzScales); @@ -207,7 +156,7 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, // ---- Update Welford running variance ---- auto v_rand = atlas::array::make_view(randSet["rand"]); - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { double f = v_rand(i, 0); double old_m = v_m(i, 0); double new_m = old_m + (f - old_m) / itr; @@ -217,14 +166,12 @@ ObsErrorDiffusion::ObsErrorDiffusion(const Parameters_ & params, } // ---- Finalize normalization: Γ_i = 1/sqrt(Var_i) ---- - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { if (v_s(i, 0) > 0.0) { v_norm(i, 0) = 1.0 / std::sqrt(v_s(i, 0) / (randomizationIterations - 1)); } } - oops::Log::trace() << "ObsErrorDiffusion::ObsErrorDiffusion constructed nobs = " - << stddev_.nobs() << std::endl; -} // end constructor +} // ----------------------------------------------------------------------------- @@ -261,8 +208,7 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // NOTE: This will only work for reading a single variable // (with single channel) from the obsSpace - const atlas::idx_t nlocs = dy.nlocs(); - const atlas::idx_t npts = geom_->functionSpace().size(); + size_t nlocs = dy.nlocs(); atlas::FieldSet fset; for (std::string var : params_.var.value()) { @@ -271,8 +217,8 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // copy data from obsVector to field auto obsView = atlas::array::make_view(obs); - for (atlas::idx_t i = 0; i < nlocs; ++i) { - obsView(obsOffset_ + i, 0) = dy.data()[i]; + for (atlas::idx_t i = 0; i < obsView.shape(0); ++i) { + obsView(i, 0) = dy.data()[i]; } // add obs "var" to fieldSet @@ -290,7 +236,7 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { auto applyNormSqrt = [&](atlas::Field & f) { auto v_f = atlas::array::make_view(f); auto v_norm = atlas::array::make_view(this->hzNorm_); - for (atlas::idx_t i = 0; i < npts; i++) { + for (atlas::idx_t i = 0; i < nlocs; i++) { v_f(i, 0) *= v_norm(i, 0); } }; @@ -307,9 +253,8 @@ void ObsErrorDiffusion::multiply(ioda::ObsVector & dy) const { // move this step into diffusion loop (above) for multiple variables? for (auto field : fset) { auto fieldView = atlas::array::make_view(field); - // using nlocs here since only copying obs points back to dy - for (atlas::idx_t i = 0; i < nlocs; ++i) { - dy[i] = fieldView(obsOffset_ + i, 0); + for (atlas::idx_t i = 0; i < fieldView.shape(0); ++i) { + dy[i] = fieldView(i, 0); } } @@ -367,8 +312,8 @@ void ObsErrorDiffusion::randomize(ioda::ObsVector & dy) const { // NOTE:: double check this works for list of more than one var in yaml for (std::string var : vars) { auto randView = atlas::array::make_view(rand[var]); - for (atlas::idx_t i = 0; i < dy.nlocs(); ++i) { - dy[i] = randView(obsOffset_ + i, 0); + for (int i = 0; i < randView.shape(0); ++i) { + dy[i] = randView(i, 1); } } } @@ -426,69 +371,5 @@ void ObsErrorDiffusion::print(std::ostream & os) const { // ----------------------------------------------------------------------------- -std::vector ObsErrorDiffusion::createControlGrid( - const std::vector& obsnodes, - const double removeWithin, - const int gridSpacing) - { - oops::Log::trace() << "ObsErrorDiffusion CreateControlGrid: start " << std::endl; - const atlas::idx_t nlocs = obsnodes.size(); - // --- 1. Build control grid --- - // -------------------------------- - // number of grid points in longitude - const int nx = static_cast(std::round(360.0 / gridSpacing)); - // number of grid points in latitude with equator in center - const int ny = static_cast(std::round(180.0 / gridSpacing)) + 1; - const atlas::util::Config cfg = atlas::util::Config("type", "regular_lonlat") - ("nx", nx) - ("ny", ny) - ("lon0", 0.0) - ("lat0", -90.0); - atlas::RegularLonLatGrid cntrlGrid(cfg); - // --- 2. Build KD-tree from obs --- - // ------------------------------------- - atlas::util::IndexKDTree obstree; // declare kd tree - std::vector indices(nlocs); // for payload - std::iota(indices.begin(), indices.end(), 0); // assign index to obs payload - obstree.build(obsnodes, indices); // build KDTree from obs locations - // --- 3. Filter control grid points --- - // -------------------------------------- - std::vector remainingGridPoints; - remainingGridPoints.reserve(nx * ny); - int nRemoved = 0; - for (atlas::idx_t j = 0; j < cntrlGrid.ny(); j++) { - for (atlas::idx_t i = 0; i < cntrlGrid.nx(); i++) { - const atlas::PointLonLat gridPt = cntrlGrid.lonlat(i, j); - const auto result = obstree.closestPoint(gridPt); - if (result.distance() <= removeWithin) { - nRemoved++; - oops::Log::debug() << "ObsErrorDiffusion: CreateControlGrid:" - << "Removed grid point (" - << gridPt.lon() << ", " << gridPt.lat() - << ") — nearest obs #" << result.payload() - << " (lon=" << obsnodes[result.payload()].lon() - << ", lat=" << obsnodes[result.payload()].lat() - << ") at distance=" << result.distance() << "m\n"; - } else { - remainingGridPoints.push_back(gridPt); - } - } - } - oops::Log::info() << "ObsErrorDiffusion: CreateControlGrid:" - << "Control grid total=" - << nx*ny - << " removed=" << nRemoved - << " remaining=" << remainingGridPoints.size() << std::endl; - // --- 4. Merge remaining control grid points with obs locations --- - //--------------------------------------------------------------------- - std::vector mergedPoints; - mergedPoints.reserve(remainingGridPoints.size() + nlocs); - mergedPoints.insert(mergedPoints.end(), - remainingGridPoints.begin(), remainingGridPoints.end()); - mergedPoints.insert(mergedPoints.end(), - obsnodes.begin(), obsnodes.end()); - oops::Log::trace() << "ObsErrorDiffusion: CreateControlGrid: end" << std::endl; - return mergedPoints; - } } // namespace ufo diff --git a/src/ufo/errors/ObsErrorDiffusion.h b/src/ufo/errors/ObsErrorDiffusion.h index ce36cedb5..ffa604092 100644 --- a/src/ufo/errors/ObsErrorDiffusion.h +++ b/src/ufo/errors/ObsErrorDiffusion.h @@ -37,13 +37,6 @@ namespace ufo { class ObsErrorDiffusionParameters : public ObsErrorParametersBase { OOPS_CONCRETE_PARAMETERS(ObsErrorDiffusionParameters, ObsErrorParametersBase) public: - class ControlGrid : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(ControlGrid, oops::Parameters) - public: - oops::RequiredParameter gridSpacing{"grid spacing in degrees", this}; - oops::RequiredParameter removeWithin{"remove within meters", this}; - }; - oops::RequiredParameter> var{"correlation variable names", "Group/Name obs variable that correlations should be computed for (note: " "this variable should be the same variable as obs space is grouped on", this}; @@ -54,11 +47,8 @@ class ObsErrorDiffusionParameters : public ObsErrorParametersBase { 100, this}; oops::Parameter InverseAccuracy{"Accuracy of inverse", 1.0e-7, this}; oops::Parameter outputDiffusionMesh{"Output diffusion mesh", false, this}; - // Add parameter for control grid - oops::OptionalParameter controlGrid{"control grid", this}; }; - // ----------------------------------------------------------------------------- /// \brief Correlated observation error covariance matrix. class ObsErrorDiffusion : public ObsErrorBase { @@ -137,17 +127,6 @@ class ObsErrorDiffusion : public ObsErrorBase { const eckit::mpi::Comm & comm_; std::unique_ptr diffusion_; atlas::Field hzNorm_; - - // Offset where obs locations start in the merged function space - // (0 if no control grid, nRemainingControlPoints otherwise) - atlas::idx_t obsOffset_ = 0; // <-- add here - - /// Creates control grid, removes close control points and - // returns merged grid points (remaining control grid + obs locations) - std::vector createControlGrid( - const std::vector& obsnodes, - const double removeWithin, - const int gridSpacing); }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/errors/ObsErrorParametersBase.h b/src/ufo/errors/ObsErrorParametersBase.h index c3ad40ba7..e493292d2 100644 --- a/src/ufo/errors/ObsErrorParametersBase.h +++ b/src/ufo/errors/ObsErrorParametersBase.h @@ -14,8 +14,6 @@ #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" -#include "ufo/errors/ObsErrorReconditioner.h" - namespace ufo { /// \brief Base obs errors parameters class @@ -26,9 +24,6 @@ class ObsErrorParametersBase : public oops::Parameters { oops::Parameter model{"covariance model", "diagonal", this}; oops::Parameter RMSEtolerance{"Obs Error test tolerance", "RMSE tolerance for" "oops::ObsErrorCovariance test", 1.0e-10, this}; - - oops::Parameter reconditioning{"reconditioning", - ObsErrorReconditionerParameters(), this}; }; } // namespace ufo diff --git a/src/ufo/errors/ObsErrorReconditioner.cc b/src/ufo/errors/ObsErrorReconditioner.cc index 93b5f8203..090c5c6af 100644 --- a/src/ufo/errors/ObsErrorReconditioner.cc +++ b/src/ufo/errors/ObsErrorReconditioner.cc @@ -16,7 +16,6 @@ ObsErrorReconditionerMethodParameterTraitsHelper::namedValues[]; ObsErrorReconditioner::ObsErrorReconditioner(const Parameters_ & params) : params_(params) { - oops::Log::trace() << "ObsErrorReconditioner::ObsErrorReconditioner starting" << std::endl; // Checking valid reconditioning options if reconditioning specified. ufo::ObsErrorReconditionerMethod recon_method = params_.ReconMethod.value(); size_t nvalid_options = 0; @@ -45,12 +44,10 @@ ObsErrorReconditioner::ObsErrorReconditioner(const Parameters_ & params) break; case ufo::ObsErrorReconditionerMethod::NORECONDITIONING: oops::Log::trace() << "'No reconditioning' option selected, " - "for advanced reconditioning; " "recondition method can be tested, " "R matrix should not change.\n"; break; } - oops::Log::trace() << "ObsErrorReconditioner::ObsErrorReconditioner finished" << std::endl; } // ObsErrorReconditioner::ObsErrorReconditioner void ObsErrorReconditioner::recondition(Eigen::MatrixXd & R) const { @@ -64,7 +61,8 @@ void ObsErrorReconditioner::recondition(Eigen::MatrixXd & R) const { // Check square matrix size_t nrows = R.rows(); - assert(nrows == R.cols()); + size_t ncols = R.cols(); + assert(nrows == ncols); // Performing eigendecomposition oops::Log::trace() << "R before reconditioning:\n" << R << std::endl << std::endl; diff --git a/src/ufo/errors/ObsErrorWithinGroupCov.cc b/src/ufo/errors/ObsErrorWithinGroupCov.cc index 19cf9512c..ff034d312 100644 --- a/src/ufo/errors/ObsErrorWithinGroupCov.cc +++ b/src/ufo/errors/ObsErrorWithinGroupCov.cc @@ -60,19 +60,6 @@ double markov(const double & distnorm, const double & maxnormdist) { return markovvalue; } -double gaussian(const double & distnorm) { - // computes a Gaussian-shaped localization -/* - // cutoff to zero at normalized distance of 2.5 - double gaussVal = 0.0; - if (distnorm < 2.5) { - gaussVal = exp(-0.5*pow(distnorm, 2)); - } - return gaussVal; -*/ - return exp(-0.5*pow(distnorm, 2)); -} - // ----------------------------------------------------------------------------- // Distance functions - base class and linear distance // ----------------------------------------------------------------------------- @@ -118,15 +105,11 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, params_(params), obspace_(obspace), coord_(coord_constructor(params_, obspace)), - stddev_(obspace, "ObsError"), - reconditioner_(nullptr) + stddev_(obspace, "ObsError") { correlations_.reserve(obspace.nrecs()); double missing_double = util::missingValue(); - // Create reconditioner - reconditioner_.reset(new ObsErrorReconditioner(params_.reconditioning.value())); - for (auto irec = obspace.recidx_begin(); irec != obspace.recidx_end(); ++irec) { std::vector rec_idx = obspace.recidx_vector(irec); size_t rec_nobs = rec_idx.size(); @@ -161,22 +144,14 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, corr(jloc, iloc) = markov(distance / params_.lscale.value(), params_.markovLengthscaleFactor.value()); break; - case CorrelationFunctions::GAUSSIAN: - corr(jloc, iloc) = gaussian(distance / params_.lscale.value()); - break; } } } - // The Markov and Gaussian correlation profiles do not guarantee a positive definite matrix - // therefore a check is needed. If the matrix is not positive definite the diagonal is - // slightly inflated to make it positive definite which has the effect of slightly inflating - // the observation errors. - if (params_.correlationFunction.value() == CorrelationFunctions::MARKOV || - (params_.correlationFunction.value() == CorrelationFunctions::GAUSSIAN && - params_.applyBasicReconditioning.value())) { - oops::Log::trace() << "ObsErrorWithinGroupCov::ObsErrorWithinGroupCov " - << "basic reconditining enabled. " - << "Checking eigenvalues of correlation matrix" << std::endl; + // The Markov correlation method does not guarantee a positive definite matrix therefore a + // check is needed. If the matrix is not positive definite the diagonal is slightly inflated + // to make it positive definite which has the effect of slightly inflating the observation + // errors. + if (params_.correlationFunction.value() == CorrelationFunctions::MARKOV) { Eigen::SelfAdjointView corr_view(corr); Eigen::SelfAdjointEigenSolver es(corr_view); double min_eigenvalue = es.eigenvalues().minCoeff(); @@ -189,119 +164,17 @@ ObsErrorWithinGroupCov::ObsErrorWithinGroupCov(const Parameters_ & params, } } correlations_.push_back(corr); - } // end loop over records + } } // ----------------------------------------------------------------------------- void ObsErrorWithinGroupCov::update(const ioda::ObsVector & obserr) { - oops::Log::trace() << "ObsErrorWithinGroupCov::update() start" << std::endl; stddev_ = obserr; - if (params_.reconditioning.value().ReconMethod.value() != - ufo::ObsErrorReconditionerMethod::NORECONDITIONING) { - this->recondition(obserr); - } - oops::Log::trace() << "ObsErrorWithinGroupCov::update() end" << std::endl; } // ----------------------------------------------------------------------------- -void ObsErrorWithinGroupCov::recondition(const ioda::ObsVector & mask) { - oops::Log::trace() << "ObsErrorWithinGroupCov::recondition() start" << std::endl; - const size_t nlocs = mask.nlocs(); - const size_t nvars = mask.nvars(); - const double missing = util::missingValue(); - - for (Eigen::MatrixXd groupCorr : correlations_) { - // preallocate data - Eigen::MatrixXd avgcorr = Eigen::MatrixXd::Zero(groupCorr.rows(), groupCorr.cols()); - size_t nused_locs = 0; - const double dnlocs = static_cast(nlocs); - - // Masking and packing R matrix at each location - // loop over all observations locations - for (size_t jloc = 0; jloc < nlocs; ++jloc) { - std::vector usedobs_indices(nvars); - // Calculating variables that are used from mask - // These are the variables that pass QC. - size_t nused = 0; - for (size_t jvar = 0; jvar < nvars; ++jvar) - if (mask[jloc * nvars + jvar] != missing) usedobs_indices[nused++] = jvar; - - // Initialising correlation matrix for the given location. - // This will be a fraction of the correlation matrix after - // the reconditioning has happened. - Eigen::MatrixXd corr_at_loc(groupCorr / dnlocs); - if (nused <= 1) { - oops::Log::trace() << "nused = " << nused - << "at jloc = " << jloc - << ", skipping reconditioning.\n"; - continue; - } - oops::Log::trace() << "\nReconditioning R matrix at jloc = " << jloc << std::endl; - Eigen::MatrixXd R = Eigen::MatrixXd::Zero(nused, nused); - - // loop over all used variables at location - for (size_t jvar = 0; jvar < nused; ++jvar) { - const size_t ivar = usedobs_indices[jvar]; - const size_t ind = jloc * nvars + ivar; - R(jvar, jvar) = stddev_[ind]*stddev_[ind]; - for (size_t jvar2 = jvar + 1; jvar2 < nused; ++jvar2) { - const size_t ivar2 = usedobs_indices[jvar2]; - const size_t ind2 = jloc * nvars + ivar2; - R(jvar, jvar2) = groupCorr(ivar, ivar2) - * stddev_[ind] - * stddev_[ind2]; - R(jvar2, jvar) = groupCorr(ivar2, ivar) - * stddev_[ind2] - * stddev_[ind]; - } - } - - // Recondition the R matrix - this->reconditioner_->recondition(R); - - // Unpacking the reconditioned matrix - // into stddev_ and groupCorr members - // loop over all used variables at location - for (size_t jvar = 0; jvar < nused; ++jvar) { - const size_t ivar = usedobs_indices[jvar]; - const size_t ind = jloc * nvars + ivar; - // Updating stddev_ - const double stddev_ind = std::sqrt(R(jvar, jvar)); - stddev_[ind] = stddev_ind; - for (size_t jvar2 = jvar + 1; jvar2 < nused; ++jvar2) { - // Ensuring location independent correlations - // by using average of reconditioned correlations at each location - const double stddev_ind2 = std::sqrt(R(jvar2, jvar2)); - const size_t ivar2 = usedobs_indices[jvar2]; - corr_at_loc(ivar, ivar2) = R(jvar, jvar2) - / (stddev_ind - * stddev_ind2 - * dnlocs); - corr_at_loc(ivar2, ivar) = R(jvar2, jvar) - / (stddev_ind2 - * stddev_ind - * dnlocs); - } - } - nused_locs++; - avgcorr += corr_at_loc; - } - - // Reassigning the correlations to the renormalised - // average correlations, if any locations are used - if (nused_locs > 0) { - groupCorr = avgcorr - * dnlocs - / static_cast(nused_locs); - } - } - oops::Log::trace() << "ObsErrorWithinGroupCov::recondition() end" << std::endl; -} // recondition - -// ----------------------------------------------------------------------------- - void ObsErrorWithinGroupCov::multiply(ioda::ObsVector & dy) const { // R * dy = D^{1/2} * C * D^{1/2} * dy // where D^{1/2} - diagonal matrix with stddev_ on the diagonal diff --git a/src/ufo/errors/ObsErrorWithinGroupCov.h b/src/ufo/errors/ObsErrorWithinGroupCov.h index 9314e0a2e..68089824b 100644 --- a/src/ufo/errors/ObsErrorWithinGroupCov.h +++ b/src/ufo/errors/ObsErrorWithinGroupCov.h @@ -21,7 +21,6 @@ #include "ufo/errors/ObsErrorBase.h" #include "ufo/errors/ObsErrorParametersBase.h" -#include "ufo/errors/ObsErrorReconditioner.h" namespace ioda { class ObsSpace; @@ -45,8 +44,7 @@ struct DistanceFunctionsParameterTraitsHelper { enum class CorrelationFunctions { GC99, - MARKOV, - GAUSSIAN + MARKOV }; struct CorrelationFunctionsParameterTraitsHelper { @@ -54,8 +52,7 @@ struct CorrelationFunctionsParameterTraitsHelper { static constexpr char enumTypeName[] = "CorrelationFunctions"; static constexpr util::NamedEnumerator namedValues[] = { { CorrelationFunctions::GC99, "gc99" }, - { CorrelationFunctions::MARKOV, "markov" }, - { CorrelationFunctions::GAUSSIAN, "gaussian"} + { CorrelationFunctions::MARKOV, "markov" } }; }; @@ -87,7 +84,7 @@ class ObsErrorWithinGroupCovParameters : public ObsErrorParametersBase { public: oops::Parameter correlationFunction{"correlation function", "Correlation function to use for correlation computations. " - "Currently only 'gc99', 'markov', and 'gaussian' are supported", + "Currently only 'gc99' and 'markov' are supported", CorrelationFunctions::GC99, this}; oops::RequiredParameter lscale{"correlation lengthscale", "Correlation lengthscale used with the correlation functions", this}; @@ -102,10 +99,6 @@ class ObsErrorWithinGroupCovParameters : public ObsErrorParametersBase { "The lengthscale factor is multiplied by the lengthscale to provide the limit in " "which correlations are evaluated. Beyond this distance correlation values are set to " "zero.", 1.0, this}; - oops::Parameter applyBasicReconditioning{"apply basic reconditioning", - "Apply a simple ridge regression reconditiong to the gaussian correlation profile." - "This is the same reconditioning that is applied to the markov correlation method", - false, this}; }; // ----------------------------------------------------------------------------- @@ -181,8 +174,6 @@ class ObsErrorWithinGroupCov : public ObsErrorBase { void print(std::ostream &) const override; /// Multiply only by correlations (diagnostics) void multiplyCorrelations(ioda::ObsVector & y) const; - /// Recondition the R matrix - called by update - void recondition(const ioda::ObsVector & mask); /// ObsError configuration as oops::Paramters Parameters_ params_; /// ObsSpace reference @@ -194,8 +185,6 @@ class ObsErrorWithinGroupCov : public ObsErrorBase { /// Each element holds a lower-triangle of the correlation matrix for all locations /// within one group std::vector correlations_; - /// Create reconditioner - std::unique_ptr reconditioner_; }; // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/AcceptList.cc b/src/ufo/filters/AcceptList.cc index abda07112..f2eab5885 100644 --- a/src/ufo/filters/AcceptList.cc +++ b/src/ufo/filters/AcceptList.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- AcceptList::AcceptList(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "AcceptList constructor" << std::endl; diff --git a/src/ufo/filters/AcceptList.h b/src/ufo/filters/AcceptList.h index 5a71b0582..69b922a06 100644 --- a/src/ufo/filters/AcceptList.h +++ b/src/ufo/filters/AcceptList.h @@ -54,8 +54,8 @@ class AcceptList : public FilterBase, static const std::string classname() {return "ufo::AcceptList";} AcceptList(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/AverageObsToGeoValLevels.cc b/src/ufo/filters/AverageObsToGeoValLevels.cc index 07416d1d4..e11923014 100644 --- a/src/ufo/filters/AverageObsToGeoValLevels.cc +++ b/src/ufo/filters/AverageObsToGeoValLevels.cc @@ -30,8 +30,8 @@ namespace ufo { AverageObsToGeoValLevels::AverageObsToGeoValLevels( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "AverageObsToGeoValLevels constructor" << std::endl; @@ -63,7 +63,7 @@ void AverageObsToGeoValLevels::applyFilter(const std::vector & apply, const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "AverageObsToGeoValLevels applyFilter start" << std::endl; - oops::Log::debug() << "AverageObsToGeoValLevels obserr: " << obserr_ << std::endl; + oops::Log::debug() << "AverageObsToGeoValLevels obserr: " << *obserr_ << std::endl; const float missing = util::missingValue(); ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); diff --git a/src/ufo/filters/AverageObsToGeoValLevels.h b/src/ufo/filters/AverageObsToGeoValLevels.h index 43c338678..e4aac8b08 100644 --- a/src/ufo/filters/AverageObsToGeoValLevels.h +++ b/src/ufo/filters/AverageObsToGeoValLevels.h @@ -81,8 +81,8 @@ class AverageObsToGeoValLevels : public FilterBase, static const std::string classname() {return "ufo::AverageObsToGeoValLevels";} AverageObsToGeoValLevels(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~AverageObsToGeoValLevels(); private: diff --git a/src/ufo/filters/BackgroundCheck.cc b/src/ufo/filters/BackgroundCheck.cc index 431103087..901235f5a 100644 --- a/src/ufo/filters/BackgroundCheck.cc +++ b/src/ufo/filters/BackgroundCheck.cc @@ -29,8 +29,8 @@ namespace ufo { // ----------------------------------------------------------------------------- BackgroundCheck::BackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr, VariableNameMap(parameters.AliasFile.value())), parameters_(parameters) { @@ -94,7 +94,7 @@ void BackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "BackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); const float missing = util::missingValue(); - oops::Log::debug() << "BackgroundCheck obserr: " << obserr_ << std::endl; + oops::Log::debug() << "BackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); std::string test_hofx = parameters_.test_hofx.value(); @@ -115,10 +115,10 @@ void BackgroundCheck::applyFilter(const std::vector & apply, for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - const bool obserrNotMissing = (obserr_[iv][jobs] + const bool obserrNotMissing = ((*obserr_)[iv][jobs] != util::missingValue>()); - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass && obserrNotMissing) { + decltype((*obserr_)[iv][jobs])>>()); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && obserrNotMissing) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); @@ -176,16 +176,16 @@ void BackgroundCheck::applyFilter(const std::vector & apply, for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - const bool obserrNotMissing = (obserr_[iv][jobs] + const bool obserrNotMissing = ((*obserr_)[iv][jobs] != util::missingValue>()); - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass && obserrNotMissing) { + decltype((*obserr_)[iv][jobs])>>()); + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass && obserrNotMissing) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); ASSERT(bias[jobs] != missing); const std::vector &errorMultiplier = - (thresholdWrtBGerror || thresholdWrtEnsembleSpread) ? hofxerr : obserr_[iv]; + (thresholdWrtBGerror || thresholdWrtEnsembleSpread) ? hofxerr : (*obserr_)[iv]; // Threshold for current observation float zz = (thr[jobs] == std::numeric_limits::max()) ? abs_thr[jobs] : std::min(abs_thr[jobs], thr[jobs] * errorMultiplier[jobs]); diff --git a/src/ufo/filters/BackgroundCheck.h b/src/ufo/filters/BackgroundCheck.h index e4302409f..cd70c7636 100644 --- a/src/ufo/filters/BackgroundCheck.h +++ b/src/ufo/filters/BackgroundCheck.h @@ -103,8 +103,8 @@ class BackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::BackgroundCheck";} BackgroundCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BackgroundCheck(); private: diff --git a/src/ufo/filters/BayesianBackgroundCheck.cc b/src/ufo/filters/BayesianBackgroundCheck.cc index c4aacaf8c..c645c54b1 100644 --- a/src/ufo/filters/BayesianBackgroundCheck.cc +++ b/src/ufo/filters/BayesianBackgroundCheck.cc @@ -35,8 +35,8 @@ namespace ufo { BayesianBackgroundCheck::BayesianBackgroundCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr, VariableNameMap(parameters.AliasFile.value())), parameters_(parameters) @@ -100,7 +100,7 @@ void BayesianBackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "BayesianBackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); - oops::Log::debug() << "BayesianBackgroundCheck obserr: " << obserr_ << std::endl; + oops::Log::debug() << "BayesianBackgroundCheck obserr: " << *obserr_ << std::endl; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); @@ -280,7 +280,7 @@ void BayesianBackgroundCheck::applyFilter(const std::vector & apply, // create reduced vectors, copied from full ones, fulfilling applycondition: std::vector firstComponentObVal_reduced = reduceVector(firstComponentObVal, j_reduced); - std::vector ObsErr_reduced = reduceVector(obserr_[varname1], j_reduced); + std::vector ObsErr_reduced = reduceVector((*obserr_)[varname1], j_reduced); std::vector hofx1_reduced = reduceVector(hofx1, j_reduced); std::vector hofxerr_reduced = reduceVector(hofxerr, j_reduced); std::vector PdBad_reduced = reduceVector(PdBad, j_reduced); diff --git a/src/ufo/filters/BayesianBackgroundCheck.h b/src/ufo/filters/BayesianBackgroundCheck.h index 2b203fae2..60c75c11f 100644 --- a/src/ufo/filters/BayesianBackgroundCheck.h +++ b/src/ufo/filters/BayesianBackgroundCheck.h @@ -132,8 +132,8 @@ class BayesianBackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::BayesianBackgroundCheck";} BayesianBackgroundCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~BayesianBackgroundCheck(); private: diff --git a/src/ufo/filters/BayesianBackgroundQCFlags.cc b/src/ufo/filters/BayesianBackgroundQCFlags.cc index 989100435..66fbdf434 100644 --- a/src/ufo/filters/BayesianBackgroundQCFlags.cc +++ b/src/ufo/filters/BayesianBackgroundQCFlags.cc @@ -23,8 +23,8 @@ namespace ufo { BayesianBackgroundQCFlags::BayesianBackgroundQCFlags (ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "BayesianBackgroundQCFlags constructor" << std::endl; diff --git a/src/ufo/filters/BayesianBackgroundQCFlags.h b/src/ufo/filters/BayesianBackgroundQCFlags.h index 4df7e5bec..d5978920c 100644 --- a/src/ufo/filters/BayesianBackgroundQCFlags.h +++ b/src/ufo/filters/BayesianBackgroundQCFlags.h @@ -63,8 +63,8 @@ class BayesianBackgroundQCFlags : public FilterBase, static const std::string classname() {return "ufo::BayesianBackgroundQCFlags";} BayesianBackgroundQCFlags(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BayesianBackgroundQCFlags(); private: diff --git a/src/ufo/filters/BlackList.cc b/src/ufo/filters/BlackList.cc index 6de08fd80..0105cadfc 100644 --- a/src/ufo/filters/BlackList.cc +++ b/src/ufo/filters/BlackList.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- BlackList::BlackList(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "BlackList constructor" << std::endl; diff --git a/src/ufo/filters/BlackList.h b/src/ufo/filters/BlackList.h index 2275333f9..35ee576da 100644 --- a/src/ufo/filters/BlackList.h +++ b/src/ufo/filters/BlackList.h @@ -46,8 +46,8 @@ class BlackList : public FilterBase, static const std::string classname() {return "ufo::BlackList";} BlackList(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BlackList(); private: diff --git a/src/ufo/filters/CMakeLists.txt b/src/ufo/filters/CMakeLists.txt index 45947e7a0..993bcb480 100644 --- a/src/ufo/filters/CMakeLists.txt +++ b/src/ufo/filters/CMakeLists.txt @@ -259,8 +259,6 @@ set ( filters_files obsfunctions/CloudDetectMinResidualIR.h obsfunctions/CloudFirstGuessMinimumResidual.cc obsfunctions/CloudFirstGuessMinimumResidual.h - obsfunctions/CircularDifference.cc - obsfunctions/CircularDifference.h obsfunctions/MetOfficeAllSkyErrorModel.cc obsfunctions/MetOfficeAllSkyErrorModel.h obsfunctions/ModelHeightAdjustedWindVectorComponent.cc @@ -329,8 +327,6 @@ set ( filters_files obsfunctions/SatwindIndivErrors.h obsfunctions/SatWindsLNVDCheck.cc obsfunctions/SatWindsLNVDCheck.h - obsfunctions/TimeBinner.cc - obsfunctions/TimeBinner.h obsfunctions/WindsSPDBCheck.cc obsfunctions/WindsSPDBCheck.h obsfunctions/SatWindsErrnormCheck.cc diff --git a/src/ufo/filters/ConventionalProfileProcessing.cc b/src/ufo/filters/ConventionalProfileProcessing.cc index 707b334f4..1e36e2fb6 100644 --- a/src/ufo/filters/ConventionalProfileProcessing.cc +++ b/src/ufo/filters/ConventionalProfileProcessing.cc @@ -31,8 +31,8 @@ namespace ufo { ConventionalProfileProcessing::ConventionalProfileProcessing (ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "ConventionalProfileProcessing constructor" << std::endl; @@ -207,7 +207,7 @@ namespace ufo { // Handles individual profile data ProfileDataHandler profileDataHandler(data_, - flags_, + *flags_, options_.DHParameters, apply, filtervars, diff --git a/src/ufo/filters/ConventionalProfileProcessing.h b/src/ufo/filters/ConventionalProfileProcessing.h index 083fa9aea..15f93981c 100644 --- a/src/ufo/filters/ConventionalProfileProcessing.h +++ b/src/ufo/filters/ConventionalProfileProcessing.h @@ -86,8 +86,8 @@ namespace ufo { static const std::string classname() {return "ufo::ConventionalProfileProcessing";} ConventionalProfileProcessing(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ConventionalProfileProcessing(); /// Return the number of mismatches between values produced by the checking routines diff --git a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc index 9bf95d8ac..3dcb70260 100644 --- a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc +++ b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.cc @@ -19,8 +19,8 @@ namespace ufo { CopyFlagsFromExtendedToOriginalSpace::CopyFlagsFromExtendedToOriginalSpace (ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "CopyFlagsFromExtendedToOriginalSpace constructor start" << std::endl; @@ -55,7 +55,7 @@ void CopyFlagsFromExtendedToOriginalSpace::applyFilter(const std::vector & const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "CopyFlagsFromExtendedToOriginalSpace applyFilter start" << std::endl; - oops::Log::debug() << "CopyFlagsFromExtendedToOriginalSpace obserr: " << obserr_ << std::endl; + oops::Log::debug() << "CopyFlagsFromExtendedToOriginalSpace obserr: " << *obserr_ << std::endl; // Number of locations. const size_t nlocs = obsdb_.nlocs(); diff --git a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h index 9125373dc..37fcb43ab 100644 --- a/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h +++ b/src/ufo/filters/CopyFlagsFromExtendedToOriginalSpace.h @@ -71,8 +71,8 @@ class CopyFlagsFromExtendedToOriginalSpace : public FilterBase, static const std::string classname() {return "ufo::CopyFlagsFromExtendedToOriginalSpace";} CopyFlagsFromExtendedToOriginalSpace(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> flags, + std::shared_ptr> obserr); ~CopyFlagsFromExtendedToOriginalSpace() override; private: diff --git a/src/ufo/filters/CreateDiagnosticFlags.cc b/src/ufo/filters/CreateDiagnosticFlags.cc index 6217e7db9..1adbf5006 100644 --- a/src/ufo/filters/CreateDiagnosticFlags.cc +++ b/src/ufo/filters/CreateDiagnosticFlags.cc @@ -22,9 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- CreateDiagnosticFlags::CreateDiagnosticFlags(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr> qcflags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(qcflags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "CreateDiagnosticFlags constructor start" << std::endl; diff --git a/src/ufo/filters/CreateDiagnosticFlags.h b/src/ufo/filters/CreateDiagnosticFlags.h index 2e43a8aca..4cf5d339b 100644 --- a/src/ufo/filters/CreateDiagnosticFlags.h +++ b/src/ufo/filters/CreateDiagnosticFlags.h @@ -98,8 +98,8 @@ class CreateDiagnosticFlags : public ObsProcessorBase, static const std::string classname() {return "ufo::CreateDiagnosticFlags";} CreateDiagnosticFlags(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> qcflags, + std::shared_ptr> obserr); ~CreateDiagnosticFlags() override; private: diff --git a/src/ufo/filters/DifferenceCheck.cc b/src/ufo/filters/DifferenceCheck.cc index 80ef13e61..c4d3b9cc7 100644 --- a/src/ufo/filters/DifferenceCheck.cc +++ b/src/ufo/filters/DifferenceCheck.cc @@ -60,8 +60,8 @@ void flagWhereDiffOut(const std::vector &refValues, std::vector &t // ----------------------------------------------------------------------------- DifferenceCheck::DifferenceCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/DifferenceCheck.h b/src/ufo/filters/DifferenceCheck.h index 8f82b7560..05ede00c4 100644 --- a/src/ufo/filters/DifferenceCheck.h +++ b/src/ufo/filters/DifferenceCheck.h @@ -69,8 +69,8 @@ class DifferenceCheck : public FilterBase, static const std::string classname() {return "ufo::DifferenceCheck";} DifferenceCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~DifferenceCheck(); private: diff --git a/src/ufo/filters/EnsembleStatistics.cc b/src/ufo/filters/EnsembleStatistics.cc index a7699204f..0d3ecc2e3 100644 --- a/src/ufo/filters/EnsembleStatistics.cc +++ b/src/ufo/filters/EnsembleStatistics.cc @@ -55,9 +55,9 @@ eckit::mpi::Comm & getOrCreateEnsembleComm(size_t geoCommSize, size_t timeCommSi // ----------------------------------------------------------------------------- EnsembleStatistics::EnsembleStatistics(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) -: ObsProcessorBase(obsdb, true /*deferToPost?*/, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) +: ObsProcessorBase(obsdb, true /*deferToPost?*/, std::move(flags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "EnsembleStatistics constructor start" << std::endl; diff --git a/src/ufo/filters/EnsembleStatistics.h b/src/ufo/filters/EnsembleStatistics.h index 7f5a4ae61..33f18caef 100644 --- a/src/ufo/filters/EnsembleStatistics.h +++ b/src/ufo/filters/EnsembleStatistics.h @@ -89,8 +89,8 @@ class EnsembleStatistics : public ObsProcessorBase, static const std::string classname() {return "ufo::EnsembleStatistics";} EnsembleStatistics(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~EnsembleStatistics() override; private: diff --git a/src/ufo/filters/FilterBase.cc b/src/ufo/filters/FilterBase.cc index 9124824d5..88237d39b 100644 --- a/src/ufo/filters/FilterBase.cc +++ b/src/ufo/filters/FilterBase.cc @@ -30,10 +30,10 @@ namespace ufo { FilterBase::FilterBase(ioda::ObsSpace & os, const FilterParametersBaseWithAbstractActions & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const VariableNameMap & nameMap) - : ObsProcessorBase(os, parameters.deferToPost, flags, obserr), + : ObsProcessorBase(os, parameters.deferToPost, std::move(flags), std::move(obserr)), filtervars_(), nameMap_(nameMap), whereParameters_(parameters.where), @@ -75,13 +75,13 @@ FilterBase::FilterBase(ioda::ObsSpace & os, // ----------------------------------------------------------------------------- FilterBase::FilterBase(ioda::ObsSpace & os, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const VariableNameMap & nameMap) : FilterBase(os, oops::validateAndDeserialize(config), - flags, - obserr, + std::move(flags), + std::move(obserr), nameMap) {} @@ -127,7 +127,7 @@ void FilterBase::doFilter() { // Take actions for (const std::unique_ptr &actionParameters : actionsParameters_) { FilterAction action(*actionParameters); - action.apply(vars, flagged, data_, this->qcFlag(), flags_, obserr_); + action.apply(vars, flagged, data_, this->qcFlag(), *flags_, *obserr_); } // Done diff --git a/src/ufo/filters/FilterBase.h b/src/ufo/filters/FilterBase.h index 5a1d6913f..cd41fef94 100644 --- a/src/ufo/filters/FilterBase.h +++ b/src/ufo/filters/FilterBase.h @@ -42,8 +42,8 @@ namespace ufo { /// In the former case, `Filter` should provide a constructor with the following signature: /// /// Filter(ioda::ObsSpace &os, const eckit::Configuration &conf, -/// ioda::ObsDataVector &flags, -/// ioda::ObsDataVector &obserr); +/// std::shared_ptr > qcflags, +/// std::shared_ptr > obserrors); /// /// In the latter case, a subclass of FilterParametersBaseWithAbstractAction holding the settings /// of the filter in question should be defined. (Usually it can in fact be a subclass of @@ -53,18 +53,18 @@ namespace ufo { /// of that subclass and provide a constructor with the following signature: /// /// Filter(ioda::ObsSpace &os, const Parameters_ ¶ms, -/// ioda::ObsDataVector &flags, -/// ioda::ObsDataVector &obserr); +/// std::shared_ptr > qcflags, +/// std::shared_ptr > obserrors); /// class FilterBase : public ObsProcessorBase { public: FilterBase(ioda::ObsSpace &, const FilterParametersBaseWithAbstractActions ¶meters, - ioda::ObsDataVector &, - ioda::ObsDataVector &, + std::shared_ptr >, + std::shared_ptr >, const VariableNameMap & nameMap = VariableNameMap(boost::none)); FilterBase(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &, + std::shared_ptr >, + std::shared_ptr >, const VariableNameMap & nameMap = VariableNameMap(boost::none)); ~FilterBase(); diff --git a/src/ufo/filters/FinalCheck.cc b/src/ufo/filters/FinalCheck.cc index 7be7246eb..16c9bbf73 100644 --- a/src/ufo/filters/FinalCheck.cc +++ b/src/ufo/filters/FinalCheck.cc @@ -22,10 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- FinalCheck::FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ &, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - - : ObsProcessorBase(obsdb, true /*deferToPost?*/, flags, obserr) + std::shared_ptr> qcflags, + std::shared_ptr> obserr) + : ObsProcessorBase(obsdb, true /*deferToPost?*/, std::move(qcflags), std::move(obserr)) { oops::Log::trace() << "FinalCheck constructor" << std::endl; } @@ -54,8 +53,8 @@ void FinalCheck::doFilter() { const float missing = util::missingValue(); for (size_t jv = 0; jv < obsdb_.obsvariables().size(); ++jv) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (flags_[jv][jobs] == QCflags::pass && obserr_[jv][jobs] == missing) { - flags_[jv][jobs] = QCflags::missing; + if ((*flags_)[jv][jobs] == QCflags::pass && (*obserr_)[jv][jobs] == missing) { + (*flags_)[jv][jobs] = QCflags::missing; } } } @@ -65,7 +64,7 @@ void FinalCheck::doFilter() { for (size_t jv = 0; jv < obsdb_.obsvariables().size(); ++jv) { if (!obsdb_.assimvariables().has(obsdb_.obsvariables()[jv])) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - flags_[jv][jobs] = QCflags::processed; + (*flags_)[jv][jobs] = QCflags::processed; } } } diff --git a/src/ufo/filters/FinalCheck.h b/src/ufo/filters/FinalCheck.h index 803a17967..09b6dbecd 100644 --- a/src/ufo/filters/FinalCheck.h +++ b/src/ufo/filters/FinalCheck.h @@ -42,8 +42,8 @@ class FinalCheck : public ObsProcessorBase, static const std::string classname() {return "ufo::FinalCheck";} FinalCheck(ioda::ObsSpace & obsdb, const Parameters_ & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> qcflags, + std::shared_ptr> obserr); ~FinalCheck() override; void doFilter() override; diff --git a/src/ufo/filters/Gaussian_Thinning.cc b/src/ufo/filters/Gaussian_Thinning.cc index 96d824974..244148b7c 100644 --- a/src/ufo/filters/Gaussian_Thinning.cc +++ b/src/ufo/filters/Gaussian_Thinning.cc @@ -43,8 +43,8 @@ namespace ufo { Gaussian_Thinning::Gaussian_Thinning(ioda::ObsSpace & obsdb, const GaussianThinningParameters & params, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, params, flags, obserr), options_(params) { oops::Log::trace() << "Gaussian_Thinning constructor" << std::endl; oops::Log::debug() << "Gaussian_Thinning: config = " << options_ << std::endl; @@ -66,7 +66,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, // returns what it has been passed without modification. const RecordHandler recordHandler(obsdb_, filtervars, - flags_, + *flags_, retainOnlyIfAllFilterVariablesAreValid); // If records are treated as single obs and a category variable is also used, @@ -87,7 +87,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, obsAccessor.getValidObservationIds(options_.recordsAreSingleObs ? recordHandler.changeApplyIfRecordsAreSingleObs(apply) : apply, - flags_, + *flags_, filtervars, !retainOnlyIfAllFilterVariablesAreValid); } @@ -231,7 +231,7 @@ void Gaussian_Thinning::applyFilter(const std::vector & apply, // Optionally reject all filter variables if any has failed QC and ob is invalid for thinning if (retainOnlyIfAllFilterVariablesAreValid) - obsAccessor.flagObservationsForAnyFilterVariableFailingQC(apply, flags_, filtervars, flagged); + obsAccessor.flagObservationsForAnyFilterVariableFailingQC(apply, *flags_, filtervars, flagged); oops::Log::trace() << "Gaussian_Thinning applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/Gaussian_Thinning.h b/src/ufo/filters/Gaussian_Thinning.h index c1abb66ec..f58d54596 100644 --- a/src/ufo/filters/Gaussian_Thinning.h +++ b/src/ufo/filters/Gaussian_Thinning.h @@ -62,8 +62,8 @@ class Gaussian_Thinning : public FilterBase, static const std::string classname() {return "ufo::Gaussian_Thinning";} Gaussian_Thinning(ioda::ObsSpace &obsdb, const GaussianThinningParameters ¶ms, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/GeoVaLsWriter.cc b/src/ufo/filters/GeoVaLsWriter.cc index 4a37a8271..d6c05ec05 100644 --- a/src/ufo/filters/GeoVaLsWriter.cc +++ b/src/ufo/filters/GeoVaLsWriter.cc @@ -18,7 +18,7 @@ namespace ufo { // ----------------------------------------------------------------------------- GeoVaLsWriter::GeoVaLsWriter(const ioda::ObsSpace &, const Parameters_ & params, - ioda::ObsDataVector &, ioda::ObsDataVector &) + ObsDataPtr_, ObsDataPtr_) : config_(params.config), novars_() { oops::Log::trace() << "GeoVaLsWriter constructor" << std::endl; } diff --git a/src/ufo/filters/GeoVaLsWriter.h b/src/ufo/filters/GeoVaLsWriter.h index 7adeedf76..9f3f06fd7 100644 --- a/src/ufo/filters/GeoVaLsWriter.h +++ b/src/ufo/filters/GeoVaLsWriter.h @@ -29,11 +29,13 @@ class ObsDiagnostics; // ----------------------------------------------------------------------------- class GeoVaLsWriter : public ObsFilterBase { + template using ObsDataPtr_ = std::shared_ptr >; + public: typedef GenericFilterParameters Parameters_; GeoVaLsWriter(const ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, ioda::ObsDataVector &); + ObsDataPtr_, ObsDataPtr_); ~GeoVaLsWriter() = default; void preProcess() override {} diff --git a/src/ufo/filters/HistoryCheck.cc b/src/ufo/filters/HistoryCheck.cc index 4f7928f00..7590dc13b 100644 --- a/src/ufo/filters/HistoryCheck.cc +++ b/src/ufo/filters/HistoryCheck.cc @@ -38,8 +38,8 @@ namespace ufo { HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::debug() << "HistoryCheck: config = " << options_ << "\n"; @@ -47,8 +47,8 @@ HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, HistoryCheck::HistoryCheck(ioda::ObsSpace &obsdb, const ufo::HistoryCheck::Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const eckit::LocalConfiguration &conf) : HistoryCheck(obsdb, parameters, flags, obserr) { @@ -93,8 +93,10 @@ void HistoryCheck::applyFilter(const std::vector & apply, widerObsSpace.put_db("ObsValue", "airTemperature", airTemperatures); } } // end of manual data entry section used for unit testing - ioda::ObsDataVector obserrWide(widerObsSpace, widerObsSpace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflagsWide(widerObsSpace, widerObsSpace.obsvariables()); + std::shared_ptr> obserrWide( + new ioda::ObsDataVector(widerObsSpace, widerObsSpace.obsvariables(), "ObsError")); + std::shared_ptr> qcflagsWide( + new ioda::ObsDataVector(widerObsSpace, widerObsSpace.obsvariables())); const oops::RequiredParameter &subtype = options_.surfaceObservationSubtype; const boost::optional &trackOptions = @@ -196,7 +198,7 @@ void HistoryCheck::applyFilter(const std::vector & apply, // counter" until the full identifier to-be-added is unique within the set. std::set wideFlaggedLocationIds; // qc flags are the same across all variables for these filters - const std::vector &wideFlags = qcflagsWide[0]; + const std::vector &wideFlags = (*qcflagsWide)[0]; for (size_t i = 0; i < wideFlags.size(); i++) { if (wideFlags[i] == QCflags::track) { obsIdentifierData obsLabel = { diff --git a/src/ufo/filters/HistoryCheck.h b/src/ufo/filters/HistoryCheck.h index 5f8742ad6..837c727c5 100644 --- a/src/ufo/filters/HistoryCheck.h +++ b/src/ufo/filters/HistoryCheck.h @@ -41,13 +41,13 @@ class HistoryCheck: public FilterBase, /// 2. Apply track check and stuck value check over wider window /// 3. Apply flags over wider window to observations in main window. HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); // This constructor is needed for unit testing HistoryCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr, const eckit::LocalConfiguration &conf); private: diff --git a/src/ufo/filters/ImpactHeightCheck.cc b/src/ufo/filters/ImpactHeightCheck.cc index ee62807ea..1399dde68 100644 --- a/src/ufo/filters/ImpactHeightCheck.cc +++ b/src/ufo/filters/ImpactHeightCheck.cc @@ -30,8 +30,8 @@ namespace ufo { ImpactHeightCheck::ImpactHeightCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ImpactHeightCheck constructor" << std::endl; @@ -250,7 +250,7 @@ void ImpactHeightCheck::applyFilter(const std::vector & apply, // Loop over all observations in the profile for (size_t jobs : obs_numbers) { // Check that this observation should be considered in this routine - if (apply[jobs] && flags_[iVar][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iVar][jobs] == QCflags::pass) { // Reject observation if it is below the minimum (either surface or sharp gradient) const float obsImpactHeight = impactParameter[iVar][jobs] - radiusCurvature[jobs]; if (parameters_.verboseOutput.value()) diff --git a/src/ufo/filters/ImpactHeightCheck.h b/src/ufo/filters/ImpactHeightCheck.h index 04499ed56..f965e1fb2 100644 --- a/src/ufo/filters/ImpactHeightCheck.h +++ b/src/ufo/filters/ImpactHeightCheck.h @@ -71,15 +71,15 @@ class ImpactHeightCheck : public FilterBase, static const std::string classname() {return "ufo::ImpactHeightCheck";} ImpactHeightCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ImpactHeightCheck(); private: void print(std::ostream &) const override; void applyFilter(const std::vector &, const Variables &, std::vector> &) const override; - int qcFlag() const override {return QCflags::superrefraction;} + int qcFlag() const override {return QCflags::domain;} Parameters_ parameters_; std::vector calcVerticalGradient(const std::vector &, const std::vector &) const; diff --git a/src/ufo/filters/MWCLWCheck.cc b/src/ufo/filters/MWCLWCheck.cc index f5d8a8966..3270e2814 100644 --- a/src/ufo/filters/MWCLWCheck.cc +++ b/src/ufo/filters/MWCLWCheck.cc @@ -26,8 +26,8 @@ namespace ufo { // ----------------------------------------------------------------------------- MWCLWCheck::MWCLWCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "MWCLWCheck constructor" << std::endl; oops::Log::debug() << "MWCLWCheck: config = " << parameters_ << std::endl; diff --git a/src/ufo/filters/MWCLWCheck.h b/src/ufo/filters/MWCLWCheck.h index ff1eb875d..2755c8436 100644 --- a/src/ufo/filters/MWCLWCheck.h +++ b/src/ufo/filters/MWCLWCheck.h @@ -60,8 +60,8 @@ class MWCLWCheck : public FilterBase, static const std::string classname() {return "ufo::MWCLWCheck";} MWCLWCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~MWCLWCheck(); private: diff --git a/src/ufo/filters/MetOfficeBuddyCheck.cc b/src/ufo/filters/MetOfficeBuddyCheck.cc index edfb13c25..69a4a0226 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.cc +++ b/src/ufo/filters/MetOfficeBuddyCheck.cc @@ -283,9 +283,9 @@ struct MetOfficeBuddyCheck::MetaData { }; MetOfficeBuddyCheck::MetOfficeBuddyCheck(ioda::ObsSpace& obsdb, const Parameters_& parameters, - ioda::ObsDataVector& flags, - ioda::ObsDataVector& obserr) - : FilterBase(obsdb, parameters, flags, obserr, + std::shared_ptr > flags, + std::shared_ptr > obserr) + : FilterBase(obsdb, parameters, std::move(flags), std::move(obserr), VariableNameMap(parameters.AliasFile.value())), options_(parameters) { oops::Log::trace() << "MetOfficeBuddyCheck constructor" << std::endl; @@ -999,7 +999,7 @@ std::vector MetOfficeBuddyCheck::getValidObservationIds( const std::vector & apply, const boost::optional & profileIndex) const { std::vector isValid = apply; - unselectRejectedLocations(isValid, filtervars_, flags_, + unselectRejectedLocations(isValid, filtervars_, *flags_, UnselectLocationIf::ALL_FILTER_VARIABLES_REJECTED); std::vector isValidAsInt(apply.begin(), apply.end()); diff --git a/src/ufo/filters/MetOfficeBuddyCheck.h b/src/ufo/filters/MetOfficeBuddyCheck.h index d39894b2d..2c711e31d 100644 --- a/src/ufo/filters/MetOfficeBuddyCheck.h +++ b/src/ufo/filters/MetOfficeBuddyCheck.h @@ -85,8 +85,8 @@ class MetOfficeBuddyCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficeBuddyCheck";} MetOfficeBuddyCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: struct MetaData; diff --git a/src/ufo/filters/MetOfficeDuplicateCheck.cc b/src/ufo/filters/MetOfficeDuplicateCheck.cc index 607fa74c5..5981b4ba5 100644 --- a/src/ufo/filters/MetOfficeDuplicateCheck.cc +++ b/src/ufo/filters/MetOfficeDuplicateCheck.cc @@ -18,8 +18,8 @@ namespace ufo { MetOfficeDuplicateCheck::MetOfficeDuplicateCheck (ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "MetOfficeDuplicateCheck constructor" << std::endl; oops::Log::debug() << "MetOfficeDuplicateCheck: config = " << options_ << std::endl; diff --git a/src/ufo/filters/MetOfficeDuplicateCheck.h b/src/ufo/filters/MetOfficeDuplicateCheck.h index eacc64111..2336956c9 100644 --- a/src/ufo/filters/MetOfficeDuplicateCheck.h +++ b/src/ufo/filters/MetOfficeDuplicateCheck.h @@ -107,8 +107,8 @@ class MetOfficeDuplicateCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficeDuplicateCheck";} MetOfficeDuplicateCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~MetOfficeDuplicateCheck() override; diff --git a/src/ufo/filters/MetOfficePressureConsistencyCheck.cc b/src/ufo/filters/MetOfficePressureConsistencyCheck.cc index 8900e6306..7695f1fe7 100644 --- a/src/ufo/filters/MetOfficePressureConsistencyCheck.cc +++ b/src/ufo/filters/MetOfficePressureConsistencyCheck.cc @@ -18,8 +18,8 @@ namespace ufo { MetOfficePressureConsistencyCheck::MetOfficePressureConsistencyCheck (ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "MetOfficePressureConsistencyCheck constructor" << std::endl; oops::Log::debug() << "MetOfficePressureConsistencyCheck: config = " << options_ << std::endl; @@ -37,7 +37,7 @@ void MetOfficePressureConsistencyCheck::applyFilter(const std::vector & ap const ObsAccessor obsAccessor = createObsAccessor(); const std::vector validObsIds = - obsAccessor.getValidObservationIds(apply, flags_, filtervars); + obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/MetOfficePressureConsistencyCheck.h b/src/ufo/filters/MetOfficePressureConsistencyCheck.h index 5e1e34a99..c64e5fbf6 100644 --- a/src/ufo/filters/MetOfficePressureConsistencyCheck.h +++ b/src/ufo/filters/MetOfficePressureConsistencyCheck.h @@ -56,8 +56,8 @@ class MetOfficePressureConsistencyCheck : public FilterBase, static const std::string classname() {return "ufo::MetOfficePressureConsistencyCheck";} MetOfficePressureConsistencyCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~MetOfficePressureConsistencyCheck() override; diff --git a/src/ufo/filters/ModelBestFitPressure.cc b/src/ufo/filters/ModelBestFitPressure.cc index 6238855c3..7dc64059a 100644 --- a/src/ufo/filters/ModelBestFitPressure.cc +++ b/src/ufo/filters/ModelBestFitPressure.cc @@ -25,8 +25,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ModelBestFitPressure::ModelBestFitPressure(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ModelBestFitPressure.h b/src/ufo/filters/ModelBestFitPressure.h index 77fb4635b..a15fc0d7c 100644 --- a/src/ufo/filters/ModelBestFitPressure.h +++ b/src/ufo/filters/ModelBestFitPressure.h @@ -93,8 +93,8 @@ class ModelBestFitPressure : public FilterBase, static const std::string classname() {return "ufo::ModelBestFitPressure";} ModelBestFitPressure(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ModelBestFitPressure(); private: diff --git a/src/ufo/filters/ModelObThreshold.cc b/src/ufo/filters/ModelObThreshold.cc index 1b54e73bd..f83f109bf 100644 --- a/src/ufo/filters/ModelObThreshold.cc +++ b/src/ufo/filters/ModelObThreshold.cc @@ -25,8 +25,8 @@ constexpr util::NamedEnumerator // ----------------------------------------------------------------------------- ModelObThreshold::ModelObThreshold(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ModelObThreshold constructor" << std::endl; diff --git a/src/ufo/filters/ModelObThreshold.h b/src/ufo/filters/ModelObThreshold.h index c6e39b398..1dc5c9c55 100644 --- a/src/ufo/filters/ModelObThreshold.h +++ b/src/ufo/filters/ModelObThreshold.h @@ -91,8 +91,8 @@ class ModelObThreshold : public FilterBase, static const std::string classname() {return "ufo::ModelObThreshold";} ModelObThreshold(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ModelObThreshold(); private: diff --git a/src/ufo/filters/ObsAccessor.cc b/src/ufo/filters/ObsAccessor.cc index 5c5bff459..803592c72 100644 --- a/src/ufo/filters/ObsAccessor.cc +++ b/src/ufo/filters/ObsAccessor.cc @@ -416,9 +416,6 @@ RecursiveSplitter ObsAccessor::splitObservationsIntoIndependentGroups( case GroupBy::SINGLE_OBS: groupObservationsByCategoryVariable(validObsIds, splitter); break; - case GroupBy::RECORD_ID: - groupObservationsByRecordNumber(validObsIds, splitter); - break; } return splitter; } diff --git a/src/ufo/filters/ObsBoundsCheck.cc b/src/ufo/filters/ObsBoundsCheck.cc index e566bc164..c41b8fd9a 100644 --- a/src/ufo/filters/ObsBoundsCheck.cc +++ b/src/ufo/filters/ObsBoundsCheck.cc @@ -67,8 +67,8 @@ void flagWhereOutOfBounds(const std::vector & apply, // ----------------------------------------------------------------------------- ObsBoundsCheck::ObsBoundsCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsBoundsCheck constructor" << std::endl; @@ -143,7 +143,7 @@ void ObsBoundsCheck::applyFilter(const std::vector & apply, std::vector testAtLocations = apply; if (onlyTestGoodFilterVarsForFlagAllFilterVars) { for (size_t iloc=0; iloc < testAtLocations.size(); iloc++) - if (flags_[filtervarslist[ifiltervar]][iloc] != QCflags::pass) { + if ((*flags_)[filtervarslist[ifiltervar]][iloc] != QCflags::pass) { testAtLocations[iloc] = false; } } diff --git a/src/ufo/filters/ObsBoundsCheck.h b/src/ufo/filters/ObsBoundsCheck.h index e6b335a69..720d1ab00 100644 --- a/src/ufo/filters/ObsBoundsCheck.h +++ b/src/ufo/filters/ObsBoundsCheck.h @@ -100,8 +100,8 @@ class ObsBoundsCheck : public FilterBase, static const std::string classname() {return "ufo::ObsBoundsCheck";} ObsBoundsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsBoundsCheck(); private: diff --git a/src/ufo/filters/ObsDerivativeCheck.cc b/src/ufo/filters/ObsDerivativeCheck.cc index fd377df23..8ca306a3d 100644 --- a/src/ufo/filters/ObsDerivativeCheck.cc +++ b/src/ufo/filters/ObsDerivativeCheck.cc @@ -28,8 +28,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDerivativeCheck::ObsDerivativeCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ObsDerivativeCheck.h b/src/ufo/filters/ObsDerivativeCheck.h index 2cb191c33..94596ef2b 100644 --- a/src/ufo/filters/ObsDerivativeCheck.h +++ b/src/ufo/filters/ObsDerivativeCheck.h @@ -83,8 +83,8 @@ class ObsDerivativeCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDerivativeCheck";} ObsDerivativeCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDerivativeCheck(); private: diff --git a/src/ufo/filters/ObsDiagnosticsWriter.cc b/src/ufo/filters/ObsDiagnosticsWriter.cc index 78ea048d4..f073b22fb 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.cc +++ b/src/ufo/filters/ObsDiagnosticsWriter.cc @@ -16,8 +16,8 @@ namespace ufo { ObsDiagnosticsWriter::ObsDiagnosticsWriter( ioda::ObsSpace &, const Parameters_ & params, - ioda::ObsDataVector &, - ioda::ObsDataVector &) + std::shared_ptr >, + std::shared_ptr >) : params_(params), extradiagvars_() { oops::Log::trace() << "ObsDiagnosticsWriter constructor" << std::endl; diff --git a/src/ufo/filters/ObsDiagnosticsWriter.h b/src/ufo/filters/ObsDiagnosticsWriter.h index f73add41f..f79477045 100644 --- a/src/ufo/filters/ObsDiagnosticsWriter.h +++ b/src/ufo/filters/ObsDiagnosticsWriter.h @@ -43,8 +43,8 @@ class ObsDiagnosticsWriter : public ObsFilterBase { public: typedef ObsDiagnosticsWriterParameters Parameters_; ObsDiagnosticsWriter(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDiagnosticsWriter() {} void preProcess() override {} diff --git a/src/ufo/filters/ObsDomainCheck.cc b/src/ufo/filters/ObsDomainCheck.cc index 6a9e3262b..c702d98bd 100644 --- a/src/ufo/filters/ObsDomainCheck.cc +++ b/src/ufo/filters/ObsDomainCheck.cc @@ -17,8 +17,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDomainCheck::ObsDomainCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsDomainCheck constructor" << std::endl; diff --git a/src/ufo/filters/ObsDomainCheck.h b/src/ufo/filters/ObsDomainCheck.h index 40aad8a0e..20bea2523 100644 --- a/src/ufo/filters/ObsDomainCheck.h +++ b/src/ufo/filters/ObsDomainCheck.h @@ -48,8 +48,8 @@ class ObsDomainCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDomainCheck";} ObsDomainCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDomainCheck(); private: diff --git a/src/ufo/filters/ObsDomainErrCheck.cc b/src/ufo/filters/ObsDomainErrCheck.cc index feb3304dc..fcc2685a1 100644 --- a/src/ufo/filters/ObsDomainErrCheck.cc +++ b/src/ufo/filters/ObsDomainErrCheck.cc @@ -24,13 +24,14 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsDomainErrCheck::ObsDomainErrCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsDomainErrCheck constructor" << std::endl; oops::Log::debug() << "ObsDomainErrCheck: config = " << parameters_ << std::endl; + ASSERT(obserr); } // ----------------------------------------------------------------------------- @@ -69,13 +70,13 @@ void ObsDomainErrCheck::applyFilter(const std::vector & inside, } else { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - ASSERT(obserr_[iv][jobs] != util::missingValue>()); + ASSERT((*obserr_)[iv][jobs] != util::missingValue>()); ASSERT(obs[jv][jobs] != util::missingValue()); - float bound = 2.5 * obserr_[iv][jobs]; - float obserrinc = parameter * std::max((values[jobs]-9.0), 0.0) * obserr_[iv][jobs]; - obserrinc = std::max(obserr_[iv][jobs], bound); - obserr_[iv][jobs] = sqrt(pow(obserr_[iv][jobs], 2) + pow(obserrinc, 2)); + float bound = 2.5 * (*obserr_)[iv][jobs]; + float obserrinc = parameter * std::max((values[jobs]-9.0), 0.0) * (*obserr_)[iv][jobs]; + obserrinc = std::max((*obserr_)[iv][jobs], bound); + (*obserr_)[iv][jobs] = sqrt(pow((*obserr_)[iv][jobs], 2) + pow(obserrinc, 2)); ++count; } } diff --git a/src/ufo/filters/ObsDomainErrCheck.h b/src/ufo/filters/ObsDomainErrCheck.h index f5311202f..4fa9a4a90 100644 --- a/src/ufo/filters/ObsDomainErrCheck.h +++ b/src/ufo/filters/ObsDomainErrCheck.h @@ -45,8 +45,8 @@ class ObsDomainErrCheck : public FilterBase, static const std::string classname() {return "ufo::ObsDomainErrCheck";} ObsDomainErrCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsDomainErrCheck(); private: diff --git a/src/ufo/filters/ObsPolygonCheck.cc b/src/ufo/filters/ObsPolygonCheck.cc index d8c4bf4c4..ebb11845b 100644 --- a/src/ufo/filters/ObsPolygonCheck.cc +++ b/src/ufo/filters/ObsPolygonCheck.cc @@ -30,8 +30,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsPolygonCheck::ObsPolygonCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsPolygonCheck constructor" << std::endl; diff --git a/src/ufo/filters/ObsPolygonCheck.h b/src/ufo/filters/ObsPolygonCheck.h index 34ef91fd9..153b9d6f5 100644 --- a/src/ufo/filters/ObsPolygonCheck.h +++ b/src/ufo/filters/ObsPolygonCheck.h @@ -70,8 +70,8 @@ class ObsPolygonCheck : public FilterBase, static const std::string classname() {return "ufo::ObsPolygonCheck";} ObsPolygonCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsPolygonCheck(); private: diff --git a/src/ufo/filters/ObsProcessorBase.cc b/src/ufo/filters/ObsProcessorBase.cc index a6d77d795..61e2d9216 100644 --- a/src/ufo/filters/ObsProcessorBase.cc +++ b/src/ufo/filters/ObsProcessorBase.cc @@ -27,16 +27,18 @@ namespace ufo { // ----------------------------------------------------------------------------- ObsProcessorBase::ObsProcessorBase(ioda::ObsSpace & os, bool deferToPost, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : obsdb_(os), flags_(flags), obserr_(obserr), data_(obsdb_), prior_(false), post_(false), deferToPost_(deferToPost) { oops::Log::trace() << "ObsProcessorBase constructor" << std::endl; - data_.associate(flags_, "QCflagsData"); - data_.associate(obserr_, "ObsErrorData"); + ASSERT(flags); + ASSERT(obserr); + data_.associate(*flags_, "QCflagsData"); + data_.associate(*obserr_, "ObsErrorData"); } // ----------------------------------------------------------------------------- diff --git a/src/ufo/filters/ObsProcessorBase.h b/src/ufo/filters/ObsProcessorBase.h index 52dd80c85..9c32b3c5c 100644 --- a/src/ufo/filters/ObsProcessorBase.h +++ b/src/ufo/filters/ObsProcessorBase.h @@ -35,8 +35,8 @@ namespace ufo { class ObsProcessorBase : public ObsFilterBase { public: ObsProcessorBase(ioda::ObsSpace &, bool deferToPost, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsProcessorBase(); void preProcess() override; @@ -54,8 +54,8 @@ class ObsProcessorBase : public ObsFilterBase { protected: ioda::ObsSpace & obsdb_; - ioda::ObsDataVector & flags_; - ioda::ObsDataVector & obserr_; + std::shared_ptr> flags_; + std::shared_ptr> obserr_; ufo::Variables allvars_; ObsFilterData data_; bool prior_; diff --git a/src/ufo/filters/ObsRefractivityGradientCheck.cc b/src/ufo/filters/ObsRefractivityGradientCheck.cc index 210a265f4..7e9b576dc 100644 --- a/src/ufo/filters/ObsRefractivityGradientCheck.cc +++ b/src/ufo/filters/ObsRefractivityGradientCheck.cc @@ -26,8 +26,8 @@ namespace ufo { ObsRefractivityGradientCheck::ObsRefractivityGradientCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ObsRefractivityGradientCheck: " @@ -91,7 +91,7 @@ void ObsRefractivityGradientCheck::applyFilter( for (size_t jv = 0; jv < filtervars.nvars(); ++jv) { const size_t iv = observed.find(filtervars.variable(jv).variable()); for (size_t isort : idx) { - if (apply[obs_numbers[isort]] && flags_[iv][obs_numbers[isort]] == QCflags::pass) { + if ( apply[obs_numbers[isort]] && (*flags_)[iv][obs_numbers[isort]] == QCflags::pass ) { if ((gradient[isort] <= parameters_.gradientMin.value() || gradient[isort] >= parameters_.gradientMax.value() || gradient[isort] == 0 || @@ -100,7 +100,7 @@ void ObsRefractivityGradientCheck::applyFilter( height[obs_numbers[isort]] < parameters_.maxCheckHeight.value()) { // reject all observations below isort for (size_t jobs = isort; jobs < idx.size(); ++jobs) { - if (apply[obs_numbers[jobs]] && flags_[iv][obs_numbers[jobs]] == QCflags::pass) { + if (apply[obs_numbers[jobs]] && (*flags_)[iv][obs_numbers[jobs]] == QCflags::pass) { flagged[jv][obs_numbers[jobs]] = true; } } diff --git a/src/ufo/filters/ObsRefractivityGradientCheck.h b/src/ufo/filters/ObsRefractivityGradientCheck.h index d19201a43..bd4eaf1d5 100644 --- a/src/ufo/filters/ObsRefractivityGradientCheck.h +++ b/src/ufo/filters/ObsRefractivityGradientCheck.h @@ -59,8 +59,8 @@ class ObsRefractivityGradientCheck : public FilterBase, static const std::string classname() {return "ufo::ObsRefractivityGradientCheck";} ObsRefractivityGradientCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ObsRefractivityGradientCheck(); private: diff --git a/src/ufo/filters/OceanVerticalStabilityCheck.cc b/src/ufo/filters/OceanVerticalStabilityCheck.cc index ed05e8961..d05fbe41c 100644 --- a/src/ufo/filters/OceanVerticalStabilityCheck.cc +++ b/src/ufo/filters/OceanVerticalStabilityCheck.cc @@ -22,8 +22,8 @@ namespace ufo { OceanVerticalStabilityCheck::OceanVerticalStabilityCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "OceanVerticalStabilityCheck constructor" << std::endl; @@ -90,7 +90,7 @@ void OceanVerticalStabilityCheck::applyFilter(const std::vector & apply, // if any filter variable fails QC): const std::vector obs_indices = obsAccessor.getValidObsIdsInProfile(iProfile, apply, - flags_, + *flags_, filtervars, false); diff --git a/src/ufo/filters/OceanVerticalStabilityCheck.h b/src/ufo/filters/OceanVerticalStabilityCheck.h index 66fef178b..c886a0783 100644 --- a/src/ufo/filters/OceanVerticalStabilityCheck.h +++ b/src/ufo/filters/OceanVerticalStabilityCheck.h @@ -103,8 +103,8 @@ class OceanVerticalStabilityCheck : public FilterBase, static const std::string classname() {return "ufo::OceanVerticalStabilityCheck";} OceanVerticalStabilityCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~OceanVerticalStabilityCheck(); private: diff --git a/src/ufo/filters/ParameterSubstitution.cc b/src/ufo/filters/ParameterSubstitution.cc index 85ef36d8c..e2670d4ba 100644 --- a/src/ufo/filters/ParameterSubstitution.cc +++ b/src/ufo/filters/ParameterSubstitution.cc @@ -21,8 +21,8 @@ namespace ufo { ParameterSubstitution::ParameterSubstitution(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : parameters_(parameters) { oops::Log::trace() << "ParameterSubstitution constructor" << std::endl; diff --git a/src/ufo/filters/ParameterSubstitution.h b/src/ufo/filters/ParameterSubstitution.h index 5d456933a..ebf68165e 100644 --- a/src/ufo/filters/ParameterSubstitution.h +++ b/src/ufo/filters/ParameterSubstitution.h @@ -110,8 +110,8 @@ class ParameterSubstitution : public ObsFilterBase, ParameterSubstitution(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ParameterSubstitution() {} diff --git a/src/ufo/filters/PerformAction.cc b/src/ufo/filters/PerformAction.cc index 0aed1a4af..d27c657e1 100644 --- a/src/ufo/filters/PerformAction.cc +++ b/src/ufo/filters/PerformAction.cc @@ -50,8 +50,8 @@ std::vector> PerformActionParameters // ----------------------------------------------------------------------------- PerformAction::PerformAction(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "PerformAction constructor" << std::endl; diff --git a/src/ufo/filters/PerformAction.h b/src/ufo/filters/PerformAction.h index ba33289e8..b19a6984e 100644 --- a/src/ufo/filters/PerformAction.h +++ b/src/ufo/filters/PerformAction.h @@ -65,8 +65,8 @@ class PerformAction : public FilterBase, static const std::string classname() {return "ufo::PerformAction";} PerformAction(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/PoissonDiskThinning.cc b/src/ufo/filters/PoissonDiskThinning.cc index 98d7cb7dd..33ba79fe0 100644 --- a/src/ufo/filters/PoissonDiskThinning.cc +++ b/src/ufo/filters/PoissonDiskThinning.cc @@ -166,8 +166,8 @@ struct PoissonDiskThinning::ObsData PoissonDiskThinning::PoissonDiskThinning(ioda::ObsSpace & obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "PoissonDiskThinning constructor" << std::endl; oops::Log::debug() << "PoissonDiskThinning: config = " << options_ << std::endl; @@ -413,7 +413,7 @@ std::vector PoissonDiskThinning::getValidObservationIds( const std::vector & apply, const Variables & filtervars, const ObsAccessor &obsAccessor) const { - std::vector validObsIds = obsAccessor.getValidObservationIds(apply, flags_, filtervars); + std::vector validObsIds = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); if (!options_.shuffle) { // The user wants to process observations in fixed (non-random) order. Ensure the filter @@ -717,14 +717,14 @@ void PoissonDiskThinning::thinCategoryMedian(const ObsData &obsData, template std::array PoissonDiskThinning::getObservationPosition( size_t obsId, const ObsData &obsData) const { - std::array position = {util::missingValue()}; + std::array position; unsigned int dim = 0; if (obsData.latitudes && obsData.longitudes) { if (obsData.minLatitudeSpacings && obsData.minLongitudeSpacings) { - position.at(dim++) = (*obsData.latitudes)[obsId]; - position.at(dim++) = (*obsData.longitudes)[obsId]; + position[dim++] = (*obsData.latitudes)[obsId]; + position[dim++] = (*obsData.longitudes)[obsId]; } else { const float deg2rad = static_cast(M_PI / 180.0); const float earthRadius = static_cast(Constants::mean_earth_rad_m / 1000.0); @@ -736,20 +736,20 @@ std::array PoissonDiskThinning::getObservationPosition( const float sinLon = std::sin(lon); const float cosLon = std::cos(lon); - position.at(dim++) = earthRadius * cosLat * cosLon; - position.at(dim++) = earthRadius * cosLat * sinLon; - position.at(dim++) = earthRadius * sinLat; + position[dim++] = earthRadius * cosLat * cosLon; + position[dim++] = earthRadius * cosLat * sinLon; + position[dim++] = earthRadius * sinLat; } } if (obsData.pressures) { - position.at(dim++) = (*obsData.pressures)[obsId]; + position[dim++] = (*obsData.pressures)[obsId]; } if (obsData.times) { // We use the centre of the assimilation window as the reference time when converting // datetimes to floats. - position.at(dim++) = ((*obsData.times)[obsId] - obsdb_.timeWindow().midpoint()).toSeconds(); + position[dim++] = ((*obsData.times)[obsId] - obsdb_.timeWindow().midpoint()).toSeconds(); } return position; @@ -759,7 +759,7 @@ template std::array PoissonDiskThinning::getExclusionVolumeSemiAxes( size_t obsId, const ObsData &obsData) const { - std::array semiAxes = {util::missingValue()}; + std::array semiAxes; const int priority = obsData.priorities == boost::none ? 0 : (*obsData.priorities)[obsId]; @@ -773,20 +773,20 @@ std::array PoissonDiskThinning::getExclusionVolumeSemiAxes( const float minEuclideanDistance = earthDiameter * std::sin(minGeodesicDistance * invEarthDiameter); - semiAxes.at(dim++) = minEuclideanDistance; - semiAxes.at(dim++) = minEuclideanDistance; - semiAxes.at(dim++) = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; + semiAxes[dim++] = minEuclideanDistance; } else if (obsData.minLatitudeSpacings && obsData.minLongitudeSpacings) { - semiAxes.at(dim++) = obsData.minLatitudeSpacings->at(priority); - semiAxes.at(dim++) = obsData.minLongitudeSpacings->at(priority); + semiAxes[dim++] = obsData.minLatitudeSpacings->at(priority); + semiAxes[dim++] = obsData.minLongitudeSpacings->at(priority); } if (obsData.minVerticalSpacings) { - semiAxes.at(dim++) = obsData.minVerticalSpacings->at(priority); + semiAxes[dim++] = obsData.minVerticalSpacings->at(priority); } if (obsData.minTimeSpacings) { - semiAxes.at(dim++) = obsData.minTimeSpacings->at(priority).toSeconds(); + semiAxes[dim++] = obsData.minTimeSpacings->at(priority).toSeconds(); } return semiAxes; diff --git a/src/ufo/filters/PoissonDiskThinning.h b/src/ufo/filters/PoissonDiskThinning.h index 10a333bdd..0fbd92ae1 100644 --- a/src/ufo/filters/PoissonDiskThinning.h +++ b/src/ufo/filters/PoissonDiskThinning.h @@ -52,8 +52,8 @@ class PoissonDiskThinning : public FilterBase, static const std::string classname() {return "ufo::PoissonDiskThinning";} PoissonDiskThinning(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~PoissonDiskThinning() override; diff --git a/src/ufo/filters/PreQC.cc b/src/ufo/filters/PreQC.cc index c5891a40f..67840003a 100644 --- a/src/ufo/filters/PreQC.cc +++ b/src/ufo/filters/PreQC.cc @@ -17,8 +17,8 @@ namespace ufo { PreQC::PreQC(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "PreQC constructor" << std::endl; diff --git a/src/ufo/filters/PreQC.h b/src/ufo/filters/PreQC.h index c69de8d77..27a116a02 100644 --- a/src/ufo/filters/PreQC.h +++ b/src/ufo/filters/PreQC.h @@ -45,8 +45,8 @@ class PreQC : public FilterBase, private util::ObjectCounter { static const std::string classname() {return "ufo::PreQC";} PreQC(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/PrintFilterData.cc b/src/ufo/filters/PrintFilterData.cc index 5575faf49..d2fa4b202 100644 --- a/src/ufo/filters/PrintFilterData.cc +++ b/src/ufo/filters/PrintFilterData.cc @@ -22,9 +22,9 @@ namespace ufo { PrintFilterData::PrintFilterData(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(flags), std::move(obserr)), parameters_(parameters), os_(parameters.outputToTest ? oops::Log::test() : oops::Log::info()) { diff --git a/src/ufo/filters/PrintFilterData.h b/src/ufo/filters/PrintFilterData.h index 1308b6425..82ed81090 100644 --- a/src/ufo/filters/PrintFilterData.h +++ b/src/ufo/filters/PrintFilterData.h @@ -149,8 +149,8 @@ class PrintFilterData : public ObsProcessorBase, static const std::string classname() {return "ufo::PrintFilterData";} PrintFilterData(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: // variables Parameters_ parameters_; diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc index f543b0c7e..769b3753a 100644 --- a/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.cc @@ -23,8 +23,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ProbabilityGrossErrorWholeReport::ProbabilityGrossErrorWholeReport(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ProbabilityGrossErrorWholeReport.h b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h index efb0a4b6b..403b1d6d4 100644 --- a/src/ufo/filters/ProbabilityGrossErrorWholeReport.h +++ b/src/ufo/filters/ProbabilityGrossErrorWholeReport.h @@ -103,8 +103,8 @@ class ProbabilityGrossErrorWholeReport : public FilterBase, static const std::string classname() {return "ufo::ProbabilityGrossErrorWholeReport";} ProbabilityGrossErrorWholeReport(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr>, + std::shared_ptr>); ~ProbabilityGrossErrorWholeReport(); private: diff --git a/src/ufo/filters/ProcessAMVQI.cc b/src/ufo/filters/ProcessAMVQI.cc index 7b1c0b0ff..2081f8f5c 100644 --- a/src/ufo/filters/ProcessAMVQI.cc +++ b/src/ufo/filters/ProcessAMVQI.cc @@ -24,8 +24,8 @@ namespace ufo { // ----------------------------------------------------------------------------- ProcessAMVQI::ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : ObsProcessorBase(obsdb, false /*deferToPost?*/, flags, obserr), parameters_(parameters) { diff --git a/src/ufo/filters/ProcessAMVQI.h b/src/ufo/filters/ProcessAMVQI.h index b42277f3d..337c381b6 100644 --- a/src/ufo/filters/ProcessAMVQI.h +++ b/src/ufo/filters/ProcessAMVQI.h @@ -62,8 +62,8 @@ class ProcessAMVQI : public ObsProcessorBase, static const std::string classname() {return "ufo::ProcessAMVQI";} ProcessAMVQI(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr> flags, + std::shared_ptr> obserr); ~ProcessAMVQI() override; private: diff --git a/src/ufo/filters/ProfileAverageObsToModLevels.cc b/src/ufo/filters/ProfileAverageObsToModLevels.cc index a241426a7..9d64faa10 100644 --- a/src/ufo/filters/ProfileAverageObsToModLevels.cc +++ b/src/ufo/filters/ProfileAverageObsToModLevels.cc @@ -30,8 +30,8 @@ namespace ufo { ProfileAverageObsToModLevels::ProfileAverageObsToModLevels( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileAverageObsToModLevels constructor" << std::endl; @@ -64,7 +64,7 @@ void ProfileAverageObsToModLevels::applyFilter(const std::vector & apply, const Variables & filtervars, std::vector> & flagged) const { oops::Log::trace() << "ProfileAverageObsToModLevels applyFilter start" << std::endl; - oops::Log::debug() << "ProfileAverageObsToModLevels obserr: " << obserr_ << std::endl; + oops::Log::debug() << "ProfileAverageObsToModLevels obserr: " << *obserr_ << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); @@ -121,14 +121,14 @@ void ProfileAverageObsToModLevels::applyFilter(const std::vector & apply, ufo::setModLevelFlags(locsExt, link_obs_and_mod_inds, flagged[jv], - (flags_)[iv]); + (*flags_)[iv]); // Compute increments at each obs level, take their average for each model level, // and add the average to the HofX at that model level: ufo::averageObsToModLevels(locsOriginal, locsExt, link_obs_and_mod_inds, apply, - flags_[iv], + (*flags_)[iv], hofx, obs[jv]); } // profile jprof diff --git a/src/ufo/filters/ProfileAverageObsToModLevels.h b/src/ufo/filters/ProfileAverageObsToModLevels.h index 059c0008c..cdf50287e 100644 --- a/src/ufo/filters/ProfileAverageObsToModLevels.h +++ b/src/ufo/filters/ProfileAverageObsToModLevels.h @@ -83,8 +83,8 @@ class ProfileAverageObsToModLevels : public FilterBase, static const std::string classname() {return "ufo::ProfileAverageObsToModLevels";} ProfileAverageObsToModLevels(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileAverageObsToModLevels(); private: diff --git a/src/ufo/filters/ProfileBackgroundCheck.cc b/src/ufo/filters/ProfileBackgroundCheck.cc index 29f9679ad..9fb45a6c2 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.cc +++ b/src/ufo/filters/ProfileBackgroundCheck.cc @@ -46,8 +46,8 @@ namespace ufo { ProfileBackgroundCheck::ProfileBackgroundCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileBackgroundCheck constructor" << std::endl; @@ -71,7 +71,7 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, oops::Log::trace() << "ProfileBackgroundCheck applyFilter start" << std::endl; const oops::ObsVariables observed = obsdb_.obsvariables(); - oops::Log::debug() << "ProfileBackgroundCheck obserr: " << obserr_; + oops::Log::debug() << "ProfileBackgroundCheck obserr: " << *obserr_; ioda::ObsDataVector obs(obsdb_, filtervars.toOopsObsVariables(), "ObsValue"); ioda::ObsDataVector bias(obsdb_, filtervars.toOopsObsVariables(), "ObsBias", false); @@ -114,11 +114,11 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, // Determine whether this profile uses threshold or abs_threshold based // on the first value in the profile. for (size_t jobs : obs_numbers) { - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { // style note: use missingValue because obserr_ is declared in another file // and its underlying type could in principle change without it being obvious here. - ASSERT(obserr_[iv][jobs] != - util::missingValue>()); + ASSERT((*obserr_)[iv][jobs] != + util::missingValue>()); ASSERT(obs[jv][jobs] != util::missingValue()); ASSERT(hofx[jobs] != util::missingValue()); @@ -139,11 +139,11 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, const float yy = obs[jv][jobs] - bias[jv][jobs]; // Accumulate distance from background - if (obserr_[iv][jobs] > 0) { + if ((*obserr_)[iv][jobs] > 0) { total_nobs++; if (using_rel_thresh) { // Normalise by observation error - total_diff += std::pow((hofx[jobs] - yy) / obserr_[iv][jobs], 2); + total_diff += std::pow((hofx[jobs] - yy) / (*obserr_)[iv][jobs], 2); } else { // Use the absolute differences total_diff += std::pow(hofx[jobs] - yy, 2); @@ -161,7 +161,7 @@ void ProfileBackgroundCheck::applyFilter(const std::vector & apply, // Second run through the observations to apply the threshold to reject profiles for (size_t jobs : obs_numbers) { - if (apply[jobs] && flags_[iv][jobs] == QCflags::pass) { + if (apply[jobs] && (*flags_)[iv][jobs] == QCflags::pass) { // If rejecting this profile, then flag all elements if (total_diff > static_cast(profile_threshold)) flagged[jv][jobs] = true; } diff --git a/src/ufo/filters/ProfileBackgroundCheck.h b/src/ufo/filters/ProfileBackgroundCheck.h index c5c21059e..af7597cce 100644 --- a/src/ufo/filters/ProfileBackgroundCheck.h +++ b/src/ufo/filters/ProfileBackgroundCheck.h @@ -59,8 +59,8 @@ class ProfileBackgroundCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileBackgroundCheck";} ProfileBackgroundCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileBackgroundCheck(); private: diff --git a/src/ufo/filters/ProfileFewObsCheck.cc b/src/ufo/filters/ProfileFewObsCheck.cc index 88b3aea85..20d45634b 100644 --- a/src/ufo/filters/ProfileFewObsCheck.cc +++ b/src/ufo/filters/ProfileFewObsCheck.cc @@ -31,8 +31,8 @@ namespace ufo { ProfileFewObsCheck::ProfileFewObsCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileFewObsCheck constructor" << std::endl; @@ -97,7 +97,7 @@ void ProfileFewObsCheck::applyFilter(const std::vector & apply, for (size_t jobs : obs_numbers) { if (apply[jobs]) { numTotal++; - if (flags_[jVar][jobs] == QCflags::pass) + if ((*flags_)[jVar][jobs] == QCflags::pass) numValid++; } } @@ -122,7 +122,7 @@ void ProfileFewObsCheck::applyFilter(const std::vector & apply, const size_t iFilterVar = iVar * nChans + iChan; const size_t jVar = variableIndicesMap[iChan]; for (size_t jobs : obs_numbers) - if (apply[jobs] && flags_[jVar][jobs] == QCflags::pass) + if (apply[jobs] && (*flags_)[jVar][jobs] == QCflags::pass) flagged[iFilterVar][jobs] = true; } } diff --git a/src/ufo/filters/ProfileFewObsCheck.h b/src/ufo/filters/ProfileFewObsCheck.h index 95d9c36d4..a61f18d55 100644 --- a/src/ufo/filters/ProfileFewObsCheck.h +++ b/src/ufo/filters/ProfileFewObsCheck.h @@ -52,8 +52,8 @@ class ProfileFewObsCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileFewObsCheck";} ProfileFewObsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileFewObsCheck(); private: diff --git a/src/ufo/filters/ProfileMaxDifferenceCheck.cc b/src/ufo/filters/ProfileMaxDifferenceCheck.cc index bc5175b83..2a4f370bb 100644 --- a/src/ufo/filters/ProfileMaxDifferenceCheck.cc +++ b/src/ufo/filters/ProfileMaxDifferenceCheck.cc @@ -26,8 +26,8 @@ namespace ufo { ProfileMaxDifferenceCheck::ProfileMaxDifferenceCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileMaxDifferenceCheck: " diff --git a/src/ufo/filters/ProfileMaxDifferenceCheck.h b/src/ufo/filters/ProfileMaxDifferenceCheck.h index 9629fffb1..01d1a1ba8 100644 --- a/src/ufo/filters/ProfileMaxDifferenceCheck.h +++ b/src/ufo/filters/ProfileMaxDifferenceCheck.h @@ -61,8 +61,8 @@ class ProfileMaxDifferenceCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileMaxDifferenceCheck";} ProfileMaxDifferenceCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileMaxDifferenceCheck(); private: diff --git a/src/ufo/filters/ProfileUnFlagObsCheck.cc b/src/ufo/filters/ProfileUnFlagObsCheck.cc index 8dbdb09d7..e8f9ae3cf 100644 --- a/src/ufo/filters/ProfileUnFlagObsCheck.cc +++ b/src/ufo/filters/ProfileUnFlagObsCheck.cc @@ -77,7 +77,7 @@ class AbsoluteToleranceCreator { static std::unique_ptr create_unique(std::string name, const ProfileUnFlagObsCheck::Parameters_& parameters, const ioda::ObsSpace& obsdb) { std::unique_ptr absoluteTolerance = - getFactory()[name]->create_unique(parameters, obsdb); + std::move(getFactory()[name]->create_unique(parameters, obsdb)); return absoluteTolerance; } static std::map &getFactory() { @@ -124,8 +124,8 @@ static AbsoluteTolerancePiecewiseCreator absoluteTolerancePiecewiseCreator; ProfileUnFlagObsCheck::ProfileUnFlagObsCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "ProfileUnFlagObsCheck constructor" << std::endl; @@ -201,21 +201,21 @@ void ProfileUnFlagObsCheck::applyFilter(const std::vector & apply, for (size_t jObsProfile = 0; jObsProfile < obsNumbers.size(); ++jObsProfile) { size_t jObsGlobal = obsNumbers[jObsProfile]; const float absTol = (*absoluteToleranceCalculate)(jObsGlobal); - if ((flags_[jVar][jObsGlobal] != QCflags::pass) && - (flags_[jVar][jObsGlobal] != QCflags::passive) && - (flags_[jVar][jObsGlobal] != QCflags::missing)) { + if (((*flags_)[jVar][jObsGlobal] != QCflags::pass) && + ((*flags_)[jVar][jObsGlobal] != QCflags::passive) && + ((*flags_)[jVar][jObsGlobal] != QCflags::missing)) { bool makeValid = false; if (jObsProfile == 0) { makeValid = true; } else { const size_t jObsMinus = obsNumbers[jObsProfile-1]; - makeValid = flags_[jVar][jObsMinus] == QCflags::pass && + makeValid = (*flags_)[jVar][jObsMinus] == QCflags::pass && (std::abs(variableData[jObsMinus] - variableData[jObsGlobal]) < absTol); } if (jObsProfile < obsNumbers.size()-1) { const size_t jObsPlus = obsNumbers[jObsProfile+1]; makeValid = makeValid && - (flags_[jVar][jObsPlus] == QCflags::pass) && + ((*flags_)[jVar][jObsPlus] == QCflags::pass) && (std::abs(variableData[jObsPlus] - variableData[jObsGlobal]) < absTol); } if (makeValid) { diff --git a/src/ufo/filters/ProfileUnFlagObsCheck.h b/src/ufo/filters/ProfileUnFlagObsCheck.h index 6d7934844..f5062167b 100644 --- a/src/ufo/filters/ProfileUnFlagObsCheck.h +++ b/src/ufo/filters/ProfileUnFlagObsCheck.h @@ -91,8 +91,8 @@ class ProfileUnFlagObsCheck : public FilterBase, static const std::string classname() {return "ufo::ProfileUnFlagObsCheck";} ProfileUnFlagObsCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ProfileUnFlagObsCheck(); private: diff --git a/src/ufo/filters/QCmanager.cc b/src/ufo/filters/QCmanager.cc index c8a37b0b7..f75d3eac4 100644 --- a/src/ufo/filters/QCmanager.cc +++ b/src/ufo/filters/QCmanager.cc @@ -47,33 +47,35 @@ void updateQCFlags(const std::vector *obsValues, std::vector& qcflag // ----------------------------------------------------------------------------- QCmanager::QCmanager(ioda::ObsSpace & obsdb, const Parameters_ & /*parameters*/, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & /*obserr*/) - : obsdb_(obsdb), nogeovals_(), nodiags_(), flags_(flags) + std::shared_ptr > qcflags, + std::shared_ptr > /*obserr*/) + : obsdb_(obsdb), nogeovals_(), nodiags_(), flags_(qcflags) { oops::Log::trace() << "QCmanager constructor start" << std::endl; + ASSERT(qcflags); + const oops::ObsVariables &allObservedVars = obsdb.obsvariables(); const oops::ObsVariables &initialObservedVars = obsdb.initial_obsvariables(); const oops::ObsVariables &derivedObservedVars = obsdb.derived_obsvariables(); ASSERT(allObservedVars.size() == initialObservedVars.size() + derivedObservedVars.size()); - ASSERT(flags_.nvars() == allObservedVars.size()); - ASSERT(flags_.nlocs() == obsdb_.nlocs()); + ASSERT(flags_->nvars() == allObservedVars.size()); + ASSERT(flags_->nlocs() == obsdb_.nlocs()); const ioda::ObsDataVector obs(obsdb, initialObservedVars, "ObsValue"); // Iterate over initial observed variables for (size_t jv = 0; jv < initialObservedVars.size(); ++jv) { const ioda::ObsDataRow ¤tObsValues = obs[jv]; - ioda::ObsDataRow ¤tQCFlags = flags[obs.varnames()[jv]]; + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[obs.varnames()[jv]]; updateQCFlags(¤tObsValues, currentQCFlags); } // Iterate over derived variables and if they don't exist yet, set their QC flags to // 'missing'. for (size_t jv = 0; jv < derivedObservedVars.size(); ++jv) { - ioda::ObsDataRow ¤tQCFlags = flags[derivedObservedVars[jv]]; + ioda::ObsDataRow ¤tQCFlags = (*qcflags)[derivedObservedVars[jv]]; if (!obsdb.has("ObsValue", derivedObservedVars[jv])) { updateQCFlags(nullptr, currentQCFlags); } else { @@ -101,8 +103,8 @@ void QCmanager::postFilter(const GeoVaLs &, /*geovals*/ for (size_t jv = 0; jv < allObservedVars.size(); ++jv) { for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { const size_t iobs = allObservedVars.size() * jobs + jv; - if (flags_[allObservedVars[jv]][jobs] == 0 && hofx[iobs] == missing) { - flags_[allObservedVars[jv]][jobs] = QCflags::Hfailed; + if ((*flags_)[allObservedVars[jv]][jobs] == 0 && hofx[iobs] == missing) { + (*flags_)[allObservedVars[jv]][jobs] = QCflags::Hfailed; } } } @@ -163,8 +165,9 @@ void QCmanager::print(std::ostream & os) const { const std::string varName = allObservedVars[jvar]; std::unique_ptr>> accumulator = obsdb_.distribution()->createAccumulator(cases.size()); + for (size_t jobs = 0; jobs < nlocs; ++jobs) { - const int actualFlag = flags_[jvar][jobs]; + const int actualFlag = (*flags_)[jvar][jobs]; for (size_t jcase = 0; jcase < cases.size(); ++jcase) if (actualFlag == cases[jcase].first) accumulator->addTerm(jobs, jcase, 1); @@ -223,7 +226,7 @@ void QCmanager::print(std::ostream & os) const { const std::vector countDiags = accumulatorDiags->computeResult(); if (obsdb_.comm().rank() == 0) { - const std::string info = "QC " + flags_.obstype() + " " + varName + ": "; + const std::string info = "QC " + flags_->obstype() + " " + varName + ": "; // Normal cases for (size_t i = numSpecialCases; i < counts.size(); ++i) diff --git a/src/ufo/filters/QCmanager.h b/src/ufo/filters/QCmanager.h index 4e7407afa..3bac7f509 100644 --- a/src/ufo/filters/QCmanager.h +++ b/src/ufo/filters/QCmanager.h @@ -41,8 +41,8 @@ class QCmanager : public ObsFilterBase { typedef QCmanagerParameters Parameters_; QCmanager(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~QCmanager(); void preProcess() override {} @@ -62,7 +62,7 @@ class QCmanager : public ObsFilterBase { ioda::ObsSpace & obsdb_; const oops::Variables nogeovals_; const oops::ObsVariables nodiags_; - ioda::ObsDataVector & flags_; + std::shared_ptr> flags_; }; } // namespace ufo diff --git a/src/ufo/filters/SatName.cc b/src/ufo/filters/SatName.cc index d5707d9a2..fff1550de 100644 --- a/src/ufo/filters/SatName.cc +++ b/src/ufo/filters/SatName.cc @@ -63,8 +63,8 @@ std::string get_sat_name(int SatID, const std::vector &Sat // ----------------------------------------------------------------------------- SatName::SatName(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SatName constructor" << std::endl; diff --git a/src/ufo/filters/SatName.h b/src/ufo/filters/SatName.h index cc3ab42b9..a1ea0e2cf 100644 --- a/src/ufo/filters/SatName.h +++ b/src/ufo/filters/SatName.h @@ -78,8 +78,8 @@ class SatName : public FilterBase, typedef SatNameParameters Parameters_; static const std::string classname() {return "ufo::SatName";} SatName(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SatName(); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/SatwindInversionCorrection.cc b/src/ufo/filters/SatwindInversionCorrection.cc index 0407a573a..cf0a09a44 100644 --- a/src/ufo/filters/SatwindInversionCorrection.cc +++ b/src/ufo/filters/SatwindInversionCorrection.cc @@ -22,9 +22,9 @@ namespace ufo { // ----------------------------------------------------------------------------- SatwindInversionCorrection::SatwindInversionCorrection(ioda::ObsSpace & obsdb, - const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const Parameters_ & parameters, + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SatwindInversion constructor" << std::endl; diff --git a/src/ufo/filters/SatwindInversionCorrection.h b/src/ufo/filters/SatwindInversionCorrection.h index e82646fe3..6bf69933f 100644 --- a/src/ufo/filters/SatwindInversionCorrection.h +++ b/src/ufo/filters/SatwindInversionCorrection.h @@ -73,8 +73,8 @@ class SatwindInversionCorrection : public FilterBase, static const std::string classname() {return "ufo::SatwindInversionCorrection";} SatwindInversionCorrection(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SatwindInversionCorrection(); private: diff --git a/src/ufo/filters/SpikeAndStepCheck.cc b/src/ufo/filters/SpikeAndStepCheck.cc index 98e0cac65..340dcec0f 100644 --- a/src/ufo/filters/SpikeAndStepCheck.cc +++ b/src/ufo/filters/SpikeAndStepCheck.cc @@ -31,8 +31,8 @@ namespace ufo { SpikeAndStepCheck::SpikeAndStepCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SpikeAndStepCheck constructor" << std::endl; @@ -144,7 +144,7 @@ void SpikeAndStepCheck::applyFilter(const std::vector & apply, // if any filter variable fails QC): const std::vector obs_indices = obsAccessor.getValidObsIdsInProfile(iProfile, apply, - flags_, + *flags_, filtervars, false); // Struct of y, x, dy, dx, dy/dx: diff --git a/src/ufo/filters/SpikeAndStepCheck.h b/src/ufo/filters/SpikeAndStepCheck.h index da5c12fcf..a5911afd4 100644 --- a/src/ufo/filters/SpikeAndStepCheck.h +++ b/src/ufo/filters/SpikeAndStepCheck.h @@ -161,8 +161,8 @@ class SpikeAndStepCheck : public FilterBase, static const std::string classname() {return "ufo::SpikeAndStepCheck";} SpikeAndStepCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~SpikeAndStepCheck(); /// \brief Vectors of dependent and independent variables, their differences, and gradient diff --git a/src/ufo/filters/StuckCheck.cc b/src/ufo/filters/StuckCheck.cc index e714adfb2..6d98553a2 100644 --- a/src/ufo/filters/StuckCheck.cc +++ b/src/ufo/filters/StuckCheck.cc @@ -28,8 +28,8 @@ namespace ufo { StuckCheck::StuckCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "StuckCheck constructor" << std::endl; diff --git a/src/ufo/filters/StuckCheck.h b/src/ufo/filters/StuckCheck.h index 9602fe6cf..7d3ed1a8f 100644 --- a/src/ufo/filters/StuckCheck.h +++ b/src/ufo/filters/StuckCheck.h @@ -48,8 +48,8 @@ class StuckCheck: public FilterBase, static const std::string classname() {return "ufo::StuckCheck";} StuckCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~StuckCheck() override; diff --git a/src/ufo/filters/SuperOb.cc b/src/ufo/filters/SuperOb.cc index 498c78802..13ee3856b 100644 --- a/src/ufo/filters/SuperOb.cc +++ b/src/ufo/filters/SuperOb.cc @@ -16,8 +16,8 @@ namespace ufo { SuperOb::SuperOb(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "SuperOb constructor start" << std::endl; @@ -37,7 +37,7 @@ SuperOb::SuperOb(ioda::ObsSpace & obsdb, std::unique_ptr superOb = SuperObFactory::create(params.superObName, - data_, apply, filtervars_, flags, flagged); + data_, apply, filtervars_, *flags, flagged); if (superOb->requireHofX()) { allvars_ += Variables(filtervars_, "HofX"); @@ -60,7 +60,7 @@ void SuperOb::applyFilter(const std::vector & apply, std::unique_ptr superOb = SuperObFactory::create(params.superObName, - data_, apply, filtervars, flags_, flagged); + data_, apply, filtervars, *flags_, flagged); superOb->runAlgorithm(); oops::Log::trace() << "SuperOb applyFilter complete" << std::endl; diff --git a/src/ufo/filters/SuperOb.h b/src/ufo/filters/SuperOb.h index 1dadb1634..94f8c56d4 100644 --- a/src/ufo/filters/SuperOb.h +++ b/src/ufo/filters/SuperOb.h @@ -70,8 +70,8 @@ class SuperOb : public FilterBase, static const std::string classname() {return "ufo::SuperOb";} SuperOb(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperOb(); private: diff --git a/src/ufo/filters/SuperRefractionCheckImpactParameter.cc b/src/ufo/filters/SuperRefractionCheckImpactParameter.cc index 68d7489da..ef76194f0 100644 --- a/src/ufo/filters/SuperRefractionCheckImpactParameter.cc +++ b/src/ufo/filters/SuperRefractionCheckImpactParameter.cc @@ -31,8 +31,8 @@ namespace ufo { SuperRefractionCheckImpactParameter::SuperRefractionCheckImpactParameter( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SuperRefractionCheckImpactParameter constructor" << std::endl; @@ -164,7 +164,7 @@ void SuperRefractionCheckImpactParameter::applyFilter( } for (size_t iVar = 0; iVar < filtervars.nvars(); ++iVar) { for (size_t iobs : obs_numbers) { - if (apply[iobs] && flags_[iVar][iobs] == QCflags::pass && + if (apply[iobs] && (*flags_)[iVar][iobs] == QCflags::pass && impactParameterObs[0][iobs] <= impactParameterModel[kLevel] && kLevel > 0) { flagged[iVar][iobs] = true; @@ -175,7 +175,7 @@ void SuperRefractionCheckImpactParameter::applyFilter( } else { // profileCheck = false for (size_t iVar = 0; iVar < filtervars.nvars(); ++iVar) { for (size_t iobs : obs_numbers) { - if (apply[iobs] && flags_[iVar][iobs] == QCflags::pass) { + if (apply[iobs] && (*flags_)[iVar][iobs] == QCflags::pass) { std::vector refracProfile; std::vector heightProfile; // count the number of valid levels of model profiles for iobs diff --git a/src/ufo/filters/SuperRefractionCheckImpactParameter.h b/src/ufo/filters/SuperRefractionCheckImpactParameter.h index 1438cb174..055837748 100644 --- a/src/ufo/filters/SuperRefractionCheckImpactParameter.h +++ b/src/ufo/filters/SuperRefractionCheckImpactParameter.h @@ -53,8 +53,8 @@ class SuperRefractionCheckImpactParameter : public FilterBase, static const std::string classname() {return "ufo::SuperRefractionCheckImpactParameter";} SuperRefractionCheckImpactParameter(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperRefractionCheckImpactParameter(); private: diff --git a/src/ufo/filters/SuperRefractionCheckNBAM.cc b/src/ufo/filters/SuperRefractionCheckNBAM.cc index 6af211021..cdf14f7ba 100644 --- a/src/ufo/filters/SuperRefractionCheckNBAM.cc +++ b/src/ufo/filters/SuperRefractionCheckNBAM.cc @@ -30,8 +30,8 @@ namespace ufo { SuperRefractionCheckNBAM::SuperRefractionCheckNBAM( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "SuperRefractionCheckNBAM constructor" << std::endl; diff --git a/src/ufo/filters/SuperRefractionCheckNBAM.h b/src/ufo/filters/SuperRefractionCheckNBAM.h index 7a9c651e6..84c520f98 100644 --- a/src/ufo/filters/SuperRefractionCheckNBAM.h +++ b/src/ufo/filters/SuperRefractionCheckNBAM.h @@ -65,8 +65,8 @@ class SuperRefractionCheckNBAM : public FilterBase, static std::string classname() {return std::string("ufo::SuperRefractionCheckNBAM");} SuperRefractionCheckNBAM(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~SuperRefractionCheckNBAM(); private: diff --git a/src/ufo/filters/TemporalThinning.cc b/src/ufo/filters/TemporalThinning.cc index 7aad4f2dc..a8404091e 100644 --- a/src/ufo/filters/TemporalThinning.cc +++ b/src/ufo/filters/TemporalThinning.cc @@ -304,8 +304,8 @@ typename TemporalThinner::ForwardValidObsIndexIterator TemporalThinner::findNear } // namespace TemporalThinning::TemporalThinning(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TemporalThinning constructor" << std::endl; @@ -328,7 +328,7 @@ void TemporalThinning::applyFilter(const std::vector & apply, // returns what it has been passed without modification. // The value of `retainOnlyIfAllFilterVariablesAreValid` is set to `false` // because that is the default value used in the `ObsAccessor` class. - const RecordHandler recordHandler(obsdb_, filtervars, flags_, false); + const RecordHandler recordHandler(obsdb_, filtervars, *flags_, false); // If records are treated as single obs and a category variable is also used, // ensure that there are no records with multiple values of the category variable. @@ -380,7 +380,7 @@ std::vector TemporalThinning::identifyThinnedObservations( const Variables & filtervars, const ObsAccessor &obsAccessor) const { const std::vector validObsIds - = obsAccessor.getValidObservationIds(apply, flags_, filtervars); + = obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/TemporalThinning.h b/src/ufo/filters/TemporalThinning.h index f0ff435d2..85ea5dc59 100644 --- a/src/ufo/filters/TemporalThinning.h +++ b/src/ufo/filters/TemporalThinning.h @@ -45,8 +45,8 @@ class TemporalThinning : public FilterBase, static const std::string classname() {return "ufo::TemporalThinning";} TemporalThinning(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TemporalThinning() override; diff --git a/src/ufo/filters/Thinning.cc b/src/ufo/filters/Thinning.cc index 0c241d02f..6aa31d05a 100644 --- a/src/ufo/filters/Thinning.cc +++ b/src/ufo/filters/Thinning.cc @@ -21,8 +21,8 @@ namespace ufo { // ----------------------------------------------------------------------------- Thinning::Thinning(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "Thinning constructor" << std::endl; diff --git a/src/ufo/filters/Thinning.h b/src/ufo/filters/Thinning.h index ab3f1610f..f66ea915a 100644 --- a/src/ufo/filters/Thinning.h +++ b/src/ufo/filters/Thinning.h @@ -58,8 +58,8 @@ class Thinning : public FilterBase, static const std::string classname() {return "ufo::Thinning";} Thinning(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~Thinning(); private: diff --git a/src/ufo/filters/TrackCheck.cc b/src/ufo/filters/TrackCheck.cc index 6798ebae9..e812e85d4 100644 --- a/src/ufo/filters/TrackCheck.cc +++ b/src/ufo/filters/TrackCheck.cc @@ -100,8 +100,8 @@ float TrackCheck::TrackObservation::getFailedChecksFraction() { TrackCheck::TrackCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TrackCheck constructor" << std::endl; @@ -125,7 +125,7 @@ void TrackCheck::applyFilter(const std::vector & apply, const std::vector validObsIds = options_.ignoreExistingQCFlags ? obsAccessor.getValidObservationIds(apply) : - obsAccessor.getValidObservationIds(apply, flags_, filtervars); + obsAccessor.getValidObservationIds(apply, *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); TrackCheckUtils::sortTracksChronologically(validObsIds, obsAccessor, splitter); diff --git a/src/ufo/filters/TrackCheck.h b/src/ufo/filters/TrackCheck.h index 1bc4e86b9..86b191069 100644 --- a/src/ufo/filters/TrackCheck.h +++ b/src/ufo/filters/TrackCheck.h @@ -82,8 +82,8 @@ class TrackCheck : public FilterBase, static const std::string classname() { return "ufo::TrackCheck"; } TrackCheck(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TrackCheck() override; diff --git a/src/ufo/filters/TrackCheckShip.cc b/src/ufo/filters/TrackCheckShip.cc index 0eda488a6..7880a9793 100644 --- a/src/ufo/filters/TrackCheckShip.cc +++ b/src/ufo/filters/TrackCheckShip.cc @@ -35,8 +35,8 @@ namespace ufo { TrackCheckShip::TrackCheckShip(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), options_(parameters) { oops::Log::trace() << "TrackCheckShip constructor" << std::endl; @@ -176,12 +176,12 @@ void TrackCheckShip::applyFilter(const std::vector & apply, // returns what it has been passed without modification. // The value of `retainOnlyIfAllFilterVariablesAreValid` is set to `false` // because that is the default value used in the `ObsAccessor` class. - const RecordHandler recordHandler(obsdb_, filtervars, flags_, false); + const RecordHandler recordHandler(obsdb_, filtervars, *flags_, false); const std::vector validObsIds = obsAccessor.getValidObservationIds(options_.recordsAreSingleObs ? recordHandler.changeApplyIfRecordsAreSingleObs(apply) : apply, - flags_, + *flags_, filtervars); RecursiveSplitter splitter = obsAccessor.splitObservationsIntoIndependentGroups(validObsIds); diff --git a/src/ufo/filters/TrackCheckShip.h b/src/ufo/filters/TrackCheckShip.h index f76255000..a1602c04f 100644 --- a/src/ufo/filters/TrackCheckShip.h +++ b/src/ufo/filters/TrackCheckShip.h @@ -63,8 +63,8 @@ class TrackCheckShip: public FilterBase, static const std::string classname() {return "ufo::TrackCheckShip";} TrackCheckShip(ioda::ObsSpace &obsdb, const Parameters_ ¶meters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); ~TrackCheckShip() override; diff --git a/src/ufo/filters/VariableAssignment.cc b/src/ufo/filters/VariableAssignment.cc index d64583fa5..7ff4c3ae7 100644 --- a/src/ufo/filters/VariableAssignment.cc +++ b/src/ufo/filters/VariableAssignment.cc @@ -492,9 +492,9 @@ void AssignmentParameters::deserialize(util::CompositePath &path, VariableAssignment::VariableAssignment(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : ObsProcessorBase(obsdb, parameters.deferToPost, flags, obserr), + std::shared_ptr > flags, + std::shared_ptr > obserr) + : ObsProcessorBase(obsdb, parameters.deferToPost, std::move(flags), std::move(obserr)), parameters_(parameters) { oops::Log::trace() << "VariableAssignment constructor" << std::endl; @@ -521,7 +521,7 @@ void VariableAssignment::doFilter() { for (const AssignmentParameters &assignment : parameters_.assignments.value()) { const ufo::Variable variable = getVariable(assignment); const ioda::ObsDtype dtype = getDataType(assignment.type, variable, obsdb_); - assignToVariable(variable, dtype, assignment, apply, data_, obsdb_, flags_); + assignToVariable(variable, dtype, assignment, apply, data_, obsdb_, *flags_); } oops::Log::trace() << "VariableAssignment doFilter complete" << std::endl; diff --git a/src/ufo/filters/VariableAssignment.h b/src/ufo/filters/VariableAssignment.h index 757465d0a..37236fb9e 100644 --- a/src/ufo/filters/VariableAssignment.h +++ b/src/ufo/filters/VariableAssignment.h @@ -163,8 +163,8 @@ class VariableAssignment : public ObsProcessorBase, static const std::string classname() {return "ufo::VariableAssignment";} VariableAssignment(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + std::shared_ptr > flags, + std::shared_ptr > obserr); private: void print(std::ostream &) const override; diff --git a/src/ufo/filters/VariableTransforms.cc b/src/ufo/filters/VariableTransforms.cc index 78153dbd1..e615464d7 100644 --- a/src/ufo/filters/VariableTransforms.cc +++ b/src/ufo/filters/VariableTransforms.cc @@ -31,8 +31,8 @@ namespace ufo { VariableTransforms::VariableTransforms( ioda::ObsSpace& obsdb, const eckit::Configuration& config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr> flags, + std::shared_ptr> obserr) : FilterBase(obsdb, config, flags, obserr), parameters_() { oops::Log::trace() << "VariableTransforms constructor" << std::endl; diff --git a/src/ufo/filters/VariableTransforms.h b/src/ufo/filters/VariableTransforms.h index f16ead255..f76e28918 100644 --- a/src/ufo/filters/VariableTransforms.h +++ b/src/ufo/filters/VariableTransforms.h @@ -49,8 +49,8 @@ class VariableTransforms : public FilterBase, // This Constructor function initializes an instance of the // filter based on options specified in the YAML configuration file. VariableTransforms(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr>, + std::shared_ptr>); // Destructor ~VariableTransforms(); diff --git a/src/ufo/filters/Variables.cc b/src/ufo/filters/Variables.cc index 2e0c8c3aa..547073a04 100644 --- a/src/ufo/filters/Variables.cc +++ b/src/ufo/filters/Variables.cc @@ -53,7 +53,7 @@ Variables::Variables(const oops::ObsVariables & oopsvars) ASSERT(oopsvars.size() % channels.size() == 0); std::vector varnames(nvars); for (size_t joopsvar = 0, jvar = 0; - joopsvar < oopsvars.size() && jvar < nvars; ++joopsvar) { + joopsvar < oopsvars.size(), jvar < nvars; ++joopsvar) { const size_t pos = oopsvars[joopsvar].find_last_of('_'); const std::string varname = oopsvars[joopsvar].substr(0, pos); if (std::find(varnames.begin(), varnames.end(), varname) == varnames.end()) { diff --git a/src/ufo/filters/actions/SetFlag.cc b/src/ufo/filters/actions/SetFlag.cc index ff8af7cdf..088f028b7 100644 --- a/src/ufo/filters/actions/SetFlag.cc +++ b/src/ufo/filters/actions/SetFlag.cc @@ -63,7 +63,7 @@ void SetFlag::apply(const Variables &vars, typedef bool (*Predicate)(int); // Pointer to a function taking a QC flag and returning true if observations with this QC flag // should be ignored and false otherwise - Predicate isIgnored = nullptr; + Predicate isIgnored; switch (parameters_.ignore.value()) { case IgnoredObservations::NONE: isIgnored = &returnFalse; diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc index ae4b16258..a771be761 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.cc @@ -29,8 +29,8 @@ namespace ufo { GNSSROOneDVarCheck::GNSSROOneDVarCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { oops::Log::trace() << "GNSSROOneDVarCheck constructor" << std::endl; @@ -91,8 +91,8 @@ void GNSSROOneDVarCheck::applyFilter(const std::vector & apply, } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc - obserr_.save("FortranERR"); // Pass latest errors to 1DVar + flags_->save("FortranQC"); // temporary measure as per ROobserror qc + obserr_->save("FortranERR"); // Pass latest errors to 1DVar // Pass it all to fortran ufo_gnssroonedvarcheck_apply_f90(key_, @@ -100,7 +100,7 @@ void GNSSROOneDVarCheck::applyFilter(const std::vector & apply, apply_char.size(), apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "GNSSROOneDVarCheck applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h index 587d3889c..ec110567a 100644 --- a/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h +++ b/src/ufo/filters/gnssroonedvarcheck/GNSSROOneDVarCheck.h @@ -57,8 +57,8 @@ class GNSSROOneDVarCheck : public FilterBase, GNSSROOneDVarCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~GNSSROOneDVarCheck(); private: diff --git a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 index 9d4319f39..9c278d1e4 100644 --- a/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 +++ b/src/ufo/filters/gnssroonedvarcheck/ufo_gnssroonedvarcheck_mod.f90 @@ -148,9 +148,9 @@ subroutine ufo_gnssroonedvarcheck_create(self, obsspace, bmatrix_filename, & end do write(message, '(A,L1)') 'no super check = ', noSuperCheck call fckit_log % debug(message) - write(message, '(A,F16.6)') 'Dry refractivity constant = ', dryRefractivityConstant + write(message, '(A,L1)') 'Dry refractivity constant = ', dryRefractivityConstant call fckit_log % debug(message) - write(message, '(A,F16.6)') 'Wet refractivity constant = ', wetRefractivityConstant + write(message, '(A,L1)') 'Wet refractivity constant = ', wetRefractivityConstant call fckit_log % debug(message) end subroutine ufo_gnssroonedvarcheck_create @@ -238,11 +238,11 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) ! ! Diagnostics to push back to the obs-space ! + integer, allocatable :: indices(:) ! The indices of the diagnostic elements to be updated integer, allocatable :: niter(:) ! Number of iterations required to converge real(kind_real), allocatable :: initial_cost(:) ! Initial cost-function value real(kind_real), allocatable :: final_cost(:) ! Final cost-function value real(kind_real), allocatable :: dfs_list(:) ! Degrees of freedom for signal - integer :: ind ! Get the obs-space information nobs = obsspace_get_nlocs(self % obsdb) @@ -426,13 +426,15 @@ subroutine ufo_gnssroonedvarcheck_apply(self, geovals, apply) end if ! Save the diagnostic information + allocate(indices(1:nobs_profile)) do ipoint = 0, nobs_profile-1 - ind = 1 + ((index_vals(start_point+ipoint) - 1) / nlevels) - niter(ind) = Ob % niter - initial_cost(ind) = O_Bdiff - final_cost(ind) = Ob % jcost - dfs_list(ind) = DFS + indices(ipoint+1) = 1 + ((index_vals(start_point+ipoint) - 1) / nlevels) end do + niter(indices) = Ob % niter + initial_cost(indices) = O_Bdiff + final_cost(indices) = Ob % jcost + dfs_list(indices) = DFS + deallocate(indices) call deallocate_singleob(Ob) end do diff --git a/src/ufo/filters/obsfunctions/CircularDifference.cc b/src/ufo/filters/obsfunctions/CircularDifference.cc deleted file mode 100644 index ac02ed1d1..000000000 --- a/src/ufo/filters/obsfunctions/CircularDifference.cc +++ /dev/null @@ -1,118 +0,0 @@ -/* - * (C) Crown copyright 2026, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "ufo/filters/obsfunctions/CircularDifference.h" - -#include - -#include "ioda/ObsDataVector.h" -#include "oops/util/Logger.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/ObsFilterData.h" - -namespace ufo { - -// Register this ObsFunction with the factory -static ObsFunctionMaker - makerCircularDifference_("CircularDifference"); - -// ----------------------------------------------------------------------------- - -CircularDifference::CircularDifference(const eckit::LocalConfiguration &conf) - : invars_() { - oops::Log::trace() << "CircularDifference constructor" << std::endl; - // Validate and deserialize parameters - options_.validateAndDeserialize(conf); - - // Add required variables - invars_ += options_.variableStart.value(); - invars_ += options_.variableEnd.value(); - - oops::Log::debug() << "CircularDifference: configured with period=" - << options_.circularPeriod.value() - << ", signed=" << options_.computeSignedDifference.value() - << std::endl; -} - -// ----------------------------------------------------------------------------- - -CircularDifference::~CircularDifference() { - oops::Log::trace() << "CircularDifference destructor" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void CircularDifference::compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const { - oops::Log::trace() << "CircularDifference compute start" << std::endl; - const size_t nlocs = in.nlocs(); - const float missing = util::missingValue(); - const float period = options_.circularPeriod.value(); - const bool signedDiff = options_.computeSignedDifference.value(); - - // Validate period - if (period <= 0.0f) { - throw eckit::BadValue("Circular period must be positive", Here()); - } - - // Get input variable data - const Variable &varStart = options_.variableStart.value(); - const Variable &varEnd = options_.variableEnd.value(); - - ioda::ObsDataVector startData(in.obsspace(), - varStart.toOopsObsVariables()); - in.get(varStart, startData); - - ioda::ObsDataVector endData(in.obsspace(), - varEnd.toOopsObsVariables()); - in.get(varEnd, endData); - - // Compute circular difference for each variable and location - for (size_t ivar = 0; ivar < out.nvars(); ++ivar) { - for (size_t iloc = 0; iloc < nlocs; ++iloc) { - const float start = startData[ivar][iloc]; - const float end = endData[ivar][iloc]; - - // Check for missing values - if (start == missing || end == missing) { - out[ivar][iloc] = missing; - continue; - } - - // Compute raw difference - float diff = end - start; - - // Normalize difference to [-period/2, period/2) - // This gives the signed circular difference - // Uses the formula: ((b - a + half) % period) - half - const float half = period / 2.0f; - diff = std::fmod(diff + half, period); - if (diff < 0.0f) { - diff += period; // Ensure non-negative result from fmod (consistent with Python's behavior) - } - diff -= half; - - // Apply signed/unsigned option - if (signedDiff) { - out[ivar][iloc] = diff; - } else { - out[ivar][iloc] = std::abs(diff); - } - } - } - oops::Log::trace() << "CircularDifference compute complete" << std::endl; -} - -// ----------------------------------------------------------------------------- - -const ufo::Variables &CircularDifference::requiredVariables() const { - return invars_; -} - -// ----------------------------------------------------------------------------- - -} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/CircularDifference.h b/src/ufo/filters/obsfunctions/CircularDifference.h deleted file mode 100644 index d255fdbd1..000000000 --- a/src/ufo/filters/obsfunctions/CircularDifference.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - * (C) Crown copyright 2026, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ -#define UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ - -#include - -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" - -#include "ufo/filters/obsfunctions/ObsFunctionBase.h" -#include "ufo/filters/Variable.h" -#include "ufo/filters/Variables.h" -#include "ufo/utils/parameters/ParameterTraitsVariable.h" - -namespace ufo { - -/// \brief Parameters for the CircularDifference ObsFunction -class CircularDifferenceParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(CircularDifferenceParameters, Parameters) - - public: - /// Start variable (value to subtract from in the difference calculation) - oops::RequiredParameter variableStart{"variable start", this}; - - /// End variable (value to subtract in the difference calculation) - oops::RequiredParameter variableEnd{"variable end", this}; - - /// Whether to return signed difference (true) or absolute difference (false) - /// Default is true (signed difference). - oops::Parameter computeSignedDifference{"signed", true, this}; - - /// Circular period (e.g., 360.0 for degrees, 2*pi for radians, 24 for hours - /// in a day, etc.) - oops::RequiredParameter circularPeriod{"circular period", this}; -}; - -/// \brief Compute minimum circular difference between two variables -/// -/// This ObsFunction computes the **minimum (shortest-path)** difference -/// between two values defined on a circular (cyclic) domain, accounting -/// for wrap-around at the circular period boundary. -/// -/// Consider two angles: For A (variable start) and B (variable end), the -/// circular difference is the shortest angular distance from A to B. -/// -/// If signed = true: -/// Returns a signed difference in the range [-period/2, period/2) -/// For angles, positive values indicate B is clockwise from A -/// For angles, negative values indicate B is counter-clockwise from A -/// -/// If signed = false: -/// Returns the absolute (unsigned) difference in the range [0, period/2] -/// -/// Example for wind directions (period = 360): -/// - A=10°, B=350°: signed=-20°, unsigned=20° -/// - A=350°, B=10°: signed=20°, unsigned=20° -/// - A=0°, B=180°: signed=-180°, unsigned=180° -/// -/// Example for days of the week (period = 7): -/// - A=1 (Monday), B=6 (Saturday): signed=-2, unsigned=2 -/// - A=3 (Wednesday), B=4 (Thursday): signed=1, unsigned=1 -/// -/// Usage example: -/// \code{.yaml} -/// obs function: -/// name: ObsFunction/CircularDifference -/// options: -/// variable start: ObsValue/windDirectionA -/// variable end: ObsValue/windDirectionB -/// signed: true -/// circular period: 360.0 -/// \endcode -/// -/// \note -/// This function always returns the minimum (shortest-path) circular -/// difference. Selecting the complementary (long-path) difference is -/// not currently supported, but may be added as a future option. -class CircularDifference : public ObsFunctionBase { - public: - static const std::string classname() { return "CircularDifference"; } - - /// Constructor takes configuration and validates parameters - explicit CircularDifference(const eckit::LocalConfiguration &); - - /// Destructor - ~CircularDifference(); - - /// Compute the circular difference - void compute(const ObsFilterData &, ioda::ObsDataVector &) const; - - /// Return the required variables for this ObsFunction - const ufo::Variables &requiredVariables() const; - - private: - /// Parameters for this ObsFunction - CircularDifferenceParameters options_; - - /// Required variables - ufo::Variables invars_; -}; - -// ----------------------------------------------------------------------------- - -} // namespace ufo - -#endif // UFO_FILTERS_OBSFUNCTIONS_CIRCULARDIFFERENCE_H_ diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc index 235ff22a7..f74dff940 100644 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualAVHRR.cc @@ -235,6 +235,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, // Minimum Residual Method (MRM) for Cloud Detection: // Determine model level index of the cloud top (lcloud) + // Find pressure of the cloud top (cldprs) // Estimate cloud fraction (cldfrac) // output: out = 0 clear channel // out = 1 cloudy channel @@ -258,12 +259,8 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, dtempf = dtempf_in[2]; } else if (snow) { dtempf = dtempf_in[3]; - } else if (mixed) { - dtempf = dtempf_in[4]; } else { - std::string errString = "The surface type is not defined for location: "; - oops::Log::error() << errString << iloc << std::endl; - throw eckit::BadValue(errString); + dtempf = dtempf_in[4]; } std::vector> dbt(nchans, std::vector(nlevs)); for (size_t ichan=0; ichan < nchans; ++ichan) { @@ -276,6 +273,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, // Set initial cloud condition int lcloud = 0; float cldfrac = 0.0; + float cldprs = prsl[0][iloc] * 0.01; // convert from [Pa] to [hPa] // Loop through vertical layer from surface to model top for (size_t k = 0 ; k < nlevs ; ++k) { @@ -307,6 +305,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, sum3 = sum; lcloud = k + 1; // array index + 1 -> model coordinate index cldfrac = cloudp; + cldprs = prsl[k][iloc] * 0.01; } } // end of vertical loop @@ -335,6 +334,7 @@ void CloudDetectMinResidualAVHRR::compute(const ObsFilterData & in, } if (fabs(sumx2) < FLT_MIN) sumx2 = copysign(1.0e-12, sumx2); dts = std::fabs(sumx / sumx2); + float dts_save = dts; if (std::abs(dts) > 1.0) { if (sea == false) { dts = std::min(dtempf, dts); diff --git a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc index dc56c6f6f..58af4a778 100644 --- a/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc +++ b/src/ufo/filters/obsfunctions/CloudDetectMinResidualIR.cc @@ -247,14 +247,23 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Minimum Residual Method (MRM) for Cloud Detection: // Determine model level index of the cloud top (lcloud) + // Find pressure of the cloud top (cldprs) // Estimate cloud fraction (cldfrac) // output: out = 0 clear channel // out = 1 cloudy channel // out = 2 clear channel but too sensitive to surface condition + // Set vectors to hold cloud infomation from cloud detection (cab be part of the output) + // std::vector cloud_prsl(nlocs); + // std::vector cloud_frac(nlocs); + // std::vector cloud_lev(nlocs); // Loop through locations for (size_t iloc=0; iloc < nlocs; ++iloc) { + // Initialize at each location + // cloud_lev[iloc] = 0; + // cloud_prsl[iloc] = 0.0; + // cloud_frac[iloc] = 0.0; float sum = 0.0, sum2 = 0.0, sum3 = 0.0; float tmp = 0.0; float cloudp = 0.0; @@ -269,6 +278,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Set initial cloud condition int lcloud = 0; float cldfrac = 0.0; + float cldprs = prsl[0][iloc] * 0.01; // convert from [Pa] to [hPa] float sum_min = 1.e20; // Loop through vertical layer from surface to model top @@ -304,6 +314,7 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, sum_min = sum; lcloud = k + 1; // array index + 1 -> model coordinate index cldfrac = cloudp; + cldprs = prsl[k][iloc] * 0.01; } } // end of vertical loop @@ -332,6 +343,10 @@ void CloudDetectMinResidualIR::compute(const ObsFilterData & in, // Active channels if (out[ichan][iloc] < 1 && tao_cld > 0.02) out[ichan][iloc] = 1; } + // cloud infomation output at model level + // cloud_lev[iloc] = lcloud; + // cloud_prsl[iloc] = cldprs; + // cloud_frac[iloc] = cldfrac; } else { // If no clouds is detected, do sensivity to surface temperature check // Initialize at each location diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc index 8fbb73a0c..09c834695 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUA.cc @@ -86,7 +86,7 @@ void HydrometeorCheckAMSUA::compute(const ObsFilterData & in, // Set channel index int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 3, ich536 = 4; - int ich544 = 5, ich890 = 14; + int ich544 = 5, ich549 = 6, ich890 = 14; // Get test groups from options const std::string &biastermgrp = options_.testBiasTerm.value(); @@ -184,7 +184,7 @@ void HydrometeorCheckAMSUA::compute(const ObsFilterData & in, // Set parameters float w1f6 = 1.0/10.0, w2f6 = 1.0/0.80; - float w2f4 = 1.0/1.80; + float w1f4 = 1.0/0.30, w2f4 = 1.0/1.80; std::vector> affected_channels(nchans, std::vector(nlocs)); // Loop over locations diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc index 78bbe7c38..6e33a67a0 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckAMSUAclr.cc @@ -91,10 +91,10 @@ void HydrometeorCheckAMSUAclr::compute(const ObsFilterData & in, // Set channel index int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 3, ich536 = 4; - int ich544 = 5, ich890 = 14; + int ich544 = 5, ich549 = 6, ich890 = 14; if (inst == "atms") { ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 4, ich536 = 5; - ich544 = 6, ich890 = 15; + ich544 = 6, ich549 = 7, ich890 = 15; } // Get test groups from options diff --git a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc index 689d7464f..2745ce030 100644 --- a/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc +++ b/src/ufo/filters/obsfunctions/HydrometeorCheckATMS.cc @@ -92,10 +92,10 @@ void HydrometeorCheckATMS::compute(const ObsFilterData & in, // Set channel index (AMSU-A like channels) int ich238 = 0, ich314 = 1, ich503 = 2, ich528 = 4, ich536 = 5; - int ich544 = 6, ich890 = 15; + int ich544 = 6, ich549 = 7, ich890 = 15; // Set channel index (MHS like channels) - int ich1650 = 16, ich1830e = 21; + int ich1650 = 16, ich1830a = 17, ich1830b = 18, ich1830c = 19, ich1830d = 20, ich1830e = 21; // Get test groups from options const std::string &biastermgrp = options_.testBiasTerm.value(); @@ -209,7 +209,7 @@ void HydrometeorCheckATMS::compute(const ObsFilterData & in, // Set parameters float w1f6 = 1.0/10.0, w2f6 = 1.0/0.80; - float w2f4 = 1.0/1.80; + float w1f4 = 1.0/0.30, w2f4 = 1.0/1.80; std::vector> affected_channels(nchans, std::vector(nlocs)); // Loop over locations diff --git a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc index 92f66f63b..4deefafa7 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorBoundMW.cc @@ -133,13 +133,13 @@ void ObsErrorBoundMW::compute(const ObsFilterData & in, } // Set channel numbers - int ich536 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { - ich536 = 5; - ich890 = 15; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; + ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { - ich536 = 6; - ich890 = 16; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; + ich544 = 7, ich549 = 8, ich890 = 16; } float thresholdfactor = 3.0; if (options_.thresholdfactor.value() != boost::none) { @@ -222,6 +222,7 @@ void ObsErrorBoundMW::compute(const ObsFilterData & in, const std::vector obserr0 = options_.obserrOriginal.value().get(); for (size_t ichan = 0; ichan < nchans; ++ichan) { + int channel = ichan + 1; in.get(Variable(flaggrp+"/brightnessTemperature", channels_)[ichan], qcflagdata); in.get(Variable(errgrp+"/brightnessTemperature", channels_)[ichan], obserrdata); for (size_t iloc = 0; iloc < nlocs; ++iloc) { diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc index be0a145d7..b8da4a7a5 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorPressureCheck.cc @@ -203,7 +203,7 @@ void ObsErrorFactorPressureCheck::compute(const ObsFilterData & data, formulas::Formulation formulation = formulas::Formulation::Rogers; int iflag; - double sat_specific_humidity = 0.0; + double sat_specific_humidity; const float grav = Constants::grav; const float deg2rad = Constants::deg2rad; const float grav_equator = Constants::grav_equator; @@ -270,7 +270,6 @@ void ObsErrorFactorPressureCheck::compute(const ObsFilterData & data, } } - sfcchk = 0.0f; if (reported_height) { fact = 0.0f; if (obs_height[iloc]-dstn[iloc] > 10.0f) { diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc index 63673694c..649cb7558 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSituDependMW.cc @@ -172,13 +172,13 @@ void ObsErrorFactorSituDependMW::compute(const ObsFilterData & in, in.get(Variable("GeoVaLs/water_area_fraction"), water_frac); // Set channel number - int ich536 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { - ich536 = 5; - ich890 = 15; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; + ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { - ich536 = 6; - ich890 = 16; + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; + ich544 = 7, ich549 = 8, ich890 = 16; } // Get Original Observation Error from ObsFunction diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc index 6cc521b44..e9dd92118 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorSurfJacobianRad.cc @@ -89,7 +89,7 @@ void ObsErrorFactorSurfJacobianRad::compute(const ObsFilterData & in, size_t nlocs = in.nlocs(); size_t nchans = channels_.size(); - size_t ich536 = 0, ich890 = 0; + size_t ich536, ich890; if (inst == "amsua") { ich536 = 5; ich890 = 15; diff --git a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc index f9d19ed73..4060c7780 100644 --- a/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc +++ b/src/ufo/filters/obsfunctions/ObsErrorFactorTopoRad.cc @@ -122,10 +122,12 @@ void ObsErrorFactorTopoRad::compute(const ObsFilterData & in, } } else if (inst == "amsua" || inst == "atms") { // Set channel numbers - int ich544 = 0, ich549 = 0, ich890 = 0; + int ich238, ich314, ich503, ich528, ich536, ich544, ich549, ich890; if (inst == "amsua") { + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 4, ich536 = 5; ich544 = 6, ich549 = 7, ich890 = 15; } else if (inst == "atms") { + ich238 = 1, ich314 = 2, ich503 = 3, ich528 = 5, ich536 = 6; ich544 = 7, ich549 = 8, ich890 = 16; } diff --git a/src/ufo/filters/obsfunctions/SolarZenith.cc b/src/ufo/filters/obsfunctions/SolarZenith.cc index bd5cf0131..ddc161b95 100644 --- a/src/ufo/filters/obsfunctions/SolarZenith.cc +++ b/src/ufo/filters/obsfunctions/SolarZenith.cc @@ -98,9 +98,9 @@ void SolarZenith::compute(const ObsFilterData & in, ioda::ObsDataVector & util::DateTime dayStart = missingDateTime; util::DateTime dayEnd = missingDateTime; // Equation of time (more details below) - double eqnt = 0.0; + double eqnt; // Sine and cosine of declination - double sinDecl = 0.0, cosDecl = 0.0; + double sinDecl, cosDecl; for (size_t loc = 0; loc < nlocs; ++loc) { if (skipRejected && rejected[loc]) { diff --git a/src/ufo/filters/obsfunctions/TimeBinner.cc b/src/ufo/filters/obsfunctions/TimeBinner.cc deleted file mode 100644 index daaf94806..000000000 --- a/src/ufo/filters/obsfunctions/TimeBinner.cc +++ /dev/null @@ -1,334 +0,0 @@ -/* - * (C) Crown copyright 2025, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "ufo/filters/obsfunctions/TimeBinner.h" - -#include -#include -#include -#include -#include - -#include "ioda/ObsDataVector.h" -#include "oops/util/DateTime.h" -#include "oops/util/Duration.h" -#include "oops/util/missingValues.h" -#include "ufo/filters/ObsFilterData.h" - -namespace ufo { - -static ObsFunctionMaker makerObsFunctionTimeBinner_( - "TimeBinner"); - -ObsFunctionTimeBinner::ObsFunctionTimeBinner( - const eckit::LocalConfiguration &conf) { - oops::Log::trace() << "ObsFunctionTimeBinner constructor" << std::endl; - options_.validateAndDeserialize(conf); - - // Add the input timestamp variable to the required variables - invars_ += options_.inputTimestampVariable.value(); - - if (options_.binnedTimestampVariable.value() != boost::none) { - const boost::optional &binnedVar = - options_.binnedTimestampVariable.value(); - invars_ += *binnedVar; - } - - // Validate that the bin window does not overlap with adjacent bins - const util::Duration binIntervalUnit = - getBinIntervalUnit(options_.binIntervalUnit.value()); - const util::Duration binWindowLowerBound{ - options_.binWindowLowerBound.value()}; - const util::Duration binWindowUpperBound{ - options_.binWindowUpperBound.value()}; - - if (binWindowLowerBound > binWindowUpperBound) { - throw eckit::BadValue( - "The bin window lower bound must be less than or equal to the bin " - "window upper bound.", - Here()); - } - if ((binWindowUpperBound - binWindowLowerBound).toSeconds() >= - binIntervalUnit.toSeconds()) { - throw eckit::BadValue( - "Bin window (lower bound to upper bound) must be less than the bin " - "interval unit. " - "This ensures no overlap with adjacent bins.", - Here()); - } - // neither the bin window upper bound or lower bound can have a magnitude of - // greater than the bin interval unit, otherwise the flooring process to - // derive the bin label would be incorrect. - if (std::abs(binWindowLowerBound.toSeconds()) > binIntervalUnit.toSeconds() || - std::abs(binWindowUpperBound.toSeconds()) > binIntervalUnit.toSeconds()) { - throw eckit::BadValue( - "Bin window lower and upper bounds must not exceed the bin interval " - "unit in magnitude. " - "This ensures correct bin label derivation.", - Here()); - } -} - -ObsFunctionTimeBinner::~ObsFunctionTimeBinner() { - oops::Log::trace() << "ObsFunctionTimeBinner destructor" << std::endl; -} - -void ObsFunctionTimeBinner::compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const { - oops::Log::trace() << "ObsFunctionTimeBinner compute start" << std::endl; - - const int missingBinValue = - util::missingValue(); // todo change to missingValue when this - // is allowed to be not just an IntObsFunction - - const size_t nlocs = in.nlocs(); - - // Retrieve input timestamps - std::vector timestamps(nlocs); - in.get(options_.inputTimestampVariable.value(), timestamps); - - // Parse bin interval and window - const util::Duration binIntervalUnit = - getBinIntervalUnit(options_.binIntervalUnit.value()); - const int64_t binIntervalUnitSecs = binIntervalUnit.toSeconds(); - const util::Duration binWindowLowerBound{ - options_.binWindowLowerBound.value()}; - const util::Duration binWindowUpperBound{ - options_.binWindowUpperBound.value()}; - const bool reverseOrder = options_.reverseChronologicalOrder.value(); - - // A fixed epoch is used for integer-second arithmetic - const util::DateTime epoch{"1970-01-01T00:00:00Z"}; - - // Bins are labelled with timestamps which are derived from a "floor" derived - // from the bin interval unit. Consider the timestamp 2018-04-17 01:20:02 UTC: - // if the bin interval unit is "hour" the floored time is 2018-04-17 01:00:00 - // UTC, if the bin interval unit is "minute" the floored time is 2018-04-17 - // 01:20:00 UTC, if the bin interval unit is "day" the floored time is - // 2018-04-17 00:00:00 UTC, etc. - // - // Which "floor"ed timestamp label is applied to a given timestamp depends on - // the bin window start and end. For example, with a bin interval unit of - // "hour" and a bin window start of -25 minutes and end of +25 minutes, the - // timestamp 2018-04-17 01:20:02 UTC would be labelled with the timestamp - // 2018-04-17 01:00:00 UTC (since it falls within the window of that bin), as - // would the timestamp 2018-04-17 00:50:00 UTC. All timestamps in that window - // would be assigned to the same bin number. - // - // Note that reverseOrder only affects the numbering of the bins, not the - // timestamps assigned to them. - // - // Bin windows do not have to cover the label timestamp. Bin window starts - // can be positive, implying that the window starts after the bin label - // timestamp. Bin window ends can also be negative, implying that the window - // ends before the bin label timestamp. However, the bin window (end - start) - // must not exceed the bin interval unit, to avoid overlap with adjacent bins - // and the start must be less than or equal to the end. - - // Iteratively find a valid reference timestamp that falls within a bin window - const util::DateTime firstBinLabelTimestamp = findFirstBinLabelTimestamp( - in.obsspace(), timestamps, binWindowLowerBound, binWindowUpperBound, - binIntervalUnit, epoch, reverseOrder); - - // Compute the bin numbers and optionally the binned timestamps - std::vector binLabelTimestamps( - nlocs, util::missingValue()); - if (firstBinLabelTimestamp == util::missingValue()) { - // No valid reference timestamp found, so all bin outputs are missing - std::fill(out[0].begin(), out[0].end(), missingBinValue); - } else { - // Assigns output bin index (respecting reverse ordering) and label - // timestamp. - const auto assignBin = [&](size_t iloc, int64_t bin, - const util::DateTime &binLabelTimestamp) { - out[0][iloc] = - reverseOrder ? static_cast(-bin) : static_cast(bin); - binLabelTimestamps[iloc] = binLabelTimestamp; - }; - // Returns true if the timestamp falls within the given bin label's window. - const auto inWindow = [&](const util::DateTime ×tamp, - const util::DateTime &binLabelTimestamp) { - return timestamp >= binLabelTimestamp + binWindowLowerBound && - timestamp <= binLabelTimestamp + binWindowUpperBound; - }; - - const int64_t firstBinLabelTimestampSecs = - (firstBinLabelTimestamp - epoch).toSeconds(); - for (size_t iloc = 0; iloc < nlocs; ++iloc) { - if (timestamps[iloc] == util::missingValue()) { - out[0][iloc] = missingBinValue; - continue; - } - // Find the candidate bin label timestamp for the current timestamp by - // flooring to the nearest bin interval unit. This uses modular arithmetic: - // timestampSecs - (timestampSecs % binIntervalUnitSecs) floors to the - // nearest multiple of binIntervalUnitSecs. Example: for a timestamp of - // 2018-04-17 01:20:02 UTC with hourly bins (binIntervalUnitSecs=3600): - // timestampSecs=1523928002, 1523928002 % 3600 = 1202, so the floor is - // 1523926800 which corresponds to 2018-04-17 01:00:00 UTC. - const int64_t timestampSecs = (timestamps[iloc] - epoch).toSeconds(); - const int64_t candidateBinLabelTimestampSecs = - timestampSecs - (timestampSecs % binIntervalUnitSecs); - const util::DateTime candidateBinLabelTimestamp = - epoch + util::Duration(candidateBinLabelTimestampSecs); - - // The candidate bin label should be exactly divisible by the bin interval - // unit, allowing the probable bin number to be deduced - assert((candidateBinLabelTimestampSecs % binIntervalUnitSecs) == 0); - int64_t candidateBinNumber = - (candidateBinLabelTimestampSecs - firstBinLabelTimestampSecs) / - binIntervalUnitSecs; - - if (inWindow(timestamps[iloc], candidateBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber, candidateBinLabelTimestamp); - continue; - } - // The timestamp doesn't fall within the candidate bin's window, but may - // still fall within the window of a neighboring bin, so check those - // before giving up and assigning missing. Note that only the - // immediately adjacent bins need to be checked, due to the validation - // that the bin window starts and ends cannot exceed the bin interval - // unit. Note also we don't check the previous bin if the candidate bin - // number is 0, since we don't allow negative bin numbers in the current - // implementation (see the documentation). - out[0][iloc] = missingBinValue; - const util::DateTime nextBinLabelTimestamp = - candidateBinLabelTimestamp + binIntervalUnit; - if (inWindow(timestamps[iloc], nextBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber + 1, nextBinLabelTimestamp); - } else if (candidateBinNumber > 0) { - const util::DateTime prevBinLabelTimestamp = - candidateBinLabelTimestamp - binIntervalUnit; - if (inWindow(timestamps[iloc], prevBinLabelTimestamp)) { - assignBin(iloc, candidateBinNumber - 1, prevBinLabelTimestamp); - } - } - } - } - // Save bin label timestamps if requested - if (options_.binnedTimestampVariable.value() != boost::none) { - const Variable &binnedVar = *options_.binnedTimestampVariable.value(); - in.obsspace().put_db(binnedVar.group(), binnedVar.variable(), - binLabelTimestamps); - } - - oops::Log::trace() << "ObsFunctionTimeBinner compute complete" << std::endl; -} - -const util::Duration ObsFunctionTimeBinner::getBinIntervalUnit( - const std::string &unit) const { - if (unit == "second" || unit == "seconds") { - return util::Duration("PT1S"); - } else if (unit == "minute" || unit == "minutes") { - return util::Duration("PT1M"); - } else if (unit == "hour" || unit == "hours") { - return util::Duration("PT1H"); - } else if (unit == "day" || unit == "days") { - return util::Duration("P1D"); - } else { - throw eckit::BadValue("Unsupported bin interval unit: " + unit, Here()); - } -} - -const ufo::Variables &ObsFunctionTimeBinner::requiredVariables() const { - return invars_; -} - -const util::DateTime ObsFunctionTimeBinner::findFirstBinLabelTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::Duration &binWindowLowerBound, - const util::Duration &binWindowUpperBound, - const util::Duration &binIntervalUnit, const util::DateTime &epoch, - bool reverseOrder) const { - // Find a valid reference timestamp (min or max, depending on order) that - // falls within a bin window. This determines the first bin label and allows - // all other bin numbers to be computed consistently. Iteratively try - // reference timestamps until one is found that fits, or return missing if - // none fit. - auto workingTimestamps = timestamps; - const int64_t binWindowLowerBoundSecs = binWindowLowerBound.toSeconds(); - const int64_t binIntervalUnitSecs = binIntervalUnit.toSeconds(); - - while (!workingTimestamps.empty()) { - // Select the earliest (chronological) or latest (reverse) timestamp. - const util::DateTime trialReferenceTimestamp = - reverseOrder - ? getGlobalMaxTimestamp(obsSpace, workingTimestamps, epoch) - : getGlobalMinTimestamp(obsSpace, workingTimestamps, epoch); - const int64_t trialReferenceTimestampSecs = - (trialReferenceTimestamp - epoch).toSeconds(); - - // Compute the candidate bin label for this reference timestamp. The label - // is derived by flooring to the bin interval unit, but first we adjust the - // timestamp to account for the window lower bound offset. This ensures that - // timestamps near (but after) a potential bin label are mapped to the - // correct label, not the previous one. For example: with a 1-hour interval - // and a relative window of [-30 min, +30 min), a reference at 19:40 - // belongs to the 20:00 bin window [19:30, 20:30), not the 19:00 window. - // The adjustment below handles this by shifting negative window lower - // bounds forward before flooring. - int64_t trialReferenceTimestampAdjustedForWindowSecs = - trialReferenceTimestampSecs; - if (binWindowLowerBoundSecs < 0) { - trialReferenceTimestampAdjustedForWindowSecs -= binWindowLowerBoundSecs; - } - - const int64_t candidateBinLabelTimestampSecs = - trialReferenceTimestampAdjustedForWindowSecs - - (trialReferenceTimestampAdjustedForWindowSecs % binIntervalUnitSecs); - const util::DateTime candidateBinLabelTimestamp = - epoch + util::Duration(candidateBinLabelTimestampSecs); - - // Check if the reference timestamp falls within this candidate bin's - // window. - const util::DateTime candidateBinWindowLowerBound = - candidateBinLabelTimestamp + binWindowLowerBound; - const util::DateTime candidateBinWindowUpperBound = - candidateBinLabelTimestamp + binWindowUpperBound; - - if (trialReferenceTimestamp >= candidateBinWindowLowerBound && - trialReferenceTimestamp <= candidateBinWindowUpperBound) { - // Found a valid first bin label. - return candidateBinLabelTimestamp; - } - - // This reference timestamp doesn't fit any bin window; remove it and - // iterate again using the next min/max timestamp. - workingTimestamps.erase( - std::remove(workingTimestamps.begin(), workingTimestamps.end(), - trialReferenceTimestamp), - workingTimestamps.end()); - } - return util::missingValue(); -} - -const util::DateTime ObsFunctionTimeBinner::getGlobalMinTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const { - const util::DateTime minTimestamp = *std::min_element( - timestamps.begin(), timestamps.end(), - [](const util::DateTime &a, const util::DateTime &b) { return a < b; }); - int64_t minTimestampSecs = (minTimestamp - epoch).toSeconds(); - obsSpace.comm().allReduceInPlace(minTimestampSecs, eckit::mpi::min()); - return epoch + util::Duration(minTimestampSecs); -} - -const util::DateTime ObsFunctionTimeBinner::getGlobalMaxTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const { - const util::DateTime maxTimestamp = *std::max_element( - timestamps.begin(), timestamps.end(), - [](const util::DateTime &a, const util::DateTime &b) { return a < b; }); - int64_t maxTimestampSecs = (maxTimestamp - epoch).toSeconds(); - obsSpace.comm().allReduceInPlace(maxTimestampSecs, eckit::mpi::max()); - return epoch + util::Duration(maxTimestampSecs); -} - -} // namespace ufo diff --git a/src/ufo/filters/obsfunctions/TimeBinner.h b/src/ufo/filters/obsfunctions/TimeBinner.h deleted file mode 100644 index ad97415ff..000000000 --- a/src/ufo/filters/obsfunctions/TimeBinner.h +++ /dev/null @@ -1,247 +0,0 @@ -/* - * (C) Crown copyright 2025, Met Office - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ -#define UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ - -#include -#include - -#include "ioda/ObsSpace.h" -#include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" -#include "ufo/filters/obsfunctions/ObsFunctionBase.h" -#include "ufo/filters/Variable.h" -#include "ufo/filters/Variables.h" -#include "ufo/utils/parameters/ParameterTraitsVariable.h" - -namespace ufo { - -/// \brief Parameters controlling IntObsFunction/TimeBinner. -/// -/// TimeBinner assigns each observation timestamp to a bin label timestamp and -/// returns a bin number as an integer ObsFunction output. -/// -/// Bin membership rule: -/// An observation timestamp t belongs to bin label T when -/// T + bin window lower bound <= t <= T + bin window upper bound -/// -/// Key constraints (validated in the implementation): -/// - bin window lower bound <= bin window upper bound -/// - (bin window upper bound - bin window lower bound) < bin interval unit -/// - |bin window lower bound| and |bin window upper bound| are each <= bin -/// interval unit -/// -/// If an observation falls outside all bin windows, the output bin number is -/// missing. -class TimeBinnerParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(TimeBinnerParameters, Parameters) - - public: - /// \brief Bin interval granularity. - /// - /// Accepted values: second, seconds, minute, minutes, hour, hours, day, days. - oops::RequiredParameter binIntervalUnit{"bin interval unit", - this}; - - /// \brief Lower bound of the bin window relative to the bin label timestamp. - /// - /// Example values: -PT30M, PT0S, PT1S. - oops::RequiredParameter binWindowLowerBound{ - "bin window lower bound", this}; - - /// \brief Upper bound of the bin window relative to the bin label timestamp. - /// - /// Example values: PT0M, PT30M, PT1H. - oops::RequiredParameter binWindowUpperBound{ - "bin window upper bound", this}; - - /// \brief Input timestamp variable. - /// - /// Default: MetaData/dateTime. - oops::Parameter inputTimestampVariable{ - "input timestamp variable", Variable{"MetaData/dateTime"}, this}; - - /// \brief Optional output variable holding bin label timestamps. - /// - /// For observations assigned to bins, the label timestamp is written. - /// For observations outside all bin windows, missing DateTime is written. - oops::OptionalParameter binnedTimestampVariable{ - "binned timestamp variable", this}; - - /// \brief If true, bin numbering is reverse chronological. - /// - /// - false: bin 0 is the earliest valid bin label. - /// - true: bin 0 is the latest valid bin label. - oops::Parameter reverseChronologicalOrder{"reverse chronological order", - false, this}; -}; - -// ----------------------------------------------------------------------------- - -/// \brief Bins observation timestamps into discrete time intervals. -/// -/// This is an IntObsFunction registered as IntObsFunction/TimeBinner. -/// -/// The output integer is the bin number. Optionally, a second variable can be -/// populated with the associated bin label timestamp. -/// -/// Example A: window ending at the hour boundary -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT0M -/// binned timestamp variable: MetaData/binTimestamps -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-16T23:50:00Z | 0 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:15:00Z | missing | missing | -/// | 2018-04-17T00:35:00Z | 1 | 2018-04-17T01:00:00Z | -/// -/// Example B: window spanning the hour boundary -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT29M59S -/// binned timestamp variable: MetaData/binTimestamps -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-17T00:15:00Z | 0 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:35:00Z | 1 | 2018-04-17T01:00:00Z | -/// -/// Example C: reverse chronological order (same window as Example B) -/// \code{.yaml} -/// - filter: Variable Assignment -/// assignments: -/// - name: MetaData/binNumbers -/// type: int -/// function: -/// name: IntObsFunction/TimeBinner -/// options: -/// bin interval unit: hour -/// bin window lower bound: -PT30M -/// bin window upper bound: PT29M59S -/// binned timestamp variable: MetaData/binTimestamps -/// reverse chronological order: true -/// \endcode -/// | input timestamp | bin number | binned timestamp | -/// |----------------------|------------|----------------------| -/// | 2018-04-17T00:15:00Z | 1 | 2018-04-17T00:00:00Z | -/// | 2018-04-17T00:35:00Z | 0 | 2018-04-17T01:00:00Z | -class ObsFunctionTimeBinner : public ObsFunctionBase { - public: - /// \brief Construct TimeBinner from YAML/local configuration. - /// - /// Validates TimeBinner options and declares required input variables. - /// - /// \param conf Local configuration containing TimeBinner options. - explicit ObsFunctionTimeBinner(const eckit::LocalConfiguration &conf); - - /// \brief Destroy the TimeBinner instance. - ~ObsFunctionTimeBinner(); - - /// \brief Compute bin numbers for all observations. - /// - /// Writes integer bin numbers to \p out. If configured, also writes the - /// corresponding bin label timestamps to ObsSpace. Observations with missing - /// input timestamps receive missing (invalid) bin numbers. - /// - /// \param in Input data accessor providing observation values. - /// \param out Output integer vector storing bin numbers. - void compute(const ObsFilterData &in, - ioda::ObsDataVector &out) const override; - - /// \brief Return variables required by this ObsFunction. - /// - /// Includes the input timestamp variable and optionally the configured output - /// binned timestamp variable. - /// - /// \return Set of variables required by TimeBinner. - const ufo::Variables &requiredVariables() const override; - - private: - TimeBinnerParameters options_; - ufo::Variables invars_; - - /// \brief Find the first valid bin label timestamp used as bin-number origin. - /// - /// Iteratively selects a trial reference timestamp (min or max depending on - /// \p reverseOrder), derives a candidate bin label, and returns the first - /// candidate whose window contains that trial timestamp. If no timestamps fit - /// into any bin window, returns missing DateTime. - /// - /// \param obsSpace ObsSpace used for MPI-aware min/max operations. - /// \param timestamps Candidate timestamps (must not be empty). - /// \param binWindowLowerBound Window lower bound offset from bin label. - /// \param binWindowUpperBound Window upper bound offset from bin label. - /// \param binIntervalUnit Bin interval unit duration (hour, minute, second, - /// day). - /// \param epoch Epoch used for integer-second arithmetic. - /// \param reverseOrder If true, select global max first; otherwise min first. - /// \return First valid bin label timestamp, or missing DateTime if none - /// found. - const util::DateTime findFirstBinLabelTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::Duration &binWindowLowerBound, - const util::Duration &binWindowUpperBound, - const util::Duration &binIntervalUnit, const util::DateTime &epoch, - bool reverseOrder) const; - - /// \brief Return global minimum timestamp across MPI ranks. - /// - /// \param obsSpace ObsSpace providing communicator. - /// \param timestamps Local timestamps (must not be empty). - /// \param epoch Epoch used for integer-second arithmetic. - /// \return Global minimum timestamp. - const util::DateTime getGlobalMinTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const; - - /// \brief Return global maximum timestamp across MPI ranks. - /// - /// \param obsSpace ObsSpace providing communicator. - /// \param timestamps Local timestamps (must not be empty). - /// \param epoch Epoch used for integer-second arithmetic. - /// \return Global maximum timestamp. - const util::DateTime getGlobalMaxTimestamp( - const ioda::ObsSpace &obsSpace, - const std::vector ×tamps, - const util::DateTime &epoch) const; - - /// \brief Convert bin interval unit string to util::Duration. - /// - /// Supported units are second(s), minute(s), hour(s), and day(s). - /// - /// \param unit Unit string from configuration. - /// \return Duration corresponding to \p unit. - /// \throw eckit::BadValue if \p unit is unsupported. - const util::Duration getBinIntervalUnit(const std::string &unit) const; -}; - -} // namespace ufo - -#endif // UFO_FILTERS_OBSFUNCTIONS_TIMEBINNER_H_ diff --git a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc index 73584558a..f5bd27dc7 100644 --- a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc +++ b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.cc @@ -29,8 +29,8 @@ namespace ufo { RefractivityOneDVarCheck::RefractivityOneDVarCheck( ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr + std::shared_ptr > flags, + std::shared_ptr > obserr ) : FilterBase(obsdb, parameters, flags, obserr), parameters_(parameters) { @@ -94,8 +94,8 @@ void RefractivityOneDVarCheck::applyFilter( } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc - obserr_.save("FortranERR"); // Pass latest errors to 1DVar + flags_->save("FortranQC"); // temporary measure as per ROobserror qc + obserr_->save("FortranERR"); // Pass latest errors to 1DVar // Pass it all to fortran ufo_refractivityonedvarcheck_apply_f90( @@ -105,7 +105,7 @@ void RefractivityOneDVarCheck::applyFilter( apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "RefractivityOneDVarCheck Filter complete" << std::endl; } diff --git a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h index ccdc622f8..6d1a5b555 100644 --- a/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h +++ b/src/ufo/filters/refractivityonedvarcheck/RefractivityOneDVarCheck.h @@ -58,8 +58,8 @@ class RefractivityOneDVarCheck : public FilterBase, RefractivityOneDVarCheck( ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~RefractivityOneDVarCheck(); private: diff --git a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc index 1c6719d82..29dad9cf2 100644 --- a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc +++ b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.cc @@ -27,8 +27,8 @@ namespace ufo { // ----------------------------------------------------------------------------- RTTOVOneDVarCheck::RTTOVOneDVarCheck(ioda::ObsSpace & obsdb, const Parameters_ & parameters, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, parameters, flags, obserr), channels_(), retrieved_vars_(), hoxdiags_retrieved_vars_(), parameters_(parameters) { @@ -117,7 +117,7 @@ void RTTOVOneDVarCheck::applyFilter(const std::vector & apply, } // Save qc flags to database for retrieval in fortran - needed for channel selection - flags_.save("FortranQC"); // temporary measure as per ROobserror qc + flags_->save("FortranQC"); // temporary measure as per ROobserror qc // Read in ObsBias for all channels in the database and save to db for access in Fortran // there is currently no mechanism for passing ioda::ObsDataVectors to Fortran. @@ -138,7 +138,7 @@ void RTTOVOneDVarCheck::applyFilter(const std::vector & apply, apply_char.size(), apply_char[0]); // Read qc flags from database - flags_.read("FortranQC"); // temporary measure as per ROobserror qc + flags_->read("FortranQC"); // temporary measure as per ROobserror qc oops::Log::trace() << "RTTOVOneDVarCheck applyFilter complete" << std::endl; } diff --git a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h index db2df0ae6..8b036526b 100644 --- a/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h +++ b/src/ufo/filters/rttovonedvarcheck/RTTOVOneDVarCheck.h @@ -54,8 +54,8 @@ class RTTOVOneDVarCheck : public FilterBase, static const std::string classname() {return "ufo::RTTOVOneDVarCheck";} RTTOVOneDVarCheck(ioda::ObsSpace &, const Parameters_ &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~RTTOVOneDVarCheck() override; private: diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 index e1d60c21f..0c47fa351 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_ob_mod.f90 @@ -243,10 +243,8 @@ subroutine ufo_rttovonedvarcheck_PrintOb(self) write(*,"(10F8.2)") self % background_T write(*,"(A)") "Emissivity: " write(*,"(10F8.2)") self % emiss(:) -if (allocated(self % pcemiss)) then - write(*,"(A)") "Emissivity PC: " - write(*,"(10F18.8)") self % pcemiss(:) -endif +write(*,"(A)") "Emissivity PC: " +write(*,"(10F18.8)") self % pcemiss(:) end subroutine diff --git a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 index 7b23fe350..138af5d8b 100644 --- a/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 +++ b/src/ufo/filters/rttovonedvarcheck/ufo_rttovonedvarcheck_setup_mod.f90 @@ -403,11 +403,9 @@ subroutine ufo_rttovonedvarcheck_print(self) write(*,*) "EmissLandDefault = ",self % EmissLandDefault write(*,*) "EmissSeaIceDefault = ",self % EmissSeaIceDefault write(*,*) "mwEmissRetrieval = ",self % mwEmissRetrieval -if (self % mwEmissRetrieval) then - write(*,*) "NumEmissElements = ",self % NumEmissElements - write(*,*) "EmissToChannelMap = ",self % EmissToChannelMap - write(*,*) "ChannelToEmissMap = ",self % ChannelToEmissMap -endif +write(*,*) "NumEmissElements = ",self % NumEmissElements +write(*,*) "EmissToChannelMap = ",self % EmissToChannelMap +write(*,*) "ChannelToEmissMap = ",self % ChannelToEmissMap write(*,*) "Use PC for Emissivity = ", self % pcemiss write(*,*) "EmisEigVecPath = ",self % EmisEigVecPath write(*,*) "EmisAtlas = ",self % EmisAtlas diff --git a/src/ufo/fov/calc_fov_conical.f90 b/src/ufo/fov/calc_fov_conical.f90 index 9ca3dbdf5..312f984ac 100644 --- a/src/ufo/fov/calc_fov_conical.f90 +++ b/src/ufo/fov/calc_fov_conical.f90 @@ -1032,11 +1032,7 @@ subroutine inside_fov_conical(instr,ichan,satellite_azimuth,lat,lon, & distance_east = r2*cos(lat*deg2rad)*dellon ! Angle to the test point - if (distance_north == 0 .and. distance_east == 0) then - bearing_to_test = 0 - else - bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) - endif + bearing_to_test = mod(atan2(distance_north,distance_east),two*pi) bearing_to_test_deg = bearing_to_test*rad2deg ! convert to degrees ! This is the arc distance to the test point diff --git a/src/ufo/fov/calc_fov_crosstrk.f90 b/src/ufo/fov/calc_fov_crosstrk.f90 index 7a2c49072..39f4e86a0 100644 --- a/src/ufo/fov/calc_fov_crosstrk.f90 +++ b/src/ufo/fov/calc_fov_crosstrk.f90 @@ -1360,11 +1360,7 @@ subroutine inside_fov_crosstrk(instr, ifov, satellite_azimuth, & distance_east = r2*cos(lat*deg2rad)*dellon ! angle to the test point - if (distance_north == 0 .and. distance_east == 0) then - bearing_to_test = 0 - else - bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) - endif + bearing_to_test = mod(atan2(distance_north,distance_east),2*pi ) bearing_to_test_deg = bearing_to_test*rad2deg ! convert to degrees ! this is the arc distance to the test point diff --git a/src/ufo/obslocalization/ObsHorLocGC99.h b/src/ufo/obslocalization/ObsHorLocGC99.h index d6a3f92b1..76b33e51b 100644 --- a/src/ufo/obslocalization/ObsHorLocGC99.h +++ b/src/ufo/obslocalization/ObsHorLocGC99.h @@ -31,8 +31,6 @@ class ObsHorLocGC99: public ufo::ObsHorLocalization { public: ObsHorLocGC99(const eckit::Configuration &, ioda::ObsSpace &); - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; protected: /// Compute GC99 localization using the set of \p localobs, the same lengthscale /// used to search for those obs, and save localization values in \p locvector. @@ -54,20 +52,6 @@ ObsHorLocGC99::ObsHorLocGC99(const eckit::Configuration & conf, // ----------------------------------------------------------------------------- -template -double ObsHorLocGC99::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - double lengthscale = ObsHorLocalization::lengthscale(); - double loc = oops::gc99(distance / lengthscale); - return (distance >= lengthscale) ? 0.0 : loc; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocGC99::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsHorLocSOAR.h b/src/ufo/obslocalization/ObsHorLocSOAR.h index 4404d842d..101e769d8 100644 --- a/src/ufo/obslocalization/ObsHorLocSOAR.h +++ b/src/ufo/obslocalization/ObsHorLocSOAR.h @@ -31,8 +31,6 @@ class ObsHorLocSOAR: public ufo::ObsHorLocalization { public: ObsHorLocSOAR(const eckit::Configuration &, ioda::ObsSpace &); - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; protected: /// Compute SOAR localization using the set of \p localobs and save localization /// values in \p locvector. @@ -58,20 +56,6 @@ ObsHorLocSOAR::ObsHorLocSOAR(const eckit::Configuration & config, // ----------------------------------------------------------------------------- -template -double ObsHorLocSOAR::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - double lengthscale = ObsHorLocalization::lengthscale(); - double loc = oops::soar(distance * options_.SOARexpDecayH); - return (distance >= lengthscale) ? 0.0 : loc; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocSOAR::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsHorLocalization.h b/src/ufo/obslocalization/ObsHorLocalization.h index 51484f090..03798a57a 100644 --- a/src/ufo/obslocalization/ObsHorLocalization.h +++ b/src/ufo/obslocalization/ObsHorLocalization.h @@ -32,10 +32,6 @@ #include "ufo/obslocalization/ObsHorLocParameters.h" #include "ufo/obslocalization/ObsLocalizationBase.h" -namespace eckit::geometry { - class Point3; -} - namespace ufo { /// Horizontal Box car observation space localization @@ -50,14 +46,6 @@ class ObsHorLocalization: public ObsLocalizationBase { void computeLocalization(const ITERATOR &, ioda::ObsVector & locvector) const override; - /// Compute localization values between two points (regardless of if those are MODEL/OBS - /// or OBS/OBS locations) and return the localization value. The lengthscale from - /// ObsHorLocParameters is used. - /// Return values should be between 0.0 and 1.0, inclusive, with 0 indicating that the points - /// are outside of localization. - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const override; - protected: struct LocalObs { /// The list of indexes for ObsVector pointing to the valid local obs. @@ -185,18 +173,6 @@ void ObsHorLocalization::computeLocalization(const ITERATOR & i, // ----------------------------------------------------------------------------- -template -double ObsHorLocalization::computeLocalization( - const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - eckit::geometry::Point2 p1_2(p1[0], p1[1]); - eckit::geometry::Point2 p2_2(p2[0], p2[1]); - double distance = atlas::util::Earth::distance(p1_2, p2_2); - return (distance >= options_.lengthscale) ? 0.0 : 1.0; -} - -// ----------------------------------------------------------------------------- - template void ObsHorLocalization::localizeLocalObs(const ITERATOR & i, ioda::ObsVector & locvector, diff --git a/src/ufo/obslocalization/ObsLocalization.h b/src/ufo/obslocalization/ObsLocalization.h index b03048b2f..58f7d03e7 100644 --- a/src/ufo/obslocalization/ObsLocalization.h +++ b/src/ufo/obslocalization/ObsLocalization.h @@ -30,8 +30,6 @@ class ObsLocalization : public util::Printable { ~ObsLocalization(); void computeLocalization(const ITERATOR &, ioda::ObsVector &) const; - double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const; private: void print(std::ostream &) const override; @@ -63,14 +61,6 @@ void ObsLocalization::computeLocalization(const ITERATOR & iter, // ----------------------------------------------------------------------------- -template -double ObsLocalization::computeLocalization(const eckit::geometry::Point3 & p1, - const eckit::geometry::Point3 & p2) const { - return obsloc_->computeLocalization(p1, p2); -} - -// ----------------------------------------------------------------------------- - template void ObsLocalization::print(std::ostream & os) const { os << *obsloc_; diff --git a/src/ufo/obslocalization/ObsLocalizationBase.h b/src/ufo/obslocalization/ObsLocalizationBase.h index afde57d41..130f6b61e 100644 --- a/src/ufo/obslocalization/ObsLocalizationBase.h +++ b/src/ufo/obslocalization/ObsLocalizationBase.h @@ -13,7 +13,6 @@ #include #include "eckit/config/Configuration.h" -#include "eckit/geometry/Point3.h" #include "ioda/ObsSpace.h" #include "ioda/ObsVector.h" #include "oops/util/Logger.h" @@ -33,17 +32,6 @@ class ObsLocalizationBase : public util::Printable, /// localization values between observations and \p point in model-space. /// Set \p locfactor to missing value for observations that are not local. virtual void computeLocalization(const ITERATOR & point, ioda::ObsVector & locfactor) const = 0; - - /// compute localization between an observation and a point in model-space, or - /// between two observations. Return values should be between 0.0 and 1.0, inclusive, - /// with 0 indicating that the points are outside of localization. - virtual double computeLocalization(const eckit::geometry::Point3 &, - const eckit::geometry::Point3 &) const { - // NOTE, this really should be a pure virtual method, but since not all - // ObsLocalizationBase derived classes will be implementing this method yet, - // we are providing a default implementation. - ABORT("computeLocalization(Point3,Point3) not implemented for this ObsLocalization class"); - return 0.0;} }; // ============================================================================= diff --git a/src/ufo/operators/crtm/ObsRadianceCRTM.cc b/src/ufo/operators/crtm/ObsRadianceCRTM.cc index 3026b0b3d..4824012b1 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTM.cc +++ b/src/ufo/operators/crtm/ObsRadianceCRTM.cc @@ -503,7 +503,7 @@ void ObsRadianceCRTM::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, ufo_radiancecrtm_simobs_f90(keyOperRadianceCRTM_, gom.toFortran(), odb_, ovec.nvars(), ovec.nlocs(), ovec.toFortran(), dvec.toFortran(), - qc_flags); + reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceCRTM::simulateObs done" << std::endl; } diff --git a/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h b/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h index 627cd5179..9f9c2e323 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h +++ b/src/ufo/operators/crtm/ObsRadianceCRTM.interface.h @@ -40,8 +40,8 @@ extern "C" { oops::Variables &, const eckit::mpi::Comm &); void ufo_radiancecrtm_delete_f90(F90hop &); void ufo_radiancecrtm_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const size_t &, const size_t &, double &, const F90goms &, - const ioda::ObsDataVector &); + const size_t &, const size_t &, double &, const F90goms &, + const void*); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc index 2402aa7ed..6fc570e37 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc +++ b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.cc @@ -61,7 +61,7 @@ ObsRadianceCRTMTLAD::~ObsRadianceCRTMTLAD() { void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags, const QCFlags_t & qc_flags) { ufo_radiancecrtm_tlad_settraj_f90(keyOperRadianceCRTM_, geovals.toFortran(), obsspace(), - ydiags.toFortran(), qc_flags); + ydiags.toFortran(), reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceCRTMTLAD::setTrajectory done" << std::endl; } @@ -69,7 +69,7 @@ void ObsRadianceCRTMTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics void ObsRadianceCRTMTLAD::simulateObsTL(const GeoVaLs & geovals, ioda::ObsVector & ovec) const { ufo_radiancecrtm_simobs_tl_f90(keyOperRadianceCRTM_, geovals.toFortran(), obsspace(), - ovec.nvars(), ovec.nlocs(), ovec.toFortran()); + ovec.nvars(), ovec.nlocs(), ovec.toFortran()); oops::Log::trace() << "ObsRadianceCRTMTLAD::simulateObsTL done" << std::endl; } diff --git a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h index 5aab917cb..705c2ded9 100644 --- a/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h +++ b/src/ufo/operators/crtm/ObsRadianceCRTMTLAD.interface.h @@ -41,11 +41,11 @@ extern "C" { oops::Variables &, const eckit::mpi::Comm &); void ufo_radiancecrtm_tlad_delete_f90(F90hop &); void ufo_radiancecrtm_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const F90goms &, const ioda::ObsDataVector &); + const F90goms &, const void*); void ufo_radiancecrtm_simobs_tl_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &); + const int &, const int &, double &); void ufo_radiancecrtm_simobs_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, const double &); + const int &, const int &, const double &); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 b/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 index 763f17447..f8c3e733b 100644 --- a/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 +++ b/src/ufo/operators/gnssro/BendMetOffice/ufo_gnssro_bendmetoffice_tlad_mod.F90 @@ -160,11 +160,9 @@ subroutine ufo_gnssro_bendmetoffice_tlad_settraj(self, geovals, obss) ! Check that the number of vertical levels in the observations is consistent ! with what we were given in setup - if (self % nlevels > 1) then - if (self % nlevels /= size(self % chanList)) then - write(err_msg,'(2A,4I8)') myname_, ' error: channel list must match nlevels', self % nlevels, size(self%chanList) - call fckit_exception%throw(err_msg) - end if + if (self % nlevels > 1 .AND. self % nlevels /= size(self % chanList)) then + write(err_msg,'(2A,4I8)') myname_, ' error: channel list must match nlevels', self % nlevels, size(self%chanList) + call fckit_exception%throw(err_msg) end if ! Get the meta-data from the observations diff --git a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc index f3ae313f9..a49d23660 100644 --- a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc +++ b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.cc @@ -30,8 +30,8 @@ namespace ufo { BackgroundCheckRONBAM::BackgroundCheckRONBAM(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + std::shared_ptr > flags, + std::shared_ptr > obserr) : FilterBase(obsdb, config, flags, obserr) { oops::Log::trace() << "BackgroundCheckRONBAM contructor start " @@ -75,7 +75,7 @@ void BackgroundCheckRONBAM::applyFilter(const std::vector & apply, data_.get(varhofx.variable(jv), hofx); for (size_t jobs = 0; jobs < obsdb_.nlocs(); ++jobs) { - if (apply[jobs] && flags_[iv][jobs] == 0) { + if (apply[jobs] && (*flags_)[iv][jobs] == 0) { ASSERT(obs[jv][jobs] != missing); ASSERT(hofx[jobs] != missing); ASSERT(impactparameter[0][jobs] != missing); diff --git a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h index b1d6e31ea..c2418880d 100644 --- a/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h +++ b/src/ufo/operators/gnssro/QC/BackgroundCheckRONBAM.h @@ -36,8 +36,8 @@ class BackgroundCheckRONBAM : public FilterBase, static const std::string classname() {return "ufo::BackgroundCheckRONBAM";} BackgroundCheckRONBAM(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~BackgroundCheckRONBAM(); private: diff --git a/src/ufo/operators/gnssro/QC/ROobserror.cc b/src/ufo/operators/gnssro/QC/ROobserror.cc index 1f58737e6..8e3f74add 100644 --- a/src/ufo/operators/gnssro/QC/ROobserror.cc +++ b/src/ufo/operators/gnssro/QC/ROobserror.cc @@ -21,9 +21,9 @@ namespace ufo { ROobserror::ROobserror(ioda::ObsSpace & obsdb, const eckit::Configuration & config, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : FilterBase(obsdb, config, flags, obserr) + std::shared_ptr > qc, + std::shared_ptr > oberr) + : FilterBase(obsdb, config, qc, oberr) { oops::Log::trace() << "ROobserror contructor start" << std::endl; // observation errors are always set up for the variables that are @@ -54,11 +54,11 @@ void ROobserror::applyFilter(const std::vector & apply, std::vector> & flagged) const { oops::Log::trace() << "ROobserror::applyFilter using priorFilter start" << std::endl; // Call the fortran routines to do the processing - flags_.save("FortranQC"); // should pass values to fortran properly - obserr_.save("FortranERR"); // should pass values to fortran properly + flags_->save("FortranQC"); // should pass values to fortran properly + obserr_->save("FortranERR"); // should pass values to fortran properly ufo_roobserror_prior_f90(key_); - flags_.read("FortranQC"); // should get values from fortran properly - obserr_.read("FortranERR"); // should get values from fortran properly + flags_->read("FortranQC"); // should get values from fortran properly + obserr_->read("FortranERR"); // should get values from fortran properly oops::Log::trace() << "ROobserror::applyFilter using priorFilter done" << std::endl; } diff --git a/src/ufo/operators/gnssro/QC/ROobserror.h b/src/ufo/operators/gnssro/QC/ROobserror.h index 777f77c29..b4f786cc7 100644 --- a/src/ufo/operators/gnssro/QC/ROobserror.h +++ b/src/ufo/operators/gnssro/QC/ROobserror.h @@ -38,8 +38,8 @@ class ROobserror : public FilterBase, static const std::string classname() {return "ufo::ROobserror";} ROobserror(ioda::ObsSpace &, const eckit::Configuration &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + std::shared_ptr >, + std::shared_ptr >); ~ROobserror(); private: diff --git a/src/ufo/operators/oasim/ObsRadianceOASIM.cc b/src/ufo/operators/oasim/ObsRadianceOASIM.cc index 9bb532453..6399a15c4 100644 --- a/src/ufo/operators/oasim/ObsRadianceOASIM.cc +++ b/src/ufo/operators/oasim/ObsRadianceOASIM.cc @@ -48,7 +48,8 @@ ObsRadianceOASIM::ObsRadianceOASIM(const ioda::ObsSpace & odb, const Parameters_ "cyano-bacteria_concentration", "coccolithophore_concentration", "dinoflagellate_concentration", - "phaeocystis_concentration"}; + "phaeocystis_concentration", + "cyano-bacteria_concentration"}; varin_.reset(new oops::Variables(vvin)); // parse channels from the config and create variable names diff --git a/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc b/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc index 898ad2b17..975c13056 100644 --- a/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc +++ b/src/ufo/operators/rttov/CPP/v14/rttovcpp_interface.cc @@ -220,7 +220,7 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, //--------------------------------------------------------------------------------- util::DateTime time1; int year, month, day, hour, minute, second; - int surftype = 0; + int surftype; int isurf = 0; for (std::size_t i = 0; i < nprofiles; i++) { @@ -369,6 +369,12 @@ void rttovcpp_interface(const GeoVaLs & geovals, const ioda::ObsSpace & odb_, // get ydiag object ufo::GeoVaLs & ydiag = d.geovals(); + // get variables in ydiag; just for debug + const oops::Variables & vars = ydiag.getVars(); + for (const oops::Variable var : vars) { + oops::Log::trace() << "ydiag name: " << var << std::endl; + } + size_t iSurface = 0; // support only one surface type for now. for (const std::string &varname : ydiag_varnames) { // check varname dimension diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc index 11384477f..a07328157 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.cc @@ -74,8 +74,8 @@ ObsRadianceRTTOV::~ObsRadianceRTTOV() { void ObsRadianceRTTOV::simulateObs(const GeoVaLs & gom, ioda::ObsVector & ovec, ObsDiagnostics & dvec, const QCFlags_t& qc_flags) const { ufo_radiancerttov_simobs_f90(keyOperRadianceRTTOV_, gom.toFortran(), odb_, - ovec.nvars(), ovec.nlocs(), ovec.toFortran(), - dvec.toFortran(), qc_flags); + ovec.nvars(), ovec.nlocs(), ovec.toFortran(), + dvec.toFortran(), reinterpret_cast(&qc_flags)); oops::Log::trace() << "ObsRadianceRTTOV::simulateObs done." << std::endl; } diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h index 8bebe8a88..fa1834276 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOV.interface.h @@ -26,12 +26,12 @@ extern "C" { // Radiance observation operator // ----------------------------------------------------------------------------- void ufo_radiancerttov_setup_f90(F90hop &, const eckit::Configuration &, - const int &, const int &, oops::Variables &, - const int &, const int &); + const int &, const int &, oops::Variables &, + const int &, const int &); void ufo_radiancerttov_delete_f90(F90hop &); void ufo_radiancerttov_simobs_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &, const F90goms &, - const ioda::ObsDataVector &); + const int &, const int &, double &, const F90goms &, + const void*); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc index aa9b92fb3..00fc6d60f 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.cc @@ -32,12 +32,9 @@ ObsRadianceRTTOVTLAD::ObsRadianceRTTOVTLAD(const ioda::ObsSpace & odb, const oops::ObsVariables & observed = odb.assimvariables(); std::vector channels_list = observed.channels(); - // values for passed qc - std::vector qc_passed = {QCflags::pass}; // call Fortran setup routine ufo_radiancerttov_tlad_setup_f90(keyOperRadianceRTTOV_, parameters.toConfiguration(), - channels_list.size(), channels_list[0], varin_, - qc_passed.size(), qc_passed[0]); + channels_list.size(), channels_list[0], varin_); oops::Log::trace() << "ObsRadianceRTTOVTLAD constructor done" << std::endl; } @@ -54,7 +51,7 @@ ObsRadianceRTTOVTLAD::~ObsRadianceRTTOVTLAD() { void ObsRadianceRTTOVTLAD::setTrajectory(const GeoVaLs & geovals, ObsDiagnostics & ydiags, const QCFlags_t & qc_flags) { ufo_radiancerttov_tlad_settraj_f90(keyOperRadianceRTTOV_, geovals.toFortran(), obsspace(), - ydiags.toFortran(), qc_flags); + ydiags.toFortran()); oops::Log::trace() << "ObsRadianceRTTOVTLAD::setTrajectory done" << std::endl; } diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h index 68800e777..a07c2ad1c 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.h @@ -15,7 +15,6 @@ #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" -#include "ufo/filters/QCflags.h" #include "ufo/LinearObsOperatorBase.h" #include "ufo/operators/rttov/Fortran/ObsRadianceRTTOVParameters.h" #include "ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h" diff --git a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h index d24e54011..d91a3b9dc 100644 --- a/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h +++ b/src/ufo/operators/rttov/Fortran/ObsRadianceRTTOVTLAD.interface.h @@ -27,15 +27,15 @@ extern "C" { // ----------------------------------------------------------------------------- void ufo_radiancerttov_tlad_setup_f90(F90hop &, const eckit::Configuration &, - const int &, const int &, - oops::Variables &, const int &, const int &); + const int &, const int &, + oops::Variables &); void ufo_radiancerttov_tlad_delete_f90(F90hop &); void ufo_radiancerttov_tlad_settraj_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const F90goms &, const ioda::ObsDataVector &); + const F90goms &); void ufo_radiancerttov_simobs_tl_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, double &); + const int &, const int &, double &); void ufo_radiancerttov_simobs_ad_f90(const F90hop &, const F90goms &, const ioda::ObsSpace &, - const int &, const int &, const double &); + const int &, const int &, const double &); // ----------------------------------------------------------------------------- } // extern C diff --git a/src/ufo/profile/ProfileAveragePressure.cc b/src/ufo/profile/ProfileAveragePressure.cc index 9afbc4dad..ef7f616fa 100644 --- a/src/ufo/profile/ProfileAveragePressure.cc +++ b/src/ufo/profile/ProfileAveragePressure.cc @@ -106,13 +106,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageRelativeHumidity.cc b/src/ufo/profile/ProfileAverageRelativeHumidity.cc index 94c1ea523..10f6a32e8 100644 --- a/src/ufo/profile/ProfileAverageRelativeHumidity.cc +++ b/src/ufo/profile/ProfileAverageRelativeHumidity.cc @@ -70,13 +70,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageTemperature.cc b/src/ufo/profile/ProfileAverageTemperature.cc index e31c1305d..a849f47b3 100644 --- a/src/ufo/profile/ProfileAverageTemperature.cc +++ b/src/ufo/profile/ProfileAverageTemperature.cc @@ -80,13 +80,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileAverageWindSpeed.cc b/src/ufo/profile/ProfileAverageWindSpeed.cc index b758df1ee..04d7646a2 100644 --- a/src/ufo/profile/ProfileAverageWindSpeed.cc +++ b/src/ufo/profile/ProfileAverageWindSpeed.cc @@ -80,13 +80,13 @@ namespace ufo { // ensure that all of the variables defined above have been added to the ObsSpace // on each rank. This prevents a hang when saving the ObsSpace. for (const auto & variableInt : variableNamesInt) { - profileDataHandler.get(variableInt); + const auto & vectorInt = profileDataHandler.get(variableInt); } for (const auto & variableFloat : variableNamesFloat) { - profileDataHandler.get(variableFloat); + const auto & vectorFloat = profileDataHandler.get(variableFloat); } for (const auto & variableBool : variableNamesBool) { - profileDataHandler.get(variableBool); + const auto & vectorBool = profileDataHandler.get(variableBool); } std::vector profiles = diff --git a/src/ufo/profile/ProfileCheckRH.cc b/src/ufo/profile/ProfileCheckRH.cc index 7de85bebc..29b17e6e7 100644 --- a/src/ufo/profile/ProfileCheckRH.cc +++ b/src/ufo/profile/ProfileCheckRH.cc @@ -36,6 +36,7 @@ namespace ufo { profileDataHandler.get(ufo::ProfileVariableNames::obs_dew_point_temperature); const std::vector &diagFlagsTTropopause = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_tropo_t); + const std::vector &diagFlagsTFinalReject = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_final_reject_t); std::vector &diagFlagsRHInterp = profileDataHandler.get(ufo::ProfileVariableNames::diagflags_interpolation_rh); diff --git a/src/ufo/utils/SaveBiasCoeffs.cc b/src/ufo/utils/SaveBiasCoeffs.cc index aeaa8d3d3..6489f9251 100644 --- a/src/ufo/utils/SaveBiasCoeffs.cc +++ b/src/ufo/utils/SaveBiasCoeffs.cc @@ -42,7 +42,7 @@ void saveBiasCoeffsWithChannels(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficients/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Channel"]}, float_params); - biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::all)); } // Save the Channel @@ -78,7 +78,7 @@ void saveBiasCoeffsWithRecords(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficients/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Variable"]}, float_params); - biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(coeffs(jpred, Eigen::all)); } // Save the stationIdentification diff --git a/src/ufo/utils/SaveBiasCovariance.cc b/src/ufo/utils/SaveBiasCovariance.cc index 2fec25d6b..c0850ba6b 100644 --- a/src/ufo/utils/SaveBiasCovariance.cc +++ b/src/ufo/utils/SaveBiasCovariance.cc @@ -41,7 +41,7 @@ void saveBiasCovarianceWithChannels(ioda::Group & parent, ioda::Variable anvarVar = ogrp.vars.createWithScales( "BiasCoefficientErrors/"+predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Channel"]}, float_params); - anvarVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::placeholders::all)); + anvarVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::all)); } // write number_obs_assimilated @@ -82,7 +82,7 @@ void saveBiasCovarianceWithRecords(ioda::Group & parent, ioda::Variable biasVar = ogrp.vars.createWithScales( "BiasCoefficientErrors/" + predictors[jpred], {ogrp.vars["Record"], ogrp.vars["Variable"]}, float_params); - biasVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::placeholders::all)); + biasVar.writeWithEigenRegular(allbcerrors(jpred, Eigen::all)); } // write number_obs_assimilated diff --git a/src/ufo/utils/dataextractor/DataExtractor.h b/src/ufo/utils/dataextractor/DataExtractor.h index d28c5551b..e51e6daad 100644 --- a/src/ufo/utils/dataextractor/DataExtractor.h +++ b/src/ufo/utils/dataextractor/DataExtractor.h @@ -15,13 +15,7 @@ #include #include -// To skip the warning about maybe-uninitialized variables in boost multi_array headers. -// Will only skip this warning for the boost multi_array headers, -// and only for this file so that we can still get the warning for our own code. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #include -#pragma GCC diagnostic pop #include #include #include diff --git a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h index 855d9d24c..96dddbd8d 100644 --- a/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h +++ b/src/ufo/utils/dataextractor/DataExtractorNetCDFBackend.h @@ -88,7 +88,7 @@ namespace ufo /// group: ErrorVariance { /// variables: /// float airTemperature(sensorChannelNumber, MetaData/sensorChannelNumber, index) ; -/// airTemperature:coordinates = "MetaData/latitudeBand +/// airTemperature:coordinates = "MetaData/latitudeBand \ /// MetaData/processingCenter MetaData/satelliteIdentifier" ; /// } /// \endcode diff --git a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc index ce3e45b19..2a64d119d 100644 --- a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc +++ b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_AdjustedHeightCoordinate::Cal_AdjustedHeightCoordinate( const GenericVariableTransformParameters & options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_AdjustedHeightCoordinate::Constructor start" << std::endl; // List of GeoVaLs this transform will need access to diff --git a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h index 71c05361d..4dfc8e5dd 100644 --- a/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h +++ b/src/ufo/variabletransforms/Cal_AdjustedHeightCoordinate.h @@ -30,9 +30,9 @@ namespace ufo { class Cal_AdjustedHeightCoordinate : public TransformBase { public: Cal_AdjustedHeightCoordinate(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc index 192255abb..8ed3bd197 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.cc +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.cc @@ -18,8 +18,8 @@ static TransformMaker Cal_HeightFromPressure::Cal_HeightFromPressure( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.heightCoord), heightGroup_(options.heightGroup), diff --git a/src/ufo/variabletransforms/Cal_HeightFromPressure.h b/src/ufo/variabletransforms/Cal_HeightFromPressure.h index a19cb26c9..073811d9e 100644 --- a/src/ufo/variabletransforms/Cal_HeightFromPressure.h +++ b/src/ufo/variabletransforms/Cal_HeightFromPressure.h @@ -44,8 +44,8 @@ class Cal_HeightFromPressure : public TransformBase { Cal_HeightFromPressure(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_Humidity.cc b/src/ufo/variabletransforms/Cal_Humidity.cc index 8f57ce994..988154ab4 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.cc +++ b/src/ufo/variabletransforms/Cal_Humidity.cc @@ -19,8 +19,8 @@ static TransformMaker Cal_RelativeHumidity::Cal_RelativeHumidity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), allowSuperSaturation_(options.AllowSuperSaturation), specifichumidityvariable_(options.SpecificHumidityVariable), @@ -427,8 +427,8 @@ static TransformMaker Cal_SpecificHumidity::Cal_SpecificHumidity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), specifichumidityvariable_(options.SpecificHumidityVariable), pressurevariable_(options.PressureVariable), @@ -651,8 +651,8 @@ static TransformMaker Cal_VirtualTemperature::Cal_VirtualTemperature( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), specifichumidityvariable_(options.SpecificHumidityVariable), temperaturevariable_(options.TemperatureVariable), diff --git a/src/ufo/variabletransforms/Cal_Humidity.h b/src/ufo/variabletransforms/Cal_Humidity.h index 268115267..754493ed7 100644 --- a/src/ufo/variabletransforms/Cal_Humidity.h +++ b/src/ufo/variabletransforms/Cal_Humidity.h @@ -79,8 +79,8 @@ class Cal_RelativeHumidity : public TransformBase { Cal_RelativeHumidity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -130,8 +130,8 @@ class Cal_SpecificHumidity : public TransformBase { Cal_SpecificHumidity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; @@ -172,8 +172,8 @@ class Cal_VirtualTemperature : public TransformBase { Cal_VirtualTemperature(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc index 18a5a3188..80ac71800 100644 --- a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc +++ b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.cc @@ -26,8 +26,8 @@ makerCal_IceThicknessFromFreeboard_("Calculate iceThickness from seaIceFreeboard Cal_IceThicknessFromFreeboard::Cal_IceThicknessFromFreeboard( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), options_(options) {} // ----------------------------------------------------------------------------- diff --git a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h index 080ae9ee8..5490a0617 100644 --- a/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h +++ b/src/ufo/variabletransforms/Cal_IceThicknessFromFreeboard.h @@ -135,8 +135,8 @@ class Cal_IceThicknessFromFreeboard : public TransformBase { typedef Cal_IceThicknessFromFreeboardParameters Parameters_; Cal_IceThicknessFromFreeboard( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_Logarithm.cc b/src/ufo/variabletransforms/Cal_Logarithm.cc index fa76c1238..16516173f 100644 --- a/src/ufo/variabletransforms/Cal_Logarithm.cc +++ b/src/ufo/variabletransforms/Cal_Logarithm.cc @@ -18,8 +18,8 @@ static TransformMaker makerCal_Logarithm_("Logarithm"); Cal_Logarithm::Cal_Logarithm( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), variable_(options.variable), group_(options.group) { diff --git a/src/ufo/variabletransforms/Cal_Logarithm.h b/src/ufo/variabletransforms/Cal_Logarithm.h index 53ffb6e4a..306cdecd0 100644 --- a/src/ufo/variabletransforms/Cal_Logarithm.h +++ b/src/ufo/variabletransforms/Cal_Logarithm.h @@ -47,8 +47,8 @@ class Cal_Logarithm : public TransformBase { typedef Cal_LogarithmParameters Parameters_; Cal_Logarithm(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_PStar.cc b/src/ufo/variabletransforms/Cal_PStar.cc index b5c70184e..d9ad4ca4f 100644 --- a/src/ufo/variabletransforms/Cal_PStar.cc +++ b/src/ufo/variabletransforms/Cal_PStar.cc @@ -23,8 +23,8 @@ static TransformMaker Cal_PStar::Cal_PStar( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { gvals_ += Variable("surf_param_a@GeoVaLs"); gvals_ += Variable("surf_param_b@GeoVaLs"); diff --git a/src/ufo/variabletransforms/Cal_PStar.h b/src/ufo/variabletransforms/Cal_PStar.h index 3e90482a5..88752a071 100644 --- a/src/ufo/variabletransforms/Cal_PStar.h +++ b/src/ufo/variabletransforms/Cal_PStar.h @@ -44,8 +44,8 @@ class Cal_PStar : public TransformBase { public: Cal_PStar(const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_PotentialTFromT.cc b/src/ufo/variabletransforms/Cal_PotentialTFromT.cc index 30337a1f3..28cfc7953 100644 --- a/src/ufo/variabletransforms/Cal_PotentialTFromT.cc +++ b/src/ufo/variabletransforms/Cal_PotentialTFromT.cc @@ -22,8 +22,8 @@ static TransformMaker Cal_PotentialTFromT::Cal_PotentialTFromT( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), pressurevariable_(options.PressureVariable), pressuregroup_(options.PressureGroup), diff --git a/src/ufo/variabletransforms/Cal_PotentialTFromT.h b/src/ufo/variabletransforms/Cal_PotentialTFromT.h index 14590f79f..e056d8f36 100644 --- a/src/ufo/variabletransforms/Cal_PotentialTFromT.h +++ b/src/ufo/variabletransforms/Cal_PotentialTFromT.h @@ -59,8 +59,8 @@ class Cal_PotentialTFromT : public TransformBase { Cal_PotentialTFromT(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc index ee9e002cf..fe891bffe 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.cc +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.cc @@ -18,8 +18,8 @@ static TransformMaker Cal_PressureFromHeightForProfile::Cal_PressureFromHeightForProfile( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), pressureCoord_(options.PressureCoord), @@ -236,8 +236,8 @@ static TransformMaker Cal_PressureFromHeightForICAO::Cal_PressureFromHeightForICAO( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), pressureCoord_(options.PressureCoord), diff --git a/src/ufo/variabletransforms/Cal_PressureFromHeight.h b/src/ufo/variabletransforms/Cal_PressureFromHeight.h index 3acc48a0a..d63ae169c 100644 --- a/src/ufo/variabletransforms/Cal_PressureFromHeight.h +++ b/src/ufo/variabletransforms/Cal_PressureFromHeight.h @@ -48,8 +48,8 @@ class Cal_PressureFromHeightForProfile : public TransformBase { Cal_PressureFromHeightForProfile(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -80,8 +80,8 @@ class Cal_PressureFromHeightForICAO : public TransformBase { Cal_PressureFromHeightForICAO(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run check void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc index df58c73ac..11d4c0d96 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.cc @@ -20,8 +20,8 @@ makerCal_ProfileHorizontalDrift_("ProfileHorizontalDrift"); Cal_ProfileHorizontalDrift::Cal_ProfileHorizontalDrift (const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), heightCoord_(options.HeightCoord), keep_in_window_(options.keep_in_window), diff --git a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h index 0d9ef53da..dd39a0c7c 100644 --- a/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h +++ b/src/ufo/variabletransforms/Cal_ProfileHorizontalDrift.h @@ -60,8 +60,8 @@ class Cal_ProfileHorizontalDrift : public TransformBase { Cal_ProfileHorizontalDrift(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc index 08ed910ca..44f374697 100644 --- a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc +++ b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.cc @@ -21,8 +21,8 @@ static TransformMaker Cal_QNHtoQFEpressure::Cal_QNHtoQFEpressure( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr) {} /************************************************************************************/ diff --git a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h index bb63ecdd3..dbe0e5638 100644 --- a/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h +++ b/src/ufo/variabletransforms/Cal_QNHtoQFEpressure.h @@ -36,8 +36,8 @@ class Cal_QNHtoQFEpressure : public TransformBase { public: Cal_QNHtoQFEpressure(const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; }; diff --git a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc index c640bfb7b..b239685ee 100644 --- a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc +++ b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.cc @@ -20,8 +20,8 @@ static TransformMaker makerCal_RadarBeamGeometry_("RadarB Cal_RadarBeamGeometry::Cal_RadarBeamGeometry( const Parameters_ & options, const ObsFilterData & data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) + const std::shared_ptr> & flags, + const std::shared_ptr> & obserr) : TransformBase(options, data, flags, obserr), params_(options) {} diff --git a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h index 8c6f4dcbe..3d626cfd5 100644 --- a/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h +++ b/src/ufo/variabletransforms/Cal_RadarBeamGeometry.h @@ -63,8 +63,8 @@ class Cal_RadarBeamGeometry : public TransformBase { Cal_RadarBeamGeometry(const Parameters_ & options, const ObsFilterData & data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + const std::shared_ptr> & flags, + const std::shared_ptr> & obserr); void runTransform(const std::vector &apply) override; private: diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc index 5025c7382..36bd3afe7 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.cc +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.cc @@ -19,8 +19,8 @@ static TransformMaker Cal_RemapScanPosition::Cal_RemapScanPosition( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options) {} /************************************************************************************/ diff --git a/src/ufo/variabletransforms/Cal_RemapScanPosition.h b/src/ufo/variabletransforms/Cal_RemapScanPosition.h index be06f3cd2..e97f86a16 100644 --- a/src/ufo/variabletransforms/Cal_RemapScanPosition.h +++ b/src/ufo/variabletransforms/Cal_RemapScanPosition.h @@ -52,8 +52,8 @@ class Cal_RemapScanPosition : public TransformBase { Cal_RemapScanPosition(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; private: diff --git a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc index 6474f6507..1774b9009 100644 --- a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc +++ b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.cc @@ -25,8 +25,8 @@ static TransformMaker Cal_SatBrightnessTempFromRad::Cal_SatBrightnessTempFromRad( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_(), channels_(parameters_.transformVariable.value().channels()) { variables_ += parameters_.transformVariable.value(); diff --git a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h index 22f83a204..46ef1c54d 100644 --- a/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h +++ b/src/ufo/variabletransforms/Cal_SatBrightnessTempFromRad.h @@ -127,8 +127,8 @@ class Cal_SatBrightnessTempFromRad : public TransformBase { Cal_SatBrightnessTempFromRad(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc index 730ebf364..6a495d685 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.cc @@ -29,8 +29,8 @@ static TransformMaker Cal_SatRadianceFromPCScores::Cal_SatRadianceFromPCScores( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.pcVariable.value()}), channels_(parameters_.destinationVariable.value().channels()) { @@ -128,7 +128,7 @@ void Cal_SatRadianceFromPCScores::runTransform(const std::vector &apply) { // multiplying also by a scaling factor to produce radiances in W m^-2 sr^-1 (m^-1)^-1 float scalingFactor = parameters_.scalingFactor.value(); Eigen::MatrixXf reconMatrix \ - = reconstructor(Eigen::placeholders::all, destinationChannelIndex).transpose() * pcScoresMatrix; + = reconstructor(Eigen::all, destinationChannelIndex).transpose() * pcScoresMatrix; reconMatrix *= scalingFactor; // Loop over channels and obs locations to populate derived observation variable diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h index 1eeda28d8..e4481e60e 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromPCScores.h @@ -126,9 +126,9 @@ class Cal_SatRadianceFromPCScores : public TransformBase { typedef Eigen::Matrix rowMajorMatrix; Cal_SatRadianceFromPCScores(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc index c0156b0e9..7257aa5b7 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.cc @@ -25,8 +25,8 @@ static TransformMaker Cal_SatRadianceFromScaledRadiance::Cal_SatRadianceFromScaledRadiance( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.transformVariable.value()}), channels_(parameters_.transformVariable.value().channels()) { @@ -105,8 +105,8 @@ void Cal_SatRadianceFromScaledRadiance::runTransform(const std::vector &ap if (radiance[ichan][iloc] != missingValueFloat && radiance[ichan][iloc] > 0.0f) { for (size_t iscale = 0; iscale < numScaleFactors; ++iscale) { - if ( (channels_[ichan] >= startChannelScale[iscale]) && - (channels_[ichan] <= endChannelScale[iscale]) ) { + if (channels_[ichan] >= startChannelScale[iscale] & + channels_[ichan] <= endChannelScale[iscale]) { radiance[ichan][iloc] *= std::pow(10, (-1.0f*channelScaleFactor[iscale])); break; } // if channels_ diff --git a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h index 741c3c2a6..10d3e46b8 100644 --- a/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h +++ b/src/ufo/variabletransforms/Cal_SatRadianceFromScaledRadiance.h @@ -86,9 +86,9 @@ class Cal_SatRadianceFromScaledRadiance : public TransformBase { typedef Cal_SatRadianceFromScaledRadianceParameters Parameters_; Cal_SatRadianceFromScaledRadiance(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc index cf5ec4aa4..75a560374 100644 --- a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc +++ b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.cc @@ -24,8 +24,8 @@ static TransformMaker Cal_SatZenithAngleCorrection::Cal_SatZenithAngleCorrection( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), parameters_(options), variables_({parameters_.transformVariable.value()}) { ASSERT(parameters_.coeffA.value().size() == parameters_.transformVariable.value().size()); diff --git a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h index 8a7f2f7ef..9f36f59db 100644 --- a/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h +++ b/src/ufo/variabletransforms/Cal_SatZenithAngleCorrection.h @@ -92,8 +92,8 @@ class Cal_SatZenithAngleCorrection : public TransformBase { Cal_SatZenithAngleCorrection(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return variables_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc index 48c154fd5..f8548b89d 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.cc @@ -27,8 +27,8 @@ namespace ufo { Cal_SurfaceWindScalingCombined::Cal_SurfaceWindScalingCombined( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_SurfaceWindScalingCombined::Constructor start" << std::endl; oops::Log::trace() << "Cal_SurfaceWindScalingCombined::Constructor done" << std::endl; diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h index fcb5ea266..3fd418280 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingCombined.h @@ -31,9 +31,9 @@ namespace ufo { class Cal_SurfaceWindScalingCombined : public TransformBase { public: Cal_SurfaceWindScalingCombined(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc index ddeeff99d..384d05af2 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_SurfaceWindScalingHeight::Cal_SurfaceWindScalingHeight( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_(), heightVariableGroup_(options.heightVariableGroup), heightVariableName_(options.heightVariableName) diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h index 262193b8b..a24c21b02 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingHeight.h @@ -47,8 +47,8 @@ class Cal_SurfaceWindScalingHeightParameters: public VariableTransformParameters typedef Cal_SurfaceWindScalingHeightParameters Parameters_; Cal_SurfaceWindScalingHeight(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc index d7a173b3e..66a1a1b6d 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.cc @@ -28,8 +28,8 @@ namespace ufo { Cal_SurfaceWindScalingPressure::Cal_SurfaceWindScalingPressure( const GenericVariableTransformParameters &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) : + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), gvals_() { oops::Log::trace() << "Cal_SurfaceWindScalingPressure::Constructor start" << std::endl; // List of GeoVaLs this transform will need access to diff --git a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h index 81d49f4b2..5e91509ac 100644 --- a/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h +++ b/src/ufo/variabletransforms/Cal_SurfaceWindScalingPressure.h @@ -31,9 +31,9 @@ namespace ufo { class Cal_SurfaceWindScalingPressure : public TransformBase { public: Cal_SurfaceWindScalingPressure(const GenericVariableTransformParameters &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; Variables requiredVariables() const override { return gvals_; } diff --git a/src/ufo/variabletransforms/Cal_Wind.cc b/src/ufo/variabletransforms/Cal_Wind.cc index d6451007a..7db431d4e 100644 --- a/src/ufo/variabletransforms/Cal_Wind.cc +++ b/src/ufo/variabletransforms/Cal_Wind.cc @@ -23,8 +23,8 @@ static TransformMaker Cal_WindSpeedAndDirection::Cal_WindSpeedAndDirection( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), group_(options.group), eastwardwindvariable_(options.EastwardWindVariable), @@ -89,8 +89,8 @@ static TransformMaker Cal_WindComponents::Cal_WindComponents( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), group_(options.group), windspeedvariable_(options.WindSpeedVariable), diff --git a/src/ufo/variabletransforms/Cal_Wind.h b/src/ufo/variabletransforms/Cal_Wind.h index c3820cb5b..4660d0d3b 100644 --- a/src/ufo/variabletransforms/Cal_Wind.h +++ b/src/ufo/variabletransforms/Cal_Wind.h @@ -60,8 +60,8 @@ class Cal_WindSpeedAndDirection : public TransformBase { Cal_WindSpeedAndDirection(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; @@ -90,16 +90,16 @@ class Cal_WindComponents : public TransformBase { Cal_WindComponents(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; private: - /// Group name. - std::string group_; std::string windspeedvariable_; std::string winddirectionvariable_; + /// Group name. + std::string group_; }; } // namespace ufo diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc b/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc index 828876497..ffb33c756 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanDensity.cc @@ -23,8 +23,8 @@ static TransformMaker OceanDensity::OceanDensity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDensity.h b/src/ufo/variabletransforms/OceanConversions/OceanDensity.h index fac206d66..29037c9ae 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDensity.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanDensity.h @@ -77,8 +77,8 @@ class OceanDensity : public TransformBase { typedef OceanDensityParameters Parameters_; OceanDensity(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc index 7cbdf10ff..6f6db5842 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.cc @@ -23,8 +23,8 @@ static TransformMaker OceanDepthToPressure::OceanDepthToPressure( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), depthvariable_(options.DepthVariable), depthgroup_(options.DepthGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h index 3f2ccadb9..40ea94a9d 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanDepthToPressure.h @@ -64,8 +64,8 @@ class OceanDepthToPressure : public TransformBase { typedef OceanDepthToPressureParameters Parameters_; OceanDepthToPressure(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc index 0950fdfd5..83df41237 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.cc @@ -23,8 +23,8 @@ static TransformMaker OceanPracticalSalinityToAbsoluteSalinity::OceanPracticalSalinityToAbsoluteSalinity( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), absolutesalinityvariable_(options.AbsoluteSalinityVariable), practicalsalinityvariable_(options.PracticalSalinityVariable), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h index 2403a77b9..0ab5039ff 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanPracticalSalinityToAbsoluteSalinity.h @@ -74,9 +74,9 @@ class OceanPracticalSalinityToAbsoluteSalinity : public TransformBase { public: typedef OceanPracticalSalinityToAbsoluteSalinityParameters Parameters_; OceanPracticalSalinityToAbsoluteSalinity(const Parameters_ &options, - const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const ObsFilterData &data, + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc index cef5da03e..8d168be7b 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.cc @@ -23,8 +23,8 @@ static TransformMaker OceanTempToConservativeTemp::OceanTempToConservativeTemp( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h index 1074b91ce..31a69fc70 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToConservativeTemp.h @@ -79,8 +79,8 @@ class OceanTempToConservativeTemp : public TransformBase { typedef OceanTempToConservativeTempParameters Parameters_; OceanTempToConservativeTemp(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc index 3b0cfbee5..99ef95bdd 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.cc @@ -23,8 +23,8 @@ static TransformMaker OceanTempToTheta::OceanTempToTheta( const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) : TransformBase(options, data, flags, obserr), salinityvariable_(options.SalinityVariable), salinitygroup_(options.SalinityGroup), diff --git a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h index 73539b16a..9d8dbcb18 100644 --- a/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h +++ b/src/ufo/variabletransforms/OceanConversions/OceanTempToTheta.h @@ -79,8 +79,8 @@ class OceanTempToTheta : public TransformBase { typedef OceanTempToThetaParameters Parameters_; OceanTempToTheta(const Parameters_ &options, const ObsFilterData &data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr); + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr); // Run variable conversion void runTransform(const std::vector &apply) override; diff --git a/src/ufo/variabletransforms/TransformBase.cc b/src/ufo/variabletransforms/TransformBase.cc index ea049dd8c..613b0bae4 100644 --- a/src/ufo/variabletransforms/TransformBase.cc +++ b/src/ufo/variabletransforms/TransformBase.cc @@ -14,9 +14,9 @@ namespace ufo { TransformBase::TransformBase(const VariableTransformParametersBase& options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) - : options_(options), data_(data) , flags_(flags), obserr_(obserr) { + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr) + : options_(options), data_(data) , flags_(*flags), obserr_(*obserr) { method_ = formulas::resolveMethods(options.Method.value()); UseValidDataOnly_ = options.UseValidDataOnly.value(); obsName_ = data_.obsspace().obsname(); @@ -37,8 +37,8 @@ TransformFactory::TransformFactory(const std::string& name) { std::unique_ptr TransformFactory::create( const std::string& name, const VariableTransformParametersBase& options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr) { + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr) { oops::Log::trace() << " --> TransformFactory::create" << std::endl; oops::Log::trace() << " --> name: " << name << std::endl; diff --git a/src/ufo/variabletransforms/TransformBase.h b/src/ufo/variabletransforms/TransformBase.h index 90ffb2847..e00b728e5 100644 --- a/src/ufo/variabletransforms/TransformBase.h +++ b/src/ufo/variabletransforms/TransformBase.h @@ -54,8 +54,8 @@ class TransformBase { public: TransformBase(const VariableTransformParametersBase &options, const ObsFilterData& data, - ioda::ObsDataVector & flags, - ioda::ObsDataVector & obserr); + const std::shared_ptr>& flags, + const std::shared_ptr>& obserr); /// Destructor virtual ~TransformBase() {} /// Run variable conversion @@ -141,14 +141,15 @@ class TransformBase { // Update QC flags to account for values that were previously missing but // are now present (or vice versa). if (flags_.has(varName)) { - ASSERT(flags_[varName].size() == outputObsVector.size()); + std::vector &varFlags = flags_[varName]; + ASSERT(varFlags.size() == outputObsVector.size()); const T missing = util::missingValue(); for (size_t iloc = 0; iloc < outputObsVector.size(); ++iloc) { - if (flags_[varName][iloc] == QCflags::missing && outputObsVector[iloc] != missing) - flags_[varName][iloc] = QCflags::pass; - else if (flags_[varName][iloc] == QCflags::pass && outputObsVector[iloc] == missing) - flags_[varName][iloc] = QCflags::missing; + if (varFlags[iloc] == QCflags::missing && outputObsVector[iloc] != missing) + varFlags[iloc] = QCflags::pass; + else if (varFlags[iloc] == QCflags::pass && outputObsVector[iloc] == missing) + varFlags[iloc] = QCflags::missing; } } } @@ -169,16 +170,15 @@ class TransformBase { const std::vector & dimList, const std::string &outputTag = "DerivedObsValue") { std::vector outputObsVector(obsVector); - const std::string fullVarName = varName + "_" + channel; // Fill in missing values with values of non-derived group if (options_.FillMissingDerivedFromOriginal && outputTag.substr(0, 7) == "Derived") { std::string originalTag = outputTag; originalTag.erase(0, 7); // Remove Derived from group name - if (obsdb_.has(originalTag, fullVarName)) { + if (obsdb_.has(originalTag, varName + "_" + channel)) { std::vector originalValues(obsdb_.nlocs()); // Need to skipDerived so we pickup original ObsValues - obsdb_.get_db(originalTag, fullVarName, originalValues, {}, true); + obsdb_.get_db(originalTag, varName + "_" + channel, originalValues, {}, true); const T missing = util::missingValue(); for (size_t jloc = 0; jloc < obsdb_.nlocs(); ++jloc) { if (outputObsVector[jloc] == missing && @@ -188,16 +188,17 @@ class TransformBase { } } } - obsdb_.put_db(outputTag, fullVarName, outputObsVector, dimList); - if (flags_.has(fullVarName)) { - ASSERT(flags_[fullVarName].size() == outputObsVector.size()); + obsdb_.put_db(outputTag, varName + "_" + channel, outputObsVector, dimList); + if (flags_.has(varName + "_" + channel)) { + std::vector &varFlags = flags_[varName + "_" + channel]; + ASSERT(varFlags.size() == outputObsVector.size()); const T missing = util::missingValue(); for (size_t iloc = 0; iloc < outputObsVector.size(); ++iloc) { - if (flags_[fullVarName][iloc] == QCflags::missing && outputObsVector[iloc] != missing) - flags_[fullVarName][iloc] = QCflags::pass; - else if (flags_[fullVarName][iloc] == QCflags::pass && outputObsVector[iloc] == missing) - flags_[fullVarName][iloc] = QCflags::missing; + if (varFlags[iloc] == QCflags::missing && outputObsVector[iloc] != missing) + varFlags[iloc] = QCflags::pass; + else if (varFlags[iloc] == QCflags::pass && outputObsVector[iloc] == missing) + varFlags[iloc] = QCflags::missing; } } } @@ -231,8 +232,8 @@ class TransformFactory { static std::unique_ptr create( const std::string &, const VariableTransformParametersBase &, const ObsFilterData &, - ioda::ObsDataVector &, - ioda::ObsDataVector &); + const std::shared_ptr> &, + const std::shared_ptr> &); static std::unique_ptr createParameters( const std::string &); virtual ~TransformFactory() = default; @@ -244,8 +245,8 @@ class TransformFactory { virtual std::unique_ptr make( const VariableTransformParametersBase &, const ObsFilterData &, - ioda::ObsDataVector &, - ioda::ObsDataVector &) = 0; + const std::shared_ptr> &, + const std::shared_ptr> &) = 0; virtual std::unique_ptr makeParameters() const = 0; static std::map &getMakers() { static std::map makers_; @@ -266,8 +267,8 @@ class TransformMaker : public TransformFactory { std::unique_ptr make( const VariableTransformParametersBase &options, const ObsFilterData& data, - ioda::ObsDataVector &flags, - ioda::ObsDataVector &obserr) override { + const std::shared_ptr> &flags, + const std::shared_ptr> &obserr) override { const auto &stronglyTypedParams = dynamic_cast(options); return std::unique_ptr(new T(stronglyTypedParams, data, flags, obserr)); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca8c69c78..d08963a4b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -146,13 +146,11 @@ list( APPEND ufo_test_input testinput/ufo_geovals_set_invalid_default_format.yaml testinput/ufo_geovals_get_nonexistent_var.yaml testinput/variables.yaml - testinput/unit_tests/obslocalization/obs_localization.yaml ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput/unit_tests) -file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput/unit_tests/obslocalization) CREATE_SYMLINK( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${ufo_test_input} ) # Create Data directory for test output files @@ -364,14 +362,6 @@ ecbuild_add_test( TARGET ufo_sampled_locations ENVIRONMENT OOPS_TRAPFPE=1 LIBS ufo ) -# Test obs-space localization computeLocalization(Point3, Point3) for Box car, GC99, SOAR -ecbuild_add_test( TARGET ufo_obs_localization - SOURCES mains/TestObsLocalization.cc - ufo/ObsLocalization.h - ARGS "testinput/unit_tests/obslocalization/obs_localization.yaml" - ENVIRONMENT OOPS_TRAPFPE=1 - LIBS ufo ) - ecbuild_add_test( TARGET ufo_recordhandler COMMAND ${CMAKE_BINARY_DIR}/bin/test_RecordHandler.x ARGS "testinput/recordhandler.yaml" @@ -702,7 +692,7 @@ if( ${rttov_FOUND} ) set(RTTOV_VERSION_SUFFIX "_rttov12") set(RTTOV_CAMEL_VERSION "2") endif() - + list( APPEND rttov_coef_data rttov7pred54L/rtcoef_dmsp_17_ssmis.dat rttov7pred54L/rtcoef_gcom-w_1_amsr2.dat @@ -732,7 +722,7 @@ if( ${rttov_FOUND} ) pchsr_v9.3.${RTTOV_COEF_EXT} ) - + # Set RTTOV coefficient directory if(DEFINED ENV{RTTOV_LOCAL_COEF_DIR} AND EXISTS $ENV{RTTOV_LOCAL_COEF_DIR}) set(RTTOV_COEF_DIR "$ENV{RTTOV_LOCAL_COEF_DIR}") @@ -751,7 +741,7 @@ if( ${rttov_FOUND} ) set(RTTOV_EMIS_ATLAS_DIR "${rttov_BASE_DIR}/etc/emis_data") endif() message(STATUS "RTTOV emissivity atlas directory: ${RTTOV_EMIS_ATLAS_DIR}") - + # Handle mietable (v12) or hydrotable (v14) if(rttov_VERSION VERSION_GREATER_EQUAL 14.0) # RTTOV v14: use NetCDF hydrotable file directly @@ -780,7 +770,7 @@ if( ${rttov_FOUND} ) list(APPEND rttov_coef_data "mietable/mietable_metop_amsuamhs.bin") endif() endif() - + # Symlink RTcoef files create_symlink_filename( ${RTTOV_COEF_DIR} ${CMAKE_CURRENT_BINARY_DIR}/Data diff --git a/test/mains/TestObsLocalization.cc b/test/mains/TestObsLocalization.cc deleted file mode 100644 index 8540f6af4..000000000 --- a/test/mains/TestObsLocalization.cc +++ /dev/null @@ -1,15 +0,0 @@ -/* - * (C) Copyright 2026 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "../ufo/ObsLocalization.h" -#include "oops/runs/Run.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - ufo::test::ObsLocalization tests; - return run.execute(tests); -} diff --git a/test/testinput/instrumentTests/amsua/CMakeLists.txt b/test/testinput/instrumentTests/amsua/CMakeLists.txt index e0b98522a..5dac1c9ef 100644 --- a/test/testinput/instrumentTests/amsua/CMakeLists.txt +++ b/test/testinput/instrumentTests/amsua/CMakeLists.txt @@ -117,6 +117,16 @@ if( crtm_FOUND ) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ LIBS ufo ) + # test matching GSI with small tolerance, is this still valid test consider update or remove + ufo_add_test( NAME instrument_amsua_n18_gfs_HofX_bc + ECBUILD + COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x + ARGS "${CMAKE_CURRENT_SOURCE_DIR}/amsua_n18_gfs_HofX_bc.yaml" + MPI 1 + LABELS crtm amsua n18 bc instrument ci_flake_disable tier2 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ + LIBS ufo ) + ufo_add_test( NAME instrument_amsua_n19_gfs_HofX_bc ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperator.x diff --git a/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml new file mode 100644 index 000000000..c71616147 --- /dev/null +++ b/test/testinput/instrumentTests/amsua/amsua_n18_gfs_HofX_bc.yaml @@ -0,0 +1,44 @@ +time window: + begin: 2020-11-01T09:00:00Z + end: 2020-11-01T15:00:00Z + +observations: +- obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: Data/ + obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_obs_2020110112_m.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 + geovals: + filename: Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_geoval_2020110112_m.nc4 + obs bias: + input file: Data/ufo/testinput_tier_1/instruments/radiance/satbias_amsua_n18.2020110106.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &amsua_n18_tlap Data/ufo/testinput_tier_1/instruments/radiance/amsua_n18_tlapmean.txt + - name: lapseRate + tlapse: *amsua_n18_tlap + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + vector ref: GsiHofXBc + tolerance: 1.e-7 diff --git a/test/testinput/unit_tests/errors/CMakeLists.txt b/test/testinput/unit_tests/errors/CMakeLists.txt index 50011de07..4a32d2c5e 100644 --- a/test/testinput/unit_tests/errors/CMakeLists.txt +++ b/test/testinput/unit_tests/errors/CMakeLists.txt @@ -51,7 +51,7 @@ ufo_add_test( NAME obserrordiffusion ARGS "${CMAKE_CURRENT_SOURCE_DIR}/obserrordiffusion.yaml" MPI 1 LIBS ufo - LABELS errors + LABELS errors ci_flake_disable WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) ufo_add_test( NAME obserror_dirac_diffusion @@ -110,13 +110,3 @@ ufo_add_test( NAME reconditioning_mpi LIBS ufo LABELS errors WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) - -ufo_add_test( NAME reconditioning_within_group - TIER 1 - ECBUILD - SOURCES ../../../mains/TestReconditioning.cc - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/reconditioning_withinGroup.yaml" - MPI 1 - LIBS ufo - LABELS errors - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) diff --git a/test/testinput/unit_tests/errors/diracDiffusion.yaml b/test/testinput/unit_tests/errors/diracDiffusion.yaml index f9af66977..6879f7989 100644 --- a/test/testinput/unit_tests/errors/diracDiffusion.yaml +++ b/test/testinput/unit_tests/errors/diracDiffusion.yaml @@ -20,10 +20,6 @@ observations: correlation variable names: [brightnessTemperature] correlation lengthscale: 200000. # meters normalization iterations: 10000 - control grid: - grid spacing in degrees: 10 # control grid resolution in degrees - remove within meters: 200000 # remove control points within this distance from observations (meters) - # use lengthscale plus some padding to impact mesh not diffusion test: reference filename: testref/unit_tests/errors/diracDiffusion.ref diff --git a/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml b/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml index 58b13ed57..12821cad5 100644 --- a/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml +++ b/test/testinput/unit_tests/errors/obserrorcrossvarcorr.yaml @@ -14,7 +14,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, two vars) @@ -27,7 +27,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, three vars) @@ -40,7 +40,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, four vars, one is missing from the corr file) @@ -53,7 +53,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: Sondes (cross-var corr, four vars, one is missing from the corr file) @@ -66,7 +66,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_3var.nc4 obs error test: - test localization: true + test localization: false - obs space: name: testReader (cross-var corr with four vars, three vars simulated) @@ -83,7 +83,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [0.1, 1, 0.15] @@ -102,7 +102,7 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [0.2,0.15, 1, 0.7, 0] @@ -121,6 +121,6 @@ observations: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/obserror_corr_test.nc4 obs error test: - test localization: true + test localization: false test reader: true reference: [1] diff --git a/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml b/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml index ae3eefdf8..bd518758e 100644 --- a/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml +++ b/test/testinput/unit_tests/errors/obserrorcrossvarcov.yaml @@ -1,6 +1,6 @@ time window: - begin: 2020-11-01T06:00:00Z - end: 2020-11-01T18:00:00Z + begin: 2018-04-14T12:00:00Z + end: 2018-04-15T12:00:00Z observations: @@ -67,9 +67,8 @@ observations: obs error: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/Rcov_iasi_metop-b_sea.nc4 - Obs Error test tolerance: 1e-9 obs error test: - test localization: true + test localization: false - obs space: name: iasi_metop-b (channels == all channels in the correlations file) @@ -98,6 +97,5 @@ observations: obs error: covariance model: cross variable covariances input file: Data/ufo/testinput_tier_1/Rcov_iasi_metop-b_sea.nc4 - Obs Error test tolerance: 1e-9 obs error test: - test localization: true + test localization: false diff --git a/test/testinput/unit_tests/errors/obserrordiffusion.yaml b/test/testinput/unit_tests/errors/obserrordiffusion.yaml index e0b519d68..727879130 100644 --- a/test/testinput/unit_tests/errors/obserrordiffusion.yaml +++ b/test/testinput/unit_tests/errors/obserrordiffusion.yaml @@ -17,9 +17,5 @@ observations: correlation lengthscale: 5000. normalization iterations: 1000 Obs Error test tolerance: 1.e-8 - control grid: - grid spacing in degrees: 10 # control grid resolution in degrees - remove within meters: 200000 # remove control points within this distance from observations (meters) - # use lengthscale plus some padding to impact mesh not diffusion obs error test: test localization: false diff --git a/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml b/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml index 8a978c690..93434edf5 100644 --- a/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml +++ b/test/testinput/unit_tests/errors/obserrorwithingroupcorr.yaml @@ -59,23 +59,3 @@ observations: correlation lengthscale: 13000. # in m obs error test: test localization: false - -- obs space: - name: Sondes (within group covariances with gaussian correlation function and basic reconditioning) - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 - obsgrouping: - group variables: [sequenceNumber] - sort variable: pressure - sort order: ascending - simulated variables: [airTemperature, windEastward, windNorthward] - obs error: - covariance model: within group covariances - distance function: haversine - correlation function: gaussian - apply basic reconditioning: true - correlation lengthscale: 1400. # in m - obs error test: - test localization: false diff --git a/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml b/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml deleted file mode 100644 index b22878fc1..000000000 --- a/test/testinput/unit_tests/errors/reconditioning_withinGroup.yaml +++ /dev/null @@ -1,27 +0,0 @@ -time window: - begin: 2018-04-14T21:00:00Z - end: 2018-04-15T03:00:00Z - -observations: -- obs space: - name: Sondes (within group covariances using haversine distance and gaussian correlation function w/ advanced reconditioner) - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/sondes_obs_2018041500_s.nc4 - obsgrouping: - group variables: [sequenceNumber] - sort variable: pressure - sort order: ascending - simulated variables: [airTemperature, windEastward, windNorthward] - obs error: - covariance model: within group covariances - distance function: haversine - correlation function: gaussian - correlation lengthscale: 2000. # in m - reconditioning: - recondition method: Ridge Regression - fraction: 0.9 - obs error test: - test reader: true - reference: [322.344,391.1,459.624,444.458,424.292,669.169,627.638,229.982,230.075,213.214,272.439,212.957,369.691,369.511,49.9851,2.77498,71.3079,-29.7157,254.176,71.4432,601.438,-61.1725,629.321,-66.1769,571.344,49.2934,230.431,72.1513,873.28,75.4791,831.886,71.9031,452.113,-62.8127,119.552,263.853,119.597,266.688,119.389,268.971,112.644,262.727,245.007,74.1117,266.9,78.3315,334.494,88.8221,306.8,85.1776,550.066,-33.0129,364.036,90.9901,594.635,-35.2926,408.446,17.6729,387.861,91.463,475.366,-3.80788,473.885,-2.95747,451.963,-20.5918,74.6161,269.549,410.318,-16.7671,449.217,-10.0663,343.009,5.49996,432.602,-23.5831,64.7918,261.927,518.162,68.7194,491.718,-1.27451,508.145,0.949498,533.051,61.7235,562.933,27.8447,561.976,38.6925,40.6551,274.177,32.1251,264.328,24.5098,267.456,20.9273,269.011,8.72175,273.495,3.45904,274.946,-47.1277,272.72,-55.289,273.7,-152.623,253.867,-161.42,252.169,-183.645,230.035,-177.844,215.183,-185.375,211,-176.831,196.776,-179.829,193.691,-180.072,193.4,-189.709,168.838,-189.879,166.643,-190.467,157.146,-179.252,130.908,-173.781,120.216,-175.561,122.31,-165.76,112.216,-160.996,107.915,-155.535,103.262,-150.172,99.0564,-138.639,90.9216,-137.471,90.1345] diff --git a/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt b/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt index e29d9770b..5dd4a69cf 100644 --- a/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt +++ b/test/testinput/unit_tests/filters/obsfunctions/CMakeLists.txt @@ -53,15 +53,6 @@ ufo_add_test( NAME function_arithmetic LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) # -ufo_add_test( NAME function_circulardifference - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_circulardifference.yaml" - MPI 1 - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# ufo_add_test( NAME function_aircraft_bc_offline ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x @@ -846,7 +837,7 @@ ufo_add_test( NAME function_solarzenith_skiprejected LIBS ufo LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) - + # Test for the string manipulation obsfucntion ufo_add_test( NAME function_stringmanipulation ECBUILD @@ -855,8 +846,8 @@ ufo_add_test( NAME function_stringmanipulation MPI 1 LIBS ufo LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) - + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) + # ufo_add_test( NAME function_surfacecloudmodellevelcbh ECBUILD @@ -884,24 +875,7 @@ ufo_add_test( NAME function_geocloudcreatecloudcolumn LIBS ufo LABELS filters obsfunctions WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# -ufo_add_test( NAME function_timebinner - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_timebinner.yaml" - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# -ufo_add_test( NAME function_timebinner_mpi - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/function_timebinner.yaml" - MPI 3 - LIBS ufo - LABELS filters obsfunctions - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../../ ) -# + ufo_add_test( NAME function_tropo_estimate ECBUILD COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsFilters.x diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml index 54854bdc5..b9d1dd215 100644 --- a/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml +++ b/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml @@ -74,7 +74,7 @@ observations: variable: name: QCflagsData/bendingAngle channels: *all_levels - testValue: 30 + testValue: 13 assignEqual: 0.9 assignNotEqual: 0.2 # diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml deleted file mode 100644 index 0ce11fa88..000000000 --- a/test/testinput/unit_tests/filters/obsfunctions/function_circulardifference.yaml +++ /dev/null @@ -1,47 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2030-01-01T00:00:00Z - -observations: - - obs space: - name: Test ObsFunction CircularDifference - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/circular_difference_test_obs.nc4 - simulated variables: [] - obs filters: - # signed between windDirectionA and windDirectionB (windDirectionB - windDirectionA) - - filter: Variable Assignment - assignments: - - name: MetaData/signedWindDirectionCircularDiff - type: float - function: - name: ObsFunction/CircularDifference - options: - variable start: ObsValue/windDirectionA - variable end: ObsValue/windDirectionB - signed: true - circular period: 360.0 - # unsigned between windDirectionA and windDirectionB (absolute value of windDirectionB - windDirectionA) - - filter: Variable Assignment - assignments: - - name: MetaData/unsignedWindDirectionCircularDiff - type: float - function: - name: ObsFunction/CircularDifference - options: - variable start: ObsValue/windDirectionA - variable end: ObsValue/windDirectionB - signed: false - circular period: 360.0 - compareVariables: - - test: - name: MetaData/signedWindDirectionCircularDiff - reference: - name: TestReference/signedWindDirectionCircularDiff - relTol: 1e-6 - - test: - name: MetaData/unsignedWindDirectionCircularDiff - reference: - name: TestReference/unsignedWindDirectionCircularDiff diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml deleted file mode 100644 index d22b832b5..000000000 --- a/test/testinput/unit_tests/filters/obsfunctions/function_timebinner.yaml +++ /dev/null @@ -1,208 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2030-01-01T00:00:00Z - -observations: - - obs space: - name: Test ObsFunction TimeBinner - obsdatain: - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/time_binner_test_obs.nc4 - simulated variables: [] - obs filters: - # Note: bin centres are floored from the first earliest timestamp - # globally, so, for 2018-04-17 01:20:02 UTC the first minutely bin is - # centred on 01:20:00 UTC, the first houly bin is centred on 01:00:00 UTC, - # the first daily bin on 00:00:00 UTC, etc. - # If "reverse chronological order: true" then bins are derived from the - # latest timestamp instead. - - # (A) T-30M <= t <= T where T is the top of the hour. dateTimes outside of - # this range get missing bin numbers. "binned timestamp variable" should - # take on the hour value T for all dateTimes in the bin, others get missing. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutHourMinus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT0M # up to the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutHourMinus30MTimestamps # optional - all values in a bin get the same timestamp - timestamps outside of bin are set to missing - # (B) As above, but for T-30M <= t <= T+PT29M59S (no missing values - # expected since OOPS dateTimes are to the nearest second). - - filter: Variable Assignment - assignments: - - name: MetaData/aboutHourMinus30MPlus29M59SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT29M59S # up to 29 minutes 59 seconds after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutHourMinus30MPlus29M59STimestamps - # (C) T < t <= T+PT1H (i.e. the hour after the top of the hour) - use that - # OOPS times are to the nearest second to avoid ambiguity at the hour. - - filter: Variable Assignment - assignments: - - name: MetaData/excludingHourPlus1HourBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: PT1S # from the hour (exclusive) or 1S after the hour (inclusive) - bin window upper bound: PT1H # up to one hour after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/excludingHourPlus1HourBinTimestamps - # (D) Test day interval, i.e. T-12H <= t <= T where T is midnight UTC. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutMidnightMinus12HBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: day - bin window lower bound: -PT12H # 12 hours before midnight (inclusive) - bin window upper bound: PT0M # up to midnight (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutMidnightMinus12HTimestamps - # (E) Test minute interval: T-30S <= t <= T+10S where T is the minute. - - filter: Variable Assignment - assignments: - - name: MetaData/aboutMinuteMinus30SPlus10SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: minute - bin window lower bound: -PT30S # 30 seconds before the minute (inclusive) - bin window upper bound: PT10S # up to 10 seconds after the minute (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/aboutMinuteMinus30SPlus10STimestamps - # (F) Test reverse chronological order: T-30M <= t <= T+PT29M59S where T - # is the hour, but deriving bins from the latest timestamp back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourMinus30MPlus29M59SBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT29M59S # up to 29 minutes 59 seconds after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourMinus30MPlus29M59STimestamps - reverse chronological order: true # default is false - # (G) Test reverse chronological order: T-30M <= t <= T where T is the hour. - # deriving bins from the latest timestamp back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourMinus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: -PT30M # 30 minutes before the hour (inclusive) - bin window upper bound: PT0M # up to the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourMinus30MTimestamps - reverse chronological order: true # default is false - # (H) Test reverse chronological order with opposite offset to (G): - # T <= t <= T+30M where T is the hour, deriving bins from the latest timestamp - # back to earliest. - - filter: Variable Assignment - assignments: - - name: MetaData/reverseAboutHourPlus30MBinNumbers - type: int - function: - name: IntObsFunction/TimeBinner - options: - bin interval unit: hour - bin window lower bound: PT0M # from the hour (inclusive) - bin window upper bound: PT30M # up to 30 minutes after the hour (inclusive) - input timestamp variable: MetaData/dateTime # default - binned timestamp variable: MetaData/reverseAboutHourPlus30MTimestamps - reverse chronological order: true # default is false - compareVariables: - # (A) - - test: - name: MetaData/aboutHourMinus30MBinNumbers - reference: - name: TestReference/aboutHourMinus30MBinNumbers - - test: - name: MetaData/aboutHourMinus30MTimestamps - reference: - name: TestReference/aboutHourMinus30MTimestamps - # (B) - - test: - name: MetaData/aboutHourMinus30MPlus29M59SBinNumbers - reference: - name: TestReference/aboutHourMinus30MPlus29M59SBinNumbers - - test: - name: MetaData/aboutHourMinus30MPlus29M59STimestamps - reference: - name: TestReference/aboutHourMinus30MPlus29M59STimestamps - # (C) - - test: - name: MetaData/excludingHourPlus1HourBinNumbers - reference: - name: TestReference/excludingHourPlus1HourBinNumbers - - test: - name: MetaData/excludingHourPlus1HourBinTimestamps - reference: - name: TestReference/excludingHourPlus1HourBinTimestamps - # (D) - - test: - name: MetaData/aboutMidnightMinus12HBinNumbers - reference: - name: TestReference/aboutMidnightMinus12HBinNumbers - - test: - name: MetaData/aboutMidnightMinus12HTimestamps - reference: - name: TestReference/aboutMidnightMinus12HTimestamps - # (E) - - test: - name: MetaData/aboutMinuteMinus30SPlus10SBinNumbers - reference: - name: TestReference/aboutMinuteMinus30SPlus10SBinNumbers - - test: - name: MetaData/aboutMinuteMinus30SPlus10STimestamps - reference: - name: TestReference/aboutMinuteMinus30SPlus10STimestamps - # (F) - - test: - name: MetaData/reverseAboutHourMinus30MPlus29M59SBinNumbers - reference: - name: TestReference/reverseAboutHourMinus30MPlus29M59SBinNumbers - - test: - name: MetaData/reverseAboutHourMinus30MPlus29M59STimestamps - reference: - name: TestReference/aboutHourMinus30MPlus29M59STimestamps # should be the same as (B) - # (G) - - test: - name: MetaData/reverseAboutHourMinus30MBinNumbers - reference: - name: TestReference/reverseAboutHourMinus30MBinNumbers - - test: - name: MetaData/reverseAboutHourMinus30MTimestamps - reference: - name: TestReference/aboutHourMinus30MTimestamps # should be the same as (A) - # (H) - - test: - name: MetaData/reverseAboutHourPlus30MBinNumbers - reference: - name: TestReference/reverseAboutHourPlus30MBinNumbers - - test: - name: MetaData/reverseAboutHourPlus30MTimestamps - reference: - name: TestReference/reverseAboutHourPlus30MTimestamps # would be the same as non-reverse diff --git a/test/testinput/unit_tests/obslocalization/obs_localization.yaml b/test/testinput/unit_tests/obslocalization/obs_localization.yaml deleted file mode 100644 index 33208b8d8..000000000 --- a/test/testinput/unit_tests/obslocalization/obs_localization.yaml +++ /dev/null @@ -1,78 +0,0 @@ -time window: - begin: 2000-01-01T00:00:00Z - end: 2000-01-01T06:00:00Z - -obs space: - name: simple obs - simulated variables: [windEastward] - obsdatain: - engine: - type: GenList - lons: [10.0, 20.0, 30.0] - lats: [5.0, 15.0, 25.0] - dateTimes: [3600, 3600, 3600] - epoch: "seconds since 2000-01-01T00:00:00Z" - obs errors: [1.0] - -obs localizations: - # Box car: returns 1.0 if distance <= lengthscale, 0.0 otherwise -- localization: - localization method: Horizontal Box car - lengthscale: 1000000.0 # 1000 km - point pair tests: - # Same point -> distance = 0 -> returns 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart (10 deg longitude at equator) -> outside 1000 km -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - # Points ~556 km apart (5 deg longitude at equator) -> inside 1000 km -> 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [5.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - - # GC99: applies Gaspari-Cohn taper; cutoff at lengthscale (r > 1 -> 0) -- localization: - localization method: Horizontal Gaspari-Cohn - lengthscale: 1000000.0 # 1000 km - point pair tests: - # Same point -> distance = 0 -> gc99(0) = 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart -> outside 1000 km cutoff -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - - # SOAR: applies (1 + x)*exp(-x) taper; cutoff at lengthscale -- localization: - localization method: Horizontal SOAR - lengthscale: 1000000.0 # 1000 km - soar horizontal decay: 1.0e-6 # 1/m -> x = distance * decay - point pair tests: - # Same point -> distance = 0 -> soar(0) = (1+0)*exp(0) = 1.0 - - p1: [0.0, 0.0, 0.0] - p2: [0.0, 0.0, 0.0] - reference value: 1.0 - tolerance: 1.0e-10 - # Points ~1111 km apart -> outside 1000 km cutoff -> 0.0 - - p1: [0.0, 0.0, 0.0] - p2: [10.0, 0.0, 0.0] - reference value: 0.0 - tolerance: 1.0e-10 - # Points ~556 km apart (5 deg longitude at equator) -> inside cutoff - # distance ~ 555995 m (using atlas Earth radius 6371229 m) - # x = 555995 * 1e-6 = 0.555995 - # soar(0.555995) = (1 + 0.555995) * exp(-0.555995) ~= 0.89237 - - p1: [0.0, 0.0, 0.0] - p2: [5.0, 0.0, 0.0] - reference value: 0.89237 - tolerance: 1.0e-3 diff --git a/test/testinput/unit_tests/predictors/CMakeLists.txt b/test/testinput/unit_tests/predictors/CMakeLists.txt index 961871840..d81c94e6b 100644 --- a/test/testinput/unit_tests/predictors/CMakeLists.txt +++ b/test/testinput/unit_tests/predictors/CMakeLists.txt @@ -135,15 +135,6 @@ ufo_add_test( NAME bias_linear_op LABELS unit_tests predictors WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) # -ufo_add_test( NAME bias_linear_op_tlad - ECBUILD - COMMAND ${CMAKE_BINARY_DIR}/bin/test_ObsOperatorTLAD.x - ARGS "${CMAKE_CURRENT_SOURCE_DIR}/bias_linear_op_tlad.yaml" - MPI 1 - LIBS ufo - LABELS unit_tests predictors - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../ ) -# # Test for application of bias prediction by record ufo_add_test( NAME aircraft_bc ECBUILD diff --git a/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml b/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml deleted file mode 100644 index 1f1a0f9f4..000000000 --- a/test/testinput/unit_tests/predictors/bias_linear_op_tlad.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Test obs bias increment operator - -time window: - begin: 2021-07-31T20:59:00Z - end: 2021-08-01T03:00:00Z - -observations: -- obs operator: - name: VertInterp - vertical coordinate: air_pressure - observation vertical coordinate: pressure - interpolation method: log-linear - obs space: - name: aircraft - obsdatain: - obsgrouping: - group variables: ["stationIdentification"] - sort variable: "pressure" - sort order: "descending" - engine: - type: H5File - obsfile: Data/ufo/testinput_tier_1/aircraftBCOffline_obs.nc4 - simulated variables: [airTemperature] - geovals: - filename: Data/ufo/testinput_tier_1/aircraftBCOffline_geoval.nc4 - obs bias: - input file: Data/ufo/testinput_tier_1/aircraft_bias_20210731T18Z.nc4 - bc by record: true - variational bc: - predictors: - - name: constant - - name: obsMetadataPredictor - variable: aircraftAscentRate - - name: obsMetadataPredictor - variable: aircraftAscentRate - order: 2 - covariance: # TODO: all of this section needs revisiting by experts, this is copy-paste from amsua yaml - minimal required obs number: 20 - variance range: [1.0e-6, 10.0] - step size: 1.0e-4 - largest analysis variance: 10000.0 - prior: - input file: Data/ufo/testinput_tier_1/aircraft_bias_20210731T18Z.nc4 - inflation: - ratio: 1.1 - ratio for small dataset: 2.0 - linear obs operator test: - coef TL: 1.e-5 - tolerance TL: 1.0e-10 - tolerance AD: 1.0e-13 diff --git a/test/testref/unit_tests/errors/diracDiffusion.ref b/test/testref/unit_tests/errors/diracDiffusion.ref index d53c3528f..3c1f84e67 100644 --- a/test/testref/unit_tests/errors/diracDiffusion.ref +++ b/test/testref/unit_tests/errors/diracDiffusion.ref @@ -1,6 +1,6 @@ Diffusion Dirac R*dy stats: -abi_g16 nobs= 100 Min=0.0000000000000000e+00, Max=5.8802631242318482e+00, RMS=5.9146511868699336e-01 +abi_g16 nobs= 100 Min=0.0000000000000000e+00, Max=5.7726513965964150e+00, RMS=5.8243139711273795e-01 -Final peak of dirac impulse: 5.8802631242318482e+00 -Propogated covariance at location 38: 3.0092282517488306e-01 -Propogated covariance at location 51: 5.2443889635492080e-01 +Final peak of dirac impulse: 5.7726513965964150e+00 +Propogated covariance at location 38: 4.2550394030068034e-01 +Propogated covariance at location 51: 5.7314047356895192e-01 diff --git a/test/ufo/ConventionalProfileProcessing.h b/test/ufo/ConventionalProfileProcessing.h index cbbcc3549..8bfbbf78a 100644 --- a/test/ufo/ConventionalProfileProcessing.h +++ b/test/ufo/ConventionalProfileProcessing.h @@ -74,9 +74,11 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { const Variables diagvars(varconfs); const ObsDiagnostics obsdiags(obsdiagconf, obsspace, diagvars.toOopsObsVariables()); - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); const eckit::LocalConfiguration filterConf(conf, "Conventional Profile Processing"); ufo::ConventionalProfileProcessingParameters filterParameters; @@ -159,7 +161,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -182,7 +184,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -248,7 +250,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, @@ -346,7 +348,7 @@ void testConventionalProfileProcessing(const eckit::LocalConfiguration &conf) { std::vector apply(obsspace.nlocs(), true); std::vector> flagged; ProfileDataHandler profileDataHandler(filterdata, - qcflags, + *qcflags, options.DHParameters, apply, filtervars, diff --git a/test/ufo/GaussianThinning.h b/test/ufo/GaussianThinning.h index 6b58cf9f4..6a8a895a7 100644 --- a/test/ufo/GaussianThinning.h +++ b/test/ufo/GaussianThinning.h @@ -62,8 +62,10 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "priority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "GaussianThinning"); ufo::GaussianThinningParameters filterParameters; @@ -86,8 +88,8 @@ void testGaussianThinning(const eckit::LocalConfiguration &conf) { const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices.size(), expectedThinnedObsIndices.size()); const bool equal = std::equal(thinnedObsIndices.begin(), thinnedObsIndices.end(), diff --git a/test/ufo/GeoVaLs.h b/test/ufo/GeoVaLs.h index c926ac854..520337bce 100644 --- a/test/ufo/GeoVaLs.h +++ b/test/ufo/GeoVaLs.h @@ -103,7 +103,7 @@ void testGeoVaLs() { gv.reorderzdir("air_pressure_levels", flipto); std::vector gvar(nobs); std::vector gvarref(nobs); - float sum = 0.0f; + float sum; for (size_t i = 0; i < ingeovars.size(); ++i) { size_t nlevs = gv.nlevs(ingeovars[i]); sum = 0; diff --git a/test/ufo/HistoryCheck.h b/test/ufo/HistoryCheck.h index 2971cc2d8..fa1c0526d 100644 --- a/test/ufo/HistoryCheck.h +++ b/test/ufo/HistoryCheck.h @@ -51,8 +51,10 @@ void testHistoryCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); ufo::HistoryCheck filter(obsspace, filterParameters, qcflags, obserr, conf); filter.preProcess(); @@ -60,8 +62,8 @@ void testHistoryCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected rejected obs indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::history) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::history) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/MetOfficeBuddyCheck.h b/test/ufo/MetOfficeBuddyCheck.h index f253aeca2..f3b490024 100644 --- a/test/ufo/MetOfficeBuddyCheck.h +++ b/test/ufo/MetOfficeBuddyCheck.h @@ -58,8 +58,10 @@ void testMetOfficeBuddyCheck(const eckit::LocalConfiguration &conf) { obsSpace.put_db(varGroup, varName, values); } - ioda::ObsDataVector obserr(obsSpace, obsSpace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsSpace, obsSpace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsSpace, obsSpace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsSpace, obsSpace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Met Office Buddy Check"); ufo::MetOfficeBuddyCheckParameters filterParameters; diff --git a/test/ufo/MetOfficeBuddyPairFinder.h b/test/ufo/MetOfficeBuddyPairFinder.h index a1de85373..f0e846301 100644 --- a/test/ufo/MetOfficeBuddyPairFinder.h +++ b/test/ufo/MetOfficeBuddyPairFinder.h @@ -188,6 +188,8 @@ std::vector findBuddyPairs(const MetOfficeBuddyCheckParamete } void testInvarianceToLongitude(const eckit::LocalConfiguration &conf) { + const float searchRadius = 100; // km + std::vector referenceLatitudes = conf.getFloatVector("reference.latitudes"); std::vector referenceLongitudes = conf.getFloatVector("reference.longitudes"); diff --git a/test/ufo/ObsErrorAssign.h b/test/ufo/ObsErrorAssign.h index 089f1687e..f8dbc0d67 100644 --- a/test/ufo/ObsErrorAssign.h +++ b/test/ufo/ObsErrorAssign.h @@ -49,8 +49,10 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stringVar", dat); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); const eckit::LocalConfiguration filterConf(conf, "ObsError assign"); ufo::BlackListParameters bfparam; @@ -75,9 +77,9 @@ void testObsErrorAssign(const eckit::LocalConfiguration &conf) { } } int ind = 0; - for (size_t varn = 0; varn < obserr.nvars(); ++varn) { - for (size_t locn = 0; locn < obserr.nlocs(); ++locn) { - EXPECT(oops::is_close_absolute(obserr[varn][locn], expectedObsError[ind], 1e-4f, 0, + for (size_t varn = 0; varn < obserr->nvars(); ++varn) { + for (size_t locn = 0; locn < obserr->nlocs(); ++locn) { + EXPECT(oops::is_close_absolute((*obserr)[varn][locn], expectedObsError[ind], 1e-4f, 0, oops::TestVerbosity::LOG_SUCCESS_AND_FAILURE)); ind++; } diff --git a/test/ufo/ObsFilters.h b/test/ufo/ObsFilters.h index c219b139e..ce0a101e1 100644 --- a/test/ufo/ObsFilters.h +++ b/test/ufo/ObsFilters.h @@ -191,10 +191,10 @@ class ObsFiltersParameters : public oops::Parameters { //! is not available. //! void runFinalCheck(ioda::ObsSpace &obsspace, - ioda::ObsDataVector &qcflags, - ioda::ObsDataVector &obserr) { + std::shared_ptr> qc_flags, + std::shared_ptr> obserr) { FinalCheck finalCheck(obsspace, FinalCheckParameters(), - qcflags, + qc_flags, obserr); finalCheck.doFilter(); } @@ -222,7 +222,7 @@ void convertLocalObsIndicesToGlobal(std::vector &indices, //! Return the indices of observations whose quality control flags satisfy the //! predicate in at least one variable. //! -//! \param qcflags +//! \param qc_flags //! Vector of quality control flags for all observations. //! \param obsDisttribution //! The MPI distribution used by the ObsSpace. @@ -233,16 +233,16 @@ void convertLocalObsIndicesToGlobal(std::vector &indices, //! template std::vector getObservationIndicesWhere( - const ioda::ObsDataVector &qcflags, + const ioda::ObsDataVector &qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx, const Predicate &predicate) { // Among the locations held on the calling process, identify those that satisfy the predicate. std::vector indices; - for (size_t locIndex = 0; locIndex < qcflags.nlocs(); ++locIndex) { + for (size_t locIndex = 0; locIndex < qc_flags.nlocs(); ++locIndex) { bool satisfied = false; - for (size_t varIndex = 0; varIndex < qcflags.nvars(); ++varIndex) { - if (predicate(qcflags[varIndex][locIndex])) { + for (size_t varIndex = 0; varIndex < qc_flags.nvars(); ++varIndex) { + if (predicate(qc_flags[varIndex][locIndex])) { satisfied = true; break; } @@ -268,10 +268,10 @@ std::vector getObservationIndicesWhere( //! Return the indices of observations that have passed quality control in //! at least one variable. //! -std::vector getPassedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getPassedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [](int qcFlag) { return qcFlag == 0; }); } @@ -281,10 +281,10 @@ std::vector getPassedObservationIndices(const ioda::ObsDataVector & //! Return the indices of observations that have failed quality control in //! at least one variable. //! -std::vector getFailedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getFailedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [](int qcFlag) { return qcFlag != 0; }); } @@ -294,11 +294,11 @@ std::vector getFailedObservationIndices(const ioda::ObsDataVector & //! Return the indices of observations whose quality control flag is set to \p flag in //! at least one variable. //! -std::vector getFlaggedObservationIndices(const ioda::ObsDataVector & qcflags, +std::vector getFlaggedObservationIndices(const ioda::ObsDataVector & qc_flags, const eckit::mpi::Comm &comm, const std::vector &globalIdxFromLocalIdx, int flag) { - return getObservationIndicesWhere(qcflags, comm, globalIdxFromLocalIdx, + return getObservationIndicesWhere(qc_flags, comm, globalIdxFromLocalIdx, [flag](int qcFlag) { return qcFlag == flag; }); } @@ -392,16 +392,17 @@ void expectVariablesRelativelyEqual(const ioda::ObsSpace &obsspace, void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const ObsTypeParameters ¶ms) { /// init QC and error - ioda::ObsDataVector obserr(obspace, obspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obspace, obspace.obsvariables()); + std::shared_ptr> obserrfilter = + std::make_shared>(obspace, obspace.obsvariables(), "ObsError"); + std::shared_ptr> qc_flags_ = + std::make_shared>(obspace, obspace.obsvariables()); // Create filters and run preProcess ufo::ObsFilters filters(obspace, params.filtersParams.toConfiguration(), - qcflags, obserr); + qc_flags_, obserrfilter); filters.preProcess(); - /// call priorFilter and postFilter if hofx is available oops::Variables geovars = filters.requiredVars(); oops::ObsVariables diagvars = filters.requiredHdiagnostics(); @@ -449,7 +450,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, diagvars += ybias.requiredHdiagnostics(); ufo::ObsDiagnostics diags(obspace, hop.locations(), diagvars); filters.priorFilter(gval); - hop.simulateObs(gval, hofx, ybias, qcflags, bias, diags); + hop.simulateObs(gval, hofx, ybias, *qc_flags_, bias, diags); filters.postFilter(gval, hofx, bias, diags); hofx.save("hofx"); } else if (geovars.size() > 0) { @@ -459,20 +460,20 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, oops::Log::info() << "HofX or ObsOperator sections not provided for filters, " << "postFilter not called" << std::endl; /// apply the FinalCheck filter (which should always be run after all other filters). - runFinalCheck(obspace, qcflags, obserr); - obserr.mask(qcflags); + runFinalCheck(obspace, qc_flags_, obserrfilter); + obserrfilter->mask(*qc_flags_); } else { /// no need to run priorFilter or postFilter - oops::Log::info() << "GeoVaLs not required, HofX or ObsOperator sections not " - << "provided for filters, only preProcess was called" << std::endl; + oops::Log::info() << "GeoVaLs not required, HofX or ObsOperator sections not " << + "provided for filters, only preProcess was called" << std::endl; /// apply the FinalCheck filter (which should always be run after all other filters). - runFinalCheck(obspace, qcflags, obserr); - obserr.mask(qcflags); + runFinalCheck(obspace, qc_flags_, obserrfilter); + obserrfilter->mask(*qc_flags_); } - qcflags.save("EffectiveQC"); + qc_flags_->save("EffectiveQC"); const std::string errname = "EffectiveError"; - obserr.save(errname); + obserrfilter->save(errname); // Compare with known results bool atLeastOneBenchmarkFound = false; @@ -482,14 +483,14 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &passedObsBenchmark = *params.passedObservationsBenchmark.value(); const std::vector passedObs = getPassedObservationIndices( - qcflags, obspace.comm(), obspace.index()); + *qc_flags_, obspace.comm(), obspace.index()); EXPECT_EQUAL(passedObs, passedObsBenchmark); } if (params.passedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t passedBenchmark = *params.passedBenchmark.value(); - size_t passed = numEqualTo(qcflags, ufo::QCflags::pass, + size_t passed = numEqualTo(*qc_flags_, ufo::QCflags::pass, *obspace.distribution()); EXPECT_EQUAL(passed, passedBenchmark); } @@ -499,14 +500,14 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &failedObsBenchmark = *params.failedObservationsBenchmark.value(); const std::vector failedObs = getFailedObservationIndices( - qcflags, obspace.comm(), obspace.index()); + *qc_flags_, obspace.comm(), obspace.index()); EXPECT_EQUAL(failedObs, failedObsBenchmark); } if (params.failedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t failedBenchmark = *params.failedBenchmark.value(); - size_t failed = numNonzero(qcflags, *obspace.distribution()); + size_t failed = numNonzero(*qc_flags_, *obspace.distribution()); EXPECT_EQUAL(failed, failedBenchmark); } @@ -518,7 +519,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, const std::vector &flaggedObsBenchmark = *params.flaggedObservationsBenchmark.value(); const std::vector flaggedObs = - getFlaggedObservationIndices(qcflags, obspace.comm(), + getFlaggedObservationIndices(*qc_flags_, obspace.comm(), obspace.index(), flag); EXPECT_EQUAL(flaggedObs, flaggedObsBenchmark); } @@ -526,7 +527,7 @@ void testFilters(size_t obsSpaceIndex, ioda::ObsSpace &obspace, if (params.flaggedBenchmark.value() != boost::none) { atLeastOneBenchmarkFound = true; const size_t flaggedBenchmark = *params.flaggedBenchmark.value(); - size_t flagged = numEqualTo(qcflags, flag, *obspace.distribution()); + size_t flagged = numEqualTo(*qc_flags_, flag, *obspace.distribution()); EXPECT_EQUAL(flagged, flaggedBenchmark); } } diff --git a/test/ufo/ObsLocalization.h b/test/ufo/ObsLocalization.h deleted file mode 100644 index fc508f537..000000000 --- a/test/ufo/ObsLocalization.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * (C) Copyright 2026 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef TEST_UFO_OBSLOCALIZATION_H_ -#define TEST_UFO_OBSLOCALIZATION_H_ - -#include -#include -#include -#include - -#define ECKIT_TESTING_SELF_REGISTER_CASES 0 - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/geometry/Point3.h" -#include "eckit/testing/Test.h" - -#include "ioda/ObsSpace.h" - -#include "oops/mpi/mpi.h" -#include "oops/runs/Test.h" -#include "oops/util/Logger.h" - -#include "test/TestEnvironment.h" - -#include "ufo/instantiateObsLocFactory.h" -#include "ufo/obslocalization/ObsLocalization.h" - -namespace ufo { -namespace test { - -/// \brief Tests the point3/point3 computeLocalization method of ufo obs localizations. -/// -/// For each obs localization configured under "obs localizations" in the YAML, the test: -/// 1. Constructs the localization object. -/// 2. Loops over "point pairs" specified in the config. -/// 3. For each pair, calls computeLocalization(p1, p2) and compares to a -/// reference value. -/// NOTE this does not test the geometry iterator / obsvector localization.... -/// that wasn't Travis' job that day. -void testObsLocalizationPoint3() { - typedef ioda::ObsIterator Iterator_; - - instantiateObsLocFactory(); - - const eckit::Configuration & conf = ::test::TestEnvironment::config(); - const util::TimeWindow timeWindow(conf.getSubConfiguration("time window")); - const eckit::LocalConfiguration obsconf(conf, "obs space"); - ioda::ObsSpace obsspace(obsconf, oops::mpi::world(), timeWindow, oops::mpi::myself()); - - const std::vector locConfs = - conf.getSubConfigurations("obs localizations"); - - for (const auto & locConf : locConfs) { - const eckit::LocalConfiguration locSubConf(locConf, "localization"); - const std::string name = locSubConf.getString("localization method"); - oops::Log::info() << "Testing obs localization point3/point3: " << name << std::endl; - - ObsLocalization obsloc(locSubConf, obsspace); - - const std::vector pairConfs = - locConf.getSubConfigurations("point pair tests"); - - for (const auto & pairConf : pairConfs) { - const std::vector coords1 = pairConf.getDoubleVector("p1"); - const std::vector coords2 = pairConf.getDoubleVector("p2"); - ASSERT(coords1.size() == 3); - ASSERT(coords2.size() == 3); - const eckit::geometry::Point3 p1(coords1[0], coords1[1], coords1[2]); - const eckit::geometry::Point3 p2(coords2[0], coords2[1], coords2[2]); - - const double tol = pairConf.getDouble("tolerance"); - const double ref = pairConf.getDouble("reference value"); - const double result = obsloc.computeLocalization(p1, p2); - - oops::Log::info() << " p1=" << p1 << " p2=" << p2 - << " result=" << result << " ref=" << ref << std::endl; - EXPECT(std::abs(result - ref) <= tol); - } - } -} - -// ----------------------------------------------------------------------------- - -class ObsLocalization : public oops::Test { - public: - using oops::Test::Test; - virtual ~ObsLocalization() = default; - - private: - std::string testid() const override { return "ufo::test::ObsLocalization"; } - - void register_tests() const override { - std::vector& ts = eckit::testing::specification(); - - ts.emplace_back(CASE("ufo/ObsLocalization/testObsLocalizationPoint3") - { testObsLocalizationPoint3(); }); - } - - void clear() const override {} -}; - -// ----------------------------------------------------------------------------- - -} // namespace test -} // namespace ufo - -#endif // TEST_UFO_OBSLOCALIZATION_H_ diff --git a/test/ufo/ParallelPoissonDiskThinning.h b/test/ufo/ParallelPoissonDiskThinning.h index 093361e23..c6e3063b6 100644 --- a/test/ufo/ParallelPoissonDiskThinning.h +++ b/test/ufo/ParallelPoissonDiskThinning.h @@ -48,8 +48,10 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, const eckit::LocalConfiguration obsSpaceConf(conf, "obs space"); ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::world(), timeWindow, oops::mpi::myself()); - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Poisson Disk Thinning"); ufo::PoissonDiskThinningParameters filterParameters; @@ -62,9 +64,9 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, filter.preProcess(); - std::vector isObsRetained(qcflags.nlocs(), 0); - for (size_t i = 0; i < qcflags.nlocs(); ++i) - isObsRetained[i] = (qcflags[0][i] == ufo::QCflags::pass); + std::vector isObsRetained(qcflags->nlocs(), 0); + for (size_t i = 0; i < qcflags->nlocs(); ++i) + isObsRetained[i] = ((*qcflags)[0][i] == ufo::QCflags::pass); if (obsspace.distribution()->name() == "InefficientDistribution") { // PART 1: Verify that all ranks have retained the same observations. diff --git a/test/ufo/PoissonDiskThinning.h b/test/ufo/PoissonDiskThinning.h index 6b944dd3e..fbef79ec8 100644 --- a/test/ufo/PoissonDiskThinning.h +++ b/test/ufo/PoissonDiskThinning.h @@ -61,8 +61,10 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, obsspace.put_db("MetaData", "thinningPriority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Poisson Disk Thinning"); ufo::PoissonDiskThinningParameters filterParameters; @@ -78,8 +80,8 @@ void testPoissonDiskThinning(const eckit::LocalConfiguration &conf, const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices, expectedThinnedObsIndices); } diff --git a/test/ufo/Reconditioning.h b/test/ufo/Reconditioning.h index 13d65c4fd..78897d8d5 100644 --- a/test/ufo/Reconditioning.h +++ b/test/ufo/Reconditioning.h @@ -27,7 +27,6 @@ #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" -#include "ufo/errors/ObsErrorBase.h" #include "ufo/errors/ObsErrorCrossVarCov.h" #include "ufo/errors/ObsErrorDiagonal.h" #include "ufo/errors/ObsErrorWithinGroupCov.h" @@ -53,24 +52,24 @@ void testNoReconditioning() { for (std::size_t jj = 0; jj < conf.size(); ++jj) { const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - ufo::ObsErrorParametersWrapper Params; + ufo::ObsErrorCrossVarCovParameters Params; Params.validateAndDeserialize(rconf); - if (Params.errorParams().reconditioning.value().ReconMethod.value() + if (Params.reconditioning.value().ReconMethod.value() != ufo::ObsErrorReconditionerMethod::NORECONDITIONING) continue; const eckit::LocalConfiguration obsSpaceConf(conf[jj], "obs space"); ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::myself(), timeWindow, oops::mpi::myself()); - ObsErrorBase* R = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); - ObsErrorBase* RRecon = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); + ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()); + ObsErrorCrossVarCov RRecon(Params, obsspace, oops::mpi::myself()); oops::Log::info() << "Corr before:\n" << R << std::endl; ioda::ObsVector mask(obsspace, "ObsError"); - RRecon->update(mask); + RRecon.update(mask); oops::Log::info() << "Corr after:\n" << RRecon << std::endl; - const double rmseR = R->getRMSE(); - const double rmseRR = RRecon->getRMSE(); + const double rmseR = R.getRMSE(); + const double rmseRR = RRecon.getRMSE(); EXPECT(oops::is_close(rmseR, rmseRR, 1e-10)); } } @@ -92,24 +91,21 @@ void compareKnownOutput() { const eckit::LocalConfiguration rconf(conf[jj], "obs error"); const eckit::LocalConfiguration testconf(conf[jj], "obs error test"); - ufo::ObsErrorParametersWrapper Params; + ObsErrorCrossVarCovParameters Params; ReconditioningTestParameters TestParams; Params.validateAndDeserialize(rconf); TestParams.validateAndDeserialize(testconf); - ObsErrorBase* R = ufo::ObsErrorFactory::create(Params.errorParams(), obsspace); + ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()); ioda::ObsVector mask(obsspace, "ObsError"); ioda::ObsVector sample(obsspace, "ObsValue"); std::vector refVec = TestParams.refVec.value().value(); - // update method reconditions the ObsError matrix - R->update(mask); - // multiply method applies reconditioned ObsError matrix to the sample vector - R->multiply(sample); + R.update(mask); + R.multiply(sample); std::vector sampleVec; sample.maskAndSerialize(sample, sampleVec); oops::Log::info() << "R times sample vector: " << sampleVec << std::endl; oops::Log::info() << "Reference vector: " << refVec << std::endl << std::endl; - ASSERT(sampleVec.size() == refVec.size()); for (size_t i = 0; i < sampleVec.size(); ++i) { EXPECT(oops::is_close(sampleVec[i], refVec[i], 1e-5)); @@ -133,11 +129,11 @@ void testNoValidOptionSelected() { ioda::ObsSpace obsspace(obsSpaceConf, oops::mpi::myself(), timeWindow, oops::mpi::myself()); const eckit::LocalConfiguration rconf(conf[jj], "obs error"); - ufo::ObsErrorParametersWrapper Params; + ObsErrorCrossVarCovParameters Params; Params.validateAndDeserialize(rconf); const std::string msg = conf[jj].getString("expectExceptionWithMessage"); - EXPECT_THROWS_MSG(ufo::ObsErrorFactory::create(Params.errorParams(), obsspace), msg.c_str()); + EXPECT_THROWS_MSG(ObsErrorCrossVarCov R(Params, obsspace, oops::mpi::myself()), msg.c_str()); } } diff --git a/test/ufo/SampleAndReduceOverFieldOfView.h b/test/ufo/SampleAndReduceOverFieldOfView.h index 1f1308370..969ebed4d 100644 --- a/test/ufo/SampleAndReduceOverFieldOfView.h +++ b/test/ufo/SampleAndReduceOverFieldOfView.h @@ -226,6 +226,7 @@ void testReduceOverFieldOfView() { // Make up fake samples for 3 obs with {4,5,6} samples each const int nlocs = 3; + const int nsamples = 15; const std::vector> ranges({{{0, 4}, {4, 9}, {9, 15}}}); std::vector samples({{0.2, 0.3, 0.6, 0.9, 1.7, 1.2, 1.9, 4.2, 2.9, diff --git a/test/ufo/SampledLocations.h b/test/ufo/SampledLocations.h index 454b35f6e..3db26fb9e 100644 --- a/test/ufo/SampledLocations.h +++ b/test/ufo/SampledLocations.h @@ -172,6 +172,8 @@ void testFortranTimeMask() { // ----------------------------------------------------------------------------- /// Tests operator+= (concatenation of two SampledLocations) void testConcatenate() { + typedef SampledLocationsTestFixture Test_; + const eckit::LocalConfiguration conf(::test::TestEnvironment::config()); SampledLocations locs1(conf, oops::mpi::world()); SampledLocations locs2(conf, oops::mpi::world()); diff --git a/test/ufo/StuckCheck.h b/test/ufo/StuckCheck.h index 49bb53802..01d2e27f0 100644 --- a/test/ufo/StuckCheck.h +++ b/test/ufo/StuckCheck.h @@ -45,8 +45,10 @@ void testStuckCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("ObsValue", "pressure", airPressures); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Stuck Check"); ufo::StuckCheckParameters filterParameters; @@ -57,8 +59,8 @@ void testStuckCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected_rejected_obs_indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/TemporalThinning.h b/test/ufo/TemporalThinning.h index fc4061df3..f63eafcbc 100644 --- a/test/ufo/TemporalThinning.h +++ b/test/ufo/TemporalThinning.h @@ -55,8 +55,10 @@ void testTemporalThinning(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "priority", priorities); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "TemporalThinning"); ufo::TemporalThinningParameters filterParameters; @@ -68,8 +70,8 @@ void testTemporalThinning(const eckit::LocalConfiguration &conf) { const std::vector expectedThinnedObsIndices = conf.getUnsignedVector("expected_thinned_obs_indices"); std::vector thinnedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::thinned) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::thinned) thinnedObsIndices.push_back(i); EXPECT_EQUAL(thinnedObsIndices, expectedThinnedObsIndices); } diff --git a/test/ufo/TrackCheck.h b/test/ufo/TrackCheck.h index 838c5dcae..9c6b2454f 100644 --- a/test/ufo/TrackCheck.h +++ b/test/ufo/TrackCheck.h @@ -47,8 +47,10 @@ void testTrackCheck(const eckit::LocalConfiguration &conf) { obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Track Check"); ufo::TrackCheckParameters filterParameters; @@ -59,8 +61,8 @@ void testTrackCheck(const eckit::LocalConfiguration &conf) { const std::vector expectedRejectedObsIndices = conf.getUnsignedVector("expected_rejected_obs_indices"); std::vector rejectedObsIndices; - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices.push_back(i); EXPECT_EQUAL(rejectedObsIndices, expectedRejectedObsIndices); } diff --git a/test/ufo/TrackCheckShip.h b/test/ufo/TrackCheckShip.h index 9a8587def..2a68d02b2 100644 --- a/test/ufo/TrackCheckShip.h +++ b/test/ufo/TrackCheckShip.h @@ -51,8 +51,10 @@ const boost::optional setupRunFilter( obsspace.put_db("ObsValue", "airTemperature", air_temperature); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Ship Track Check"); ufo::TrackCheckShipParameters filterParameters; @@ -60,8 +62,8 @@ const boost::optional setupRunFilter( ufo::TrackCheckShip filter(obsspace, filterParameters, qcflags, obserr); filter.preProcess(); if (filterConf.getBool("comparison test", false) && rejectedObsIndices) { - for (size_t i = 0; i < qcflags.nlocs(); ++i) - if (qcflags[0][i] == ufo::QCflags::track) + for (size_t i = 0; i < qcflags->nlocs(); ++i) + if ((*qcflags)[0][i] == ufo::QCflags::track) rejectedObsIndices->push_back(i); return boost::none; } else if (filterConf.getBool("unit testing mode", false)) { diff --git a/test/ufo/TrackCheckShipMainLoop.h b/test/ufo/TrackCheckShipMainLoop.h index 37c9fdfde..d1a54047c 100644 --- a/test/ufo/TrackCheckShipMainLoop.h +++ b/test/ufo/TrackCheckShipMainLoop.h @@ -41,8 +41,10 @@ void testFirstRejectionSimultaneousIncluded(const eckit::LocalConfiguration &con obsspace.put_db("MetaData", "stationIdentification", stationIds); } - ioda::ObsDataVector obserr(obsspace, obsspace.obsvariables(), "ObsError"); - ioda::ObsDataVector qcflags(obsspace, obsspace.obsvariables()); + std::shared_ptr> obserr(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables(), "ObsError")); + std::shared_ptr> qcflags(new ioda::ObsDataVector( + obsspace, obsspace.obsvariables())); eckit::LocalConfiguration filterConf(conf, "Ship Track Check"); ufo::TrackCheckShipParameters filterParameters; @@ -122,7 +124,7 @@ void testFirstRejectionSimultaneousIncluded(const eckit::LocalConfiguration &con } for (auto const& i : expectedFirstRejectionIndices) { - EXPECT_EQUAL(qcflags[0][i], ufo::QCflags::track); + EXPECT_EQUAL((*qcflags)[0][i], ufo::QCflags::track); // tests that the rejected observations are changing the qc flags correctly } } From cd0dfadeb323e9bbdc47510827ecb6f49427fc97 Mon Sep 17 00:00:00 2001 From: Travis Sluka Date: Thu, 23 Apr 2026 20:57:41 -0600 Subject: [PATCH 3/4] Remove duplicate cyano-bacteria_concentration from OASIM required vars (#4121) --- src/ufo/operators/oasim/ObsRadianceOASIM.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ufo/operators/oasim/ObsRadianceOASIM.cc b/src/ufo/operators/oasim/ObsRadianceOASIM.cc index 6399a15c4..9bb532453 100644 --- a/src/ufo/operators/oasim/ObsRadianceOASIM.cc +++ b/src/ufo/operators/oasim/ObsRadianceOASIM.cc @@ -48,8 +48,7 @@ ObsRadianceOASIM::ObsRadianceOASIM(const ioda::ObsSpace & odb, const Parameters_ "cyano-bacteria_concentration", "coccolithophore_concentration", "dinoflagellate_concentration", - "phaeocystis_concentration", - "cyano-bacteria_concentration"}; + "phaeocystis_concentration"}; varin_.reset(new oops::Variables(vvin)); // parse channels from the config and create variable names From d7b6c607505f3eccfb53c971f6482882f0a91917 Mon Sep 17 00:00:00 2001 From: Hailing Zhang <36716145+hailingz@users.noreply.github.com> Date: Fri, 24 Apr 2026 08:51:34 -0600 Subject: [PATCH 4/4] change superrefraction flag (#4111) * change superrefraction flag * change qcflag to 30 which represents superrefraction * Testing CI * trigger CI --------- Co-authored-by: rajichidamb Co-authored-by: Francois Hebert Co-authored-by: BenjaminRuston --- src/ufo/filters/ImpactHeightCheck.h | 2 +- .../filters/obsfunctions/function_assignvalueequalchannels.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ufo/filters/ImpactHeightCheck.h b/src/ufo/filters/ImpactHeightCheck.h index f965e1fb2..c226fc1ef 100644 --- a/src/ufo/filters/ImpactHeightCheck.h +++ b/src/ufo/filters/ImpactHeightCheck.h @@ -79,7 +79,7 @@ class ImpactHeightCheck : public FilterBase, void print(std::ostream &) const override; void applyFilter(const std::vector &, const Variables &, std::vector> &) const override; - int qcFlag() const override {return QCflags::domain;} + int qcFlag() const override {return QCflags::superrefraction;} Parameters_ parameters_; std::vector calcVerticalGradient(const std::vector &, const std::vector &) const; diff --git a/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml b/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml index b9d1dd215..54854bdc5 100644 --- a/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml +++ b/test/testinput/unit_tests/filters/obsfunctions/function_assignvalueequalchannels.yaml @@ -74,7 +74,7 @@ observations: variable: name: QCflagsData/bendingAngle channels: *all_levels - testValue: 13 + testValue: 30 assignEqual: 0.9 assignNotEqual: 0.2 #