diff --git a/Project.toml b/Project.toml index 3d409e173f8..c13d73073a2 100644 --- a/Project.toml +++ b/Project.toml @@ -60,6 +60,7 @@ Convex = "f65535da-76fb-5f13-bab9-19810c17039a" ECOS = "e2685f51-7e38-5353-a97d-a921fd2c8199" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" +OrdinaryDiffEqCore = "bbf590c4-e513-4bbe-9b18-05decba2e5d8" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" @@ -70,6 +71,7 @@ TrixiConvexECOSExt = ["Convex", "ECOS"] TrixiMakieExt = "Makie" TrixiNLsolveExt = "NLsolve" TrixiPlotsExt = "Plots" +TrixiOrdinaryDiffEqCoreExt = "OrdinaryDiffEqCore" TrixiSparseConnectivityTracerExt = "SparseConnectivityTracer" [compat] @@ -83,7 +85,7 @@ ConstructionBase = "1.5.8" Convex = "0.16" DataStructures = "0.18.15, 0.19" DelimitedFiles = "1" -DiffEqBase = "6.184" +DiffEqBase = "6.184, 7" DiffEqCallbacks = "2.35, 3, 4" Downloads = "1.6" ECOS = "1.1.2" @@ -101,6 +103,7 @@ MuladdMacro = "0.2.4" NLsolve = "4.5.1" Octavian = "0.3.28" OffsetArrays = "1.13" +OrdinaryDiffEqCore = "1.26, 2, 3, 4" P4est = "0.4.12" Plots = "1.38.13" Polyester = "=0.7.16, 0.7.18" @@ -108,9 +111,9 @@ PrecompileTools = "1.2.1" Preferences = "1.5" Printf = "1" RecipesBase = "1.3.4" -RecursiveArrayTools = "3.37" +RecursiveArrayTools = "3.37, 4" Reexport = "1.2.2" -SciMLBase = "2.128" +SciMLBase = "2.128, 3" SimpleUnPack = "1.1" SparseArrays = "1" SparseConnectivityTracer = "1.0.1" diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 4cf957da32d..ac149af4720 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -6,6 +6,6 @@ Trixi = "a7f1ee26-1774-49b1-8366-f1abc58fbfcb" [compat] BenchmarkTools = "0.5, 0.7, 1.0" -OrdinaryDiffEq = "5.65, 6" +OrdinaryDiffEq = "5.65, 6, 7" PkgBenchmark = "0.2.10" Trixi = "0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16" diff --git a/docs/Project.toml b/docs/Project.toml index a476ed00d1c..60cea53eb12 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -41,11 +41,11 @@ LaTeXStrings = "1.2" Literate = "2.9" Measurements = "2.5" NLsolve = "4.5.1" -OrdinaryDiffEqLowOrderRK = "1.2" -OrdinaryDiffEqLowStorageRK = "1.2" -OrdinaryDiffEqSDIRK = "1.1" -OrdinaryDiffEqSSPRK = "1.2" -OrdinaryDiffEqTsit5 = "1.1" +OrdinaryDiffEqLowOrderRK = "1.2, 2" +OrdinaryDiffEqLowStorageRK = "1.2, 2, 3" +OrdinaryDiffEqSDIRK = "1.1, 2" +OrdinaryDiffEqSSPRK = "1.2, 2" +OrdinaryDiffEqTsit5 = "1.1, 2" Plots = "1.9" SparseConnectivityTracer = "1.0.1" SparseMatrixColorings = "0.4.21" diff --git a/docs/src/parallelization.md b/docs/src/parallelization.md index dc5e2400dde..460e249df33 100644 --- a/docs/src/parallelization.md +++ b/docs/src/parallelization.md @@ -25,7 +25,7 @@ the same time, the latter takes precedence. If you use time integration methods from [OrdinaryDiffEq.jl](https://github.com/SciML/OrdinaryDiffEq.jl) and want to use multiple threads therein, you need to set the keyword argument -`thread = Trixi.True()` (or `thread = OrdinaryDiffEq.True()`) +`thread = Trixi.Threaded()` of the algorithms, as described in the [section on time integration methods](@ref time-integration). diff --git a/docs/src/time_integration.md b/docs/src/time_integration.md index 9ab753de205..bce37778e71 100644 --- a/docs/src/time_integration.md +++ b/docs/src/time_integration.md @@ -30,9 +30,9 @@ are the following. Further documentation can be found in the from Trixi.jl. - If you start Julia with multiple threads and want to use them also in the time integration method from OrdinaryDiffEq.jl, you need to pass the keyword argument - `thread = Trixi.True()` (or `thread = OrdinaryDiffEq.True()`) to the algorithm, e.g., - `RDPK3SpFSAL49(thread = Trixi.True())` or - `CarpenterKennedy2N54(thread = Trixi.True(), williamson_condition = false)`. + `thread = Trixi.Threaded()` to the algorithm, e.g., + `RDPK3SpFSAL49(thread = Trixi.Threaded())` or + `CarpenterKennedy2N54(thread = Trixi.Threaded(), williamson_condition = false)`. For more information on using thread-based parallelism in Trixi.jl, please refer to [Shared-memory parallelization with threads](@ref). - If you use error-based step size control (see also the section on diff --git a/examples/p4est_2d_dgsem/elixir_euler_NACA0012airfoil_mach085.jl b/examples/p4est_2d_dgsem/elixir_euler_NACA0012airfoil_mach085.jl index 94db4ecda09..067490f5da9 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_NACA0012airfoil_mach085.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_NACA0012airfoil_mach085.jl @@ -140,6 +140,6 @@ callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, ############################################################################### # run the simulation -sol = solve(ode, SSPRK54(thread = Trixi.True()); +sol = solve(ode, SSPRK54(thread = Trixi.Threaded()); dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_NACA6412airfoil_mach2.jl b/examples/p4est_2d_dgsem/elixir_euler_NACA6412airfoil_mach2.jl index 8f9a43ca795..2754d8ec364 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_NACA6412airfoil_mach2.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_NACA6412airfoil_mach2.jl @@ -103,7 +103,7 @@ callbacks = CallbackSet(summary_callback, # Run the simulation ############################################################################### -sol = solve(ode, SSPRK104(; thread = Trixi.True()); +sol = solve(ode, SSPRK104(; thread = Trixi.Threaded()); dt = 1.0, # overwritten by the `stepsize_callback` ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_cylinder_bowshock_mach3.jl b/examples/p4est_2d_dgsem/elixir_euler_cylinder_bowshock_mach3.jl index 4673700114c..025b06a3c45 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_cylinder_bowshock_mach3.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_cylinder_bowshock_mach3.jl @@ -181,6 +181,6 @@ stage_limiter! = PositivityPreservingLimiterZhangShu(thresholds = (5.0e-6, 5.0e- ############################################################################### # Run the simulation -sol = solve(ode, SSPRK33(; stage_limiter! = stage_limiter!, thread = Trixi.True()); +sol = solve(ode, SSPRK33(; stage_limiter! = stage_limiter!, thread = Trixi.Threaded()); dt = 1.6e-5, # Fixed timestep works decent here ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_double_mach_amr_adaptive_vol_int.jl b/examples/p4est_2d_dgsem/elixir_euler_double_mach_amr_adaptive_vol_int.jl index 324a0630b21..0862769e8c3 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_double_mach_amr_adaptive_vol_int.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_double_mach_amr_adaptive_vol_int.jl @@ -167,6 +167,6 @@ stage_limiter! = PositivityPreservingLimiterZhangShu(thresholds = (5.0e-6, 5.0e- ############################################################################### # run the simulation -sol = solve(ode, SSPRK43(; stage_limiter! = stage_limiter!, thread = Trixi.True()); +sol = solve(ode, SSPRK43(; stage_limiter! = stage_limiter!, thread = Trixi.Threaded()); dt = 5e-7, # Reducing initial timestep allows AMR interval of 2 instead of 1 adaptive = true, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_rayleigh_taylor_instability.jl b/examples/p4est_2d_dgsem/elixir_euler_rayleigh_taylor_instability.jl index 8a5f04c414a..fc21482f1b2 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_rayleigh_taylor_instability.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_rayleigh_taylor_instability.jl @@ -159,7 +159,7 @@ callbacks = CallbackSet(summary_callback, # run the simulation time_int_tol = 1e-6 -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = time_int_tol, reltol = time_int_tol, adaptive = true, dt = 1e-3, # needed only for tests/CI ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_subsonic_cylinder.jl b/examples/p4est_2d_dgsem/elixir_euler_subsonic_cylinder.jl index 331404eb21b..c9ec66b047a 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_subsonic_cylinder.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_subsonic_cylinder.jl @@ -126,6 +126,6 @@ callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, sav # run the simulation sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false; - thread = Trixi.True()); + thread = Trixi.Threaded()); dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_euler_supersonic_cylinder_scO2.jl b/examples/p4est_2d_dgsem/elixir_euler_supersonic_cylinder_scO2.jl index 867c2868209..ae6b2918810 100644 --- a/examples/p4est_2d_dgsem/elixir_euler_supersonic_cylinder_scO2.jl +++ b/examples/p4est_2d_dgsem/elixir_euler_supersonic_cylinder_scO2.jl @@ -133,6 +133,6 @@ stage_limiter! = PositivityPreservingLimiterZhangShu(thresholds = (5.0e-7, 1.0e- # We supply a small initial timestep to be able to use a larger AMR interval (3 instead of 1) throughout the simulation. # This pays off almost immediately as only the first couple timesteps use this timestep before it is ramped up. dt0 = 1e-8 -sol = solve(ode, SSPRK43(; stage_limiter! = stage_limiter!, thread = Trixi.True()); +sol = solve(ode, SSPRK43(; stage_limiter! = stage_limiter!, thread = Trixi.Threaded()); adaptive = true, dt = dt0, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_mhd_rotor_cfl_ramp.jl b/examples/p4est_2d_dgsem/elixir_mhd_rotor_cfl_ramp.jl index f7673eabf03..8c0482d61a7 100644 --- a/examples/p4est_2d_dgsem/elixir_mhd_rotor_cfl_ramp.jl +++ b/examples/p4est_2d_dgsem/elixir_mhd_rotor_cfl_ramp.jl @@ -141,7 +141,7 @@ callbacks = CallbackSet(summary_callback, # run the simulation sol = solve(ode, - CarpenterKennedy2N54(thread = Trixi.True(), + CarpenterKennedy2N54(thread = Trixi.Threaded(), williamson_condition = false); dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach08.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach08.jl index a86d8f2958c..05f547a7056 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach08.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach08.jl @@ -173,6 +173,6 @@ callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, sav ############################################################################### # run the simulation -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = 1e-8, reltol = 1e-8, ode_default_options()..., callback = callbacks) diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach085_restart.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach085_restart.jl index d1dca6b2993..4c9f579e2b5 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach085_restart.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_NACA0012airfoil_mach085_restart.jl @@ -83,6 +83,6 @@ callbacks = CallbackSet(summary_callback, analysis_callback, alive_callback, ############################################################################### -sol = solve(ode, SSPRK54(thread = Trixi.True()); +sol = solve(ode, SSPRK54(thread = Trixi.Threaded()); dt = dt_restart, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_RAE2822airfoil_separation.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_RAE2822airfoil_separation.jl index 5a0f14e543d..7c598c067aa 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_RAE2822airfoil_separation.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_RAE2822airfoil_separation.jl @@ -128,11 +128,11 @@ callbacks = CallbackSet(summary_callback, ############################################################################### # run the simulation -ode_algorithm = SSPRK43(thread = Trixi.True()) +ode_algorithm = SSPRK43(thread = Trixi.Threaded()) time_int_tol = 1e-4 sol = solve(ode, ode_algorithm; abstol = time_int_tol, reltol = time_int_tol, dt = 1e-6, maxiters = Inf, # long simulation - controller = PIDController(0.55, -0.27, 0.05), # optimized for SSPRK43 + controller = PIDController(ode_algorithm, beta = (0.55, -0.27, 0.05)), # optimized for SSPRK43 ode_default_options()..., callback = callbacks) diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_SD7003airfoil.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_SD7003airfoil.jl index 66f94fa1944..cdc54cf4877 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_SD7003airfoil.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_SD7003airfoil.jl @@ -146,5 +146,5 @@ callbacks = CallbackSet(summary_callback, sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false, - thread = Trixi.True()); + thread = Trixi.Threaded()); dt = 1.0, ode_default_options()..., callback = callbacks) diff --git a/examples/p4est_2d_dgsem/elixir_navierstokes_vortex_street.jl b/examples/p4est_2d_dgsem/elixir_navierstokes_vortex_street.jl index 932926770e9..51605d37e5e 100644 --- a/examples/p4est_2d_dgsem/elixir_navierstokes_vortex_street.jl +++ b/examples/p4est_2d_dgsem/elixir_navierstokes_vortex_street.jl @@ -158,7 +158,7 @@ callbacks = CallbackSet(summary_callback, # run the simulation # Moderate number of threads (e.g. 4) advisable to speed things up -ode_alg = RDPK3SpFSAL49(thread = Trixi.True()) +ode_alg = RDPK3SpFSAL49(thread = Trixi.Threaded()) time_int_tol = 1e-7 sol = solve(ode, ode_alg; # not necessary, added for overwriting in tests diff --git a/examples/p4est_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/p4est_3d_dgsem/elixir_euler_baroclinic_instability.jl index daa758d1f9b..f92119a2adb 100644 --- a/examples/p4est_3d_dgsem/elixir_euler_baroclinic_instability.jl +++ b/examples/p4est_3d_dgsem/elixir_euler_baroclinic_instability.jl @@ -294,6 +294,6 @@ callbacks = CallbackSet(summary_callback, # Use a Runge-Kutta method with automatic (error based) time step size control # Enable threading of the RK method for better performance on multiple threads -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = 1.0e-6, reltol = 1.0e-6, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_3d_dgsem/elixir_euler_circular_wind_nonconforming.jl b/examples/p4est_3d_dgsem/elixir_euler_circular_wind_nonconforming.jl index ed93bae6e8c..ccb43801045 100644 --- a/examples/p4est_3d_dgsem/elixir_euler_circular_wind_nonconforming.jl +++ b/examples/p4est_3d_dgsem/elixir_euler_circular_wind_nonconforming.jl @@ -157,6 +157,6 @@ callbacks = CallbackSet(summary_callback, # Use a Runge-Kutta method with automatic (error based) time step size control # Enable threading of the RK method for better performance on multiple threads -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = 1.0e-6, reltol = 1.0e-6, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_3d_dgsem/elixir_euler_tandem_spheres.jl b/examples/p4est_3d_dgsem/elixir_euler_tandem_spheres.jl index 60f800e7f62..781e3f39068 100644 --- a/examples/p4est_3d_dgsem/elixir_euler_tandem_spheres.jl +++ b/examples/p4est_3d_dgsem/elixir_euler_tandem_spheres.jl @@ -101,6 +101,6 @@ callbacks = CallbackSet(summary_callback, ############################################################################### tols = 1e-5 -sol = solve(ode, RDPK3SpFSAL35(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL35(thread = Trixi.Threaded()); abstol = tols, reltol = tols, ode_default_options()..., callback = callbacks); diff --git a/examples/p4est_3d_dgsem/elixir_navierstokes_blast_wave_amr.jl b/examples/p4est_3d_dgsem/elixir_navierstokes_blast_wave_amr.jl index 2c3b563df00..76a0fa8c470 100644 --- a/examples/p4est_3d_dgsem/elixir_navierstokes_blast_wave_amr.jl +++ b/examples/p4est_3d_dgsem/elixir_navierstokes_blast_wave_amr.jl @@ -109,6 +109,6 @@ callbacks = CallbackSet(summary_callback, # run the simulation time_int_tol = 1e-8 -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = time_int_tol, reltol = time_int_tol, ode_default_options()..., callback = callbacks) diff --git a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl index 1e2cc40f092..46120c58a4d 100644 --- a/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl +++ b/examples/t8code_3d_dgsem/elixir_euler_baroclinic_instability.jl @@ -294,6 +294,6 @@ callbacks = CallbackSet(summary_callback, # Use a Runge-Kutta method with automatic (error based) time step size control # Enable threading of the RK method for better performance on multiple threads -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); abstol = 1.0e-6, reltol = 1.0e-6, ode_default_options()..., callback = callbacks); diff --git a/examples/tree_1d_dgsem/elixir_euler_density_wave_adaptive_vol_int.jl b/examples/tree_1d_dgsem/elixir_euler_density_wave_adaptive_vol_int.jl index 309a36b0577..e990553fc1a 100644 --- a/examples/tree_1d_dgsem/elixir_euler_density_wave_adaptive_vol_int.jl +++ b/examples/tree_1d_dgsem/elixir_euler_density_wave_adaptive_vol_int.jl @@ -67,6 +67,7 @@ callbacks = CallbackSet(summary_callback, ############################################################################### # run the simulation -sol = solve(ode, CarpenterKennedy2N54(williamson_condition = false, thread = Trixi.True()); +sol = solve(ode, + CarpenterKennedy2N54(williamson_condition = false, thread = Trixi.Threaded()); dt = stepsize_callback(ode), ode_default_options()..., callback = callbacks); diff --git a/examples/tree_2d_dgsem/elixir_euler_astro_jet_amr_scO2.jl b/examples/tree_2d_dgsem/elixir_euler_astro_jet_amr_scO2.jl index a9f0c2d0e28..8618abe8c1f 100644 --- a/examples/tree_2d_dgsem/elixir_euler_astro_jet_amr_scO2.jl +++ b/examples/tree_2d_dgsem/elixir_euler_astro_jet_amr_scO2.jl @@ -105,7 +105,8 @@ stage_limiter! = PositivityPreservingLimiterZhangShu(thresholds = (5.0e-6, 5.0e- ############################################################################### # run the simulation +ode_algorithm = SSPRK43(stage_limiter! = stage_limiter!, thread = Trixi.Threaded()) # use adaptive time stepping based on error estimates -sol = solve(ode, SSPRK43(; stage_limiter! = stage_limiter!, thread = Trixi.True()); - controller = PIDController(0.55, -0.27, 0.05), +sol = solve(ode, ode_algorithm; + controller = PIDController(ode_algorithm, beta = (0.55, -0.27, 0.05)), ode_default_options()..., callback = callbacks); diff --git a/examples/tree_2d_dgsem/elixir_euler_colliding_flow_amr.jl b/examples/tree_2d_dgsem/elixir_euler_colliding_flow_amr.jl index a1205f58fd8..f4f58a5a114 100644 --- a/examples/tree_2d_dgsem/elixir_euler_colliding_flow_amr.jl +++ b/examples/tree_2d_dgsem/elixir_euler_colliding_flow_amr.jl @@ -119,5 +119,5 @@ stage_limiter! = PositivityPreservingLimiterZhangShu(thresholds = (5.0e-6, 5.0e- ############################################################################### # run the simulation # use adaptive time stepping based on error estimates, time step roughly dt = 5e-3 -sol = solve(ode, SSPRK43(stage_limiter!); +sol = solve(ode, SSPRK43(; stage_limiter!); ode_default_options()..., callback = callbacks); diff --git a/examples/tree_3d_dgsem/elixir_euler_sedov_blast_weak_form_sc.jl b/examples/tree_3d_dgsem/elixir_euler_sedov_blast_weak_form_sc.jl index d8d5367160b..d45e0bf64c1 100644 --- a/examples/tree_3d_dgsem/elixir_euler_sedov_blast_weak_form_sc.jl +++ b/examples/tree_3d_dgsem/elixir_euler_sedov_blast_weak_form_sc.jl @@ -117,6 +117,6 @@ callbacks = CallbackSet(summary_callback, ############################################################################### # run the simulation -sol = solve(ode, SSPRK54(thread = Trixi.True()); +sol = solve(ode, SSPRK54(thread = Trixi.Threaded()); dt = 1.0, # solve needs some value here but it will be overwritten by the stepsize_callback ode_default_options()..., callback = callbacks); diff --git a/examples/tree_3d_dgsem/elixir_euler_sedov_scO2.jl b/examples/tree_3d_dgsem/elixir_euler_sedov_scO2.jl index bb515a5c175..8946e6f61eb 100644 --- a/examples/tree_3d_dgsem/elixir_euler_sedov_scO2.jl +++ b/examples/tree_3d_dgsem/elixir_euler_sedov_scO2.jl @@ -100,5 +100,5 @@ callbacks = CallbackSet(summary_callback, ############################################################################### # run the simulation -sol = solve(ode, SSPRK54(thread = Trixi.True()); dt = 1.0, +sol = solve(ode, SSPRK54(thread = Trixi.Threaded()); dt = 1.0, ode_default_options()..., callback = callbacks); diff --git a/examples/tree_3d_dgsem/elixir_navierstokes_taylor_green_vortex_adaptive_vol_int.jl b/examples/tree_3d_dgsem/elixir_navierstokes_taylor_green_vortex_adaptive_vol_int.jl index 79c0ca15b73..08692a20a43 100644 --- a/examples/tree_3d_dgsem/elixir_navierstokes_taylor_green_vortex_adaptive_vol_int.jl +++ b/examples/tree_3d_dgsem/elixir_navierstokes_taylor_green_vortex_adaptive_vol_int.jl @@ -99,5 +99,5 @@ callbacks = CallbackSet(summary_callback, ############################################################################### # run the simulation -sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.True()); adaptive = true, +sol = solve(ode, RDPK3SpFSAL49(thread = Trixi.Threaded()); adaptive = true, ode_default_options()..., callback = callbacks) diff --git a/ext/TrixiOrdinaryDiffEqCoreExt.jl b/ext/TrixiOrdinaryDiffEqCoreExt.jl new file mode 100644 index 00000000000..96c3fd21c6a --- /dev/null +++ b/ext/TrixiOrdinaryDiffEqCoreExt.jl @@ -0,0 +1,113 @@ +module TrixiOrdinaryDiffEqCoreExt + +import Trixi: load_controller!, store_controller! +import OrdinaryDiffEqCore: OrdinaryDiffEqCore, PIController, PIDController +import HDF5: attributes + +@static if Base.pkgversion(OrdinaryDiffEqCore) >= v"4" + import OrdinaryDiffEqCore: PIControllerCache, PIDControllerCache +end + +# Support to load controller. +# OrdinaryDiffEqCore < v4: The state of PIController and PIDController lives on the +# integrator (integrator.qold) rather than on the controller struct. +# OrdinaryDiffEqCore >= v4: the legacy controllers are replaced by PIControllerCache and +# PIDControllerCache, and integrator.qold no longer exists. + +# OrdinaryDiffEqCore < v4, PI controller: +# Previous error estimate stored as integrator.qold. +function load_controller!(integrator, ::PIController, file) + if !("time_integrator_qold" in keys(attributes(file))) + error("Missing data in restart file: check the consistency of adaptive time controller with initial setup!") + end + integrator.qold = read(attributes(file)["time_integrator_qold"]) +end + +# OrdinaryDiffEqCore < v4, PID controller: +# Accumulated dt factor stored as integrator.qold. +# Error history field layout changed between v3.31 and v3.32: +# <= v3.31: err::MVector{3} (single array field) +# v3.32–v3.33: err1, err2, err3 (individual scalar fields) +function load_controller!(integrator, controller::PIDController, file) + if !("time_integrator_qold" in keys(attributes(file)) || + !("time_integrator_controller_err" in keys(attributes(file)))) + error("Missing data in restart file: check the consistency of adaptive time controller with initial setup!") + end + integrator.qold = read(attributes(file)["time_integrator_qold"]) + if hasproperty(controller, :err) + # OrdinaryDiffEqCore <= v3.31 + controller.err[:] = read(attributes(file)["time_integrator_controller_err"]) + elseif hasproperty(controller, :err1) && hasproperty(controller, :err2) && + hasproperty(controller, :err3) + # OrdinaryDiffEqCore v3.32–v3.33 + err = read(attributes(file)["time_integrator_controller_err"]) + controller.err1 = err[1] + controller.err2 = err[2] + controller.err3 = err[3] + end +end + +@static if Base.pkgversion(OrdinaryDiffEqCore) >= v"4" + # OrdinaryDiffEqCore >= v4, PI controller: + # Previous error estimate stored as PIControllerCache.errold (integrator.qold removed). + function load_controller!(integrator, controller::PIControllerCache, file) + if !("time_integrator_qold" in keys(attributes(file))) + error("Missing data in restart file: check the consistency of adaptive time controller with initial setup!") + end + controller.errold = read(attributes(file)["time_integrator_qold"]) + end + + # OrdinaryDiffEqCore >= v4, PID controller: + # Accumulated dt factor stored as PIDControllerCache.dt_factor (integrator.qold removed). + # Error history stored as PIDControllerCache.err::Vector{3}. + function load_controller!(integrator, controller::PIDControllerCache, file) + if !("time_integrator_qold" in keys(attributes(file)) || + !("time_integrator_controller_err" in keys(attributes(file)))) + error("Missing data in restart file: check the consistency of adaptive time controller with initial setup!") + end + controller.dt_factor = read(attributes(file)["time_integrator_qold"]) + controller.err[:] = read(attributes(file)["time_integrator_controller_err"]) + end +end + +# OrdinaryDiffEqCore < v4, PI controller: +# Previous error estimate stored as integrator.qold. +function store_controller!(file, ::PIController, integrator) + attributes(file)["time_integrator_qold"] = integrator.qold +end + +# OrdinaryDiffEqCore < v4, PID controller: +# Accumulated dt factor stored as integrator.qold. +# Error history field layout changed between v3.31 and v3.32: +# <= v3.31: err::MVector{3} (single array field) +# v3.32–v3.33: err1, err2, err3 (individual scalar fields) +function store_controller!(file, controller::PIDController, integrator) + attributes(file)["time_integrator_qold"] = integrator.qold + if hasproperty(controller, :err) + # OrdinaryDiffEqCore <= v3.31 + attributes(file)["time_integrator_controller_err"] = controller.err + else + # OrdinaryDiffEqCore v3.32–v3.33 + attributes(file)["time_integrator_controller_err"] = [controller.err1, + controller.err2, + controller.err3] + end +end + +@static if Base.pkgversion(OrdinaryDiffEqCore) >= v"4" + # OrdinaryDiffEqCore >= v4, PI controller: + # Previous error estimate stored as PIControllerCache.errold (integrator.qold removed). + function store_controller!(file, controller::PIControllerCache, integrator) + attributes(file)["time_integrator_qold"] = controller.errold + end + + # OrdinaryDiffEqCore >= v4, PID controller: + # Accumulated dt factor stored as PIDControllerCache.dt_factor (integrator.qold removed). + # Error history stored as PIDControllerCache.err::Vector{3}. + function store_controller!(file, controller::PIDControllerCache, integrator) + attributes(file)["time_integrator_qold"] = controller.dt_factor + attributes(file)["time_integrator_controller_err"] = controller.err + end +end + +end diff --git a/src/Trixi.jl b/src/Trixi.jl index ac57e30845c..a35879d0a1c 100644 --- a/src/Trixi.jl +++ b/src/Trixi.jl @@ -87,6 +87,8 @@ using T8code using RecipesBase: RecipesBase using RecursiveArrayTools: VectorOfArray using Static: Static, One, True, False +# OrdinaryDiffEq v7+ uses FastBroadcast.Threaded() for the thread argument; older versions use Static.True() +const Threaded = isdefined(DiffEqBase, :Threaded) ? DiffEqBase.Threaded : True @reexport using StaticArrays: SVector using StaticArrays: StaticArrays, MVector, MArray, SMatrix, @SMatrix using StrideArrays: PtrArray, StrideArray, StaticInt diff --git a/src/callbacks_step/callbacks_step.jl b/src/callbacks_step/callbacks_step.jl index a540fb49f6b..8a619f8428f 100644 --- a/src/callbacks_step/callbacks_step.jl +++ b/src/callbacks_step/callbacks_step.jl @@ -30,6 +30,20 @@ end integrator.iter == integrator.opts.maxiters end +# OrdinaryDiffEqCore < v4: legacy controller lives in integrator.opts.controller and +# doubles as its own cache (setup_controller_cache returns the controller itself). +# OrdinaryDiffEqCore >= v4: controller cache lives in integrator.controller_cache. +get_controller_cache(integrator) = hasproperty(integrator, :controller_cache) ? + integrator.controller_cache : + integrator.opts.controller + +# OrdinaryDiffEq v7+ wraps the pure controller (parameters only) in controller_cache.controller; +# older versions store it directly in controller_cache or opts.controller. +function get_controller(integrator) + cc = get_controller_cache(integrator) + return hasproperty(cc, :controller) ? cc.controller : cc +end + # `include` callback definitions in the order that we currently prefer # when combining them into a `CallbackSet` which is called *after* a complete step # The motivation is as follows: The first callbacks belong to the current time step iteration: diff --git a/src/callbacks_step/save_restart.jl b/src/callbacks_step/save_restart.jl index 387316b898d..d67e0963f1f 100644 --- a/src/callbacks_step/save_restart.jl +++ b/src/callbacks_step/save_restart.jl @@ -96,8 +96,7 @@ function (restart_callback::SaveRestartCallback)(integrator) save_restart_file(u_ode, t, dt, iter, semi, restart_callback) # If using an adaptive time stepping scheme, store controller values for restart if integrator.opts.adaptive - save_adaptive_time_integrator(integrator, integrator.opts.controller, - restart_callback) + save_adaptive_time_integrator(integrator, restart_callback) end end @@ -170,29 +169,37 @@ Load the context information for time integrators with error-based step size con saved in a `restart_file`. """ function load_adaptive_time_integrator!(integrator, restart_file::AbstractString) - controller = integrator.opts.controller # Read context information for controller h5open(restart_file, "r") do file # Ensure that the necessary information was saved - if !("time_integrator_qold" in keys(attributes(file))) || - !("time_integrator_dtpropose" in keys(attributes(file))) || - (hasproperty(controller, :err) && - !("time_integrator_controller_err" in keys(attributes(file)))) + if !("time_integrator_dtpropose" in keys(attributes(file))) error("Missing data in restart file: check the consistency of adaptive time controller with initial setup!") end - # Load data that is required both for PIController and PIDController - integrator.qold = read(attributes(file)["time_integrator_qold"]) integrator.dtpropose = read(attributes(file)["time_integrator_dtpropose"]) # Accept step to use dtpropose already in the first step integrator.accept_step = true # Reevaluate integrator.fsal_first on the first step integrator.reeval_fsal = true - # Load additional parameters for PIDController - if hasproperty(controller, :err) # Distinguish PIDController from PIController - controller.err[:] = read(attributes(file)["time_integrator_controller_err"]) - end + + load_controller!(integrator, file) end end +function load_controller!(integrator, file) + return load_controller!(integrator, get_controller_cache(integrator), file) +end + +function load_controller!(integrator, controller, file) + return error("Loading of controller $(typeof(controller)) not implemented.") +end + +function store_controller!(file, integrator) + return store_controller!(file, get_controller_cache(integrator), integrator) +end + +function store_controller!(file, controller, integrator) + return error("Storing of controller $(typeof(controller)) not implemented.") +end + include("save_restart_dg.jl") end # @muladd diff --git a/src/callbacks_step/save_restart_dg.jl b/src/callbacks_step/save_restart_dg.jl index 259dacc9a81..2c86017c305 100644 --- a/src/callbacks_step/save_restart_dg.jl +++ b/src/callbacks_step/save_restart_dg.jl @@ -336,8 +336,7 @@ function load_restart_file_on_root(mesh::Union{TreeMeshParallel, P4estMeshParall end # Store controller values for an adaptive time stepping scheme -function save_adaptive_time_integrator(integrator, - controller, restart_callback) +function save_adaptive_time_integrator(integrator, restart_callback) # Save only on root if mpi_isroot() @unpack output_directory = restart_callback @@ -348,13 +347,8 @@ function save_adaptive_time_integrator(integrator, # Open file (preserve existing content) h5open(filename, "r+") do file - # Add context information as attributes both for PIController and PIDController - attributes(file)["time_integrator_qold"] = integrator.qold attributes(file)["time_integrator_dtpropose"] = integrator.dtpropose - # For PIDController is necessary to save additional parameters - if hasproperty(controller, :err) # Distinguish PIDController from PIController - attributes(file)["time_integrator_controller_err"] = controller.err - end + store_controller!(file, integrator) end end end diff --git a/src/callbacks_step/save_solution_dg.jl b/src/callbacks_step/save_solution_dg.jl index 3419f0eefee..0e91e75803f 100644 --- a/src/callbacks_step/save_solution_dg.jl +++ b/src/callbacks_step/save_solution_dg.jl @@ -113,7 +113,7 @@ function save_solution_file(u, time, dt, timestep, data = u n_vars = nvariables(equations) else - data = map(u_node -> solution_variables(u_node, equations), u) + data = map(u_node -> solution_variables(u_node, equations), parent(u)) # Find out variable count by looking at output from `solution_variables` function. n_vars = length(data[1]) end diff --git a/src/callbacks_step/summary.jl b/src/callbacks_step/summary.jl index 554e779e1ec..02f34fa916f 100644 --- a/src/callbacks_step/summary.jl +++ b/src/callbacks_step/summary.jl @@ -208,7 +208,7 @@ function initialize_summary_callback(cb::DiscreteCallback, u, t, integrator; push!(setup, "abstol" => integrator.opts.abstol, "reltol" => integrator.opts.reltol, - "controller" => integrator.opts.controller) + "controller" => get_controller(integrator)) end summary_box(io, "Time integration", setup) println() diff --git a/test/Project.toml b/test/Project.toml index 12d22e98808..d0178da9c65 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -63,21 +63,21 @@ LinearAlgebra = "1" LinearSolve = "3.54" MPI = "0.20.23" NLsolve = "4.5.1" -OrdinaryDiffEqBDF = "1.1" -OrdinaryDiffEqCore = "1.26, 2, 3.0 - 3.31" -OrdinaryDiffEqFeagin = "1" -OrdinaryDiffEqHighOrderRK = "1.1" -OrdinaryDiffEqLowOrderRK = "1.2" -OrdinaryDiffEqLowStorageRK = "1.2" -OrdinaryDiffEqSDIRK = "1.1" -OrdinaryDiffEqSSPRK = "1.2" -OrdinaryDiffEqStabilizedRK = "1.3" -OrdinaryDiffEqTsit5 = "1.1" +OrdinaryDiffEqBDF = "1.1, 2" +OrdinaryDiffEqCore = "1.26, 2, 3, 4" +OrdinaryDiffEqFeagin = "1, 2" +OrdinaryDiffEqHighOrderRK = "1.1, 2" +OrdinaryDiffEqLowOrderRK = "1.2, 2" +OrdinaryDiffEqLowStorageRK = "1.2, 2, 3" +OrdinaryDiffEqSDIRK = "1.1, 2" +OrdinaryDiffEqSSPRK = "1.2, 2" +OrdinaryDiffEqStabilizedRK = "1.3, 2" +OrdinaryDiffEqTsit5 = "1.1, 2" Plots = "1.38.13" Printf = "1" Quadmath = "0.5.10" Random = "1" -SciMLBase = "2.128" +SciMLBase = "2.128, 3" SciMLOperators = "1.13" SparseArrays = "1" SparseConnectivityTracer = "1.0.1" diff --git a/test/runtests.jl b/test/runtests.jl index 6679fbb8bd9..b26b0b24f86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,6 +21,10 @@ const TRIXI_NTHREADS = clamp(Sys.CPU_THREADS, 2, 3) # cf. https://github.com/JuliaParallel/MPI.jl/pull/391 @test true + # Trixi automatically initializes MPI, this causes issues if precompilation occurs under MPI. + # The below MPI test uses different compilation flags and thus we want to ensure that precompilation is done with the same flags. + run(`$(Base.julia_cmd()) --threads=1 --check-bounds=yes -e "using Trixi, OrdinaryDiffEqCore"`) + # We provide a `--heap-size-hint` to avoid/reduce out-of-memory errors during CI testing run(`$(mpiexec()) -n $TRIXI_MPI_NPROCS $(Base.julia_cmd()) --threads=1 --check-bounds=yes --heap-size-hint=0.5G $(joinpath(@__DIR__, "test_mpi.jl"))`) end diff --git a/test/test_mpi_tree.jl b/test/test_mpi_tree.jl index 62c74e2fd3c..c6df01b2bfb 100644 --- a/test/test_mpi_tree.jl +++ b/test/test_mpi_tree.jl @@ -24,6 +24,7 @@ CI_ON_WINDOWS = (get(ENV, "GITHUB_ACTIONS", false) == "true") && Sys.iswindows() end @trixi_testset "elixir_advection_restart.jl" begin + # Perform a standard simulation using OrdinaryDiffEqLowStorageRK: RDPK3SpFSAL49 mpi_isroot() && println("═"^100) mpi_isroot() && @@ -35,16 +36,18 @@ CI_ON_WINDOWS = (get(ENV, "GITHUB_ACTIONS", false) == "true") && Sys.iswindows() alg = RDPK3SpFSAL49(), tspan = (0.0, 10.0)) l2_expected, linf_expected = analysis_callback(sol) + # Perform a simulation restarting from an intermediate state mpi_isroot() && println("═"^100) mpi_isroot() && println(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl")) - # Errors are exactly the same as in the elixir_advection_extended.jl trixi_include(@__MODULE__, joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), alg = RDPK3SpFSAL49(), base_elixir = "elixir_advection_timeintegration_adaptive.jl") l2_actual, linf_actual = analysis_callback(sol) + # Check whether the errors are exactly the same as in the uninterrupted run + # using the same low-storage RK method with error-based step size control. mpi_isroot() && @test l2_actual == l2_expected mpi_isroot() && @test linf_actual == linf_expected end @@ -74,7 +77,7 @@ CI_ON_WINDOWS = (get(ENV, "GITHUB_ACTIONS", false) == "true") && Sys.iswindows() @trixi_testset "elixir_advection_restart_amr.jl" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_restart_amr.jl"), - l2=[8.018497923389368e-5], + l2=[8.018498574373939e-5], linf=[0.0007307237754662355]) end diff --git a/test/test_threaded.jl b/test/test_threaded.jl index d32d3a41196..7f34819290d 100644 --- a/test/test_threaded.jl +++ b/test/test_threaded.jl @@ -40,11 +40,45 @@ Trixi.MPI.Barrier(Trixi.mpi_comm()) @test_allocations(Trixi.rhs!, semi, sol, 5000) end + # This test covers some of the code paths for the old controller interface in OrdinaryDiffEqCore < v4, + # which is still supported for backward compatibility. + @trixi_testset "elixir_advection_restart.jl with adaptive time integration" begin + # Perform a standard simulation + using OrdinaryDiffEqSSPRK: SSPRK43 + using OrdinaryDiffEqLowStorageRK: RDPK3SpFSAL49 + mod = @__MODULE__ + # SSPRK43 uses PIController, RDPK3SpFSAL49 uses PIDController + for alg in (SSPRK43(), RDPK3SpFSAL49()) + println("═"^100) + base_elixir = joinpath(EXAMPLES_DIR, "tree_2d_dgsem", + "elixir_advection_timeintegration_adaptive.jl") + println(base_elixir) + trixi_include(mod, + base_elixir, alg = alg, tspan = (0.0, 10.0)) + l2_expected, linf_expected = @invokelatest mod.analysis_callback(@invokelatest mod.sol) + + # Perform a simulation restarting from an intermediate state + println("═"^100) + elixir = joinpath(EXAMPLES_DIR, "tree_2d_dgsem", + "elixir_advection_restart.jl") + println(elixir) + trixi_include(mod, + elixir, alg = alg, + base_elixir = base_elixir) + l2_actual, linf_actual = @invokelatest mod.analysis_callback(@invokelatest mod.sol) + + # Check whether the errors are exactly the same as in the uninterrupted run + # using the default SSPRK or low-storage RK method with error-based step size control. + @test l2_actual == l2_expected + @test linf_actual == linf_expected + end + end + @trixi_testset "elixir_advection_restart.jl with threaded time integration" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "tree_2d_dgsem", "elixir_advection_restart.jl"), alg=CarpenterKennedy2N54(williamson_condition = false, - thread = Trixi.True()), + thread = Trixi.Threaded()), # Expected errors are exactly the same as in the serial test! l2=[8.005068880114254e-6], linf=[6.39093577996519e-5]) @@ -148,7 +182,7 @@ Trixi.MPI.Barrier(Trixi.mpi_comm()) @test_trixi_include(joinpath(EXAMPLES_DIR, "tree_2d_dgsem", "elixir_advection_diffusion.jl"), initial_refinement_level=2, tspan=(0.0, 0.4), polydeg=5, - alg=RDPK3SpFSAL49(thread = Trixi.True()), + alg=RDPK3SpFSAL49(thread = Trixi.Threaded()), l2=[4.0915532997994255e-6], linf=[2.3040850347877395e-5]) @@ -363,7 +397,7 @@ end @trixi_testset "elixir_euler_curved.jl with threaded time integration" begin @test_trixi_include(joinpath(EXAMPLES_DIR, "dgmulti_2d", "elixir_euler_curved.jl"), - alg=RDPK3SpFSAL49(thread = Trixi.True()), + alg=RDPK3SpFSAL49(thread = Trixi.Threaded()), l2=[ 1.720916434676505e-5, 1.5928649356300228e-5, diff --git a/test/test_tree_2d_advection.jl b/test/test_tree_2d_advection.jl index 672555c4c11..53f3aefe71b 100644 --- a/test/test_tree_2d_advection.jl +++ b/test/test_tree_2d_advection.jl @@ -59,6 +59,7 @@ end end @trixi_testset "elixir_advection_restart.jl" begin + # Perform a standard simulation using OrdinaryDiffEqSSPRK: SSPRK43 println("═"^100) println(joinpath(EXAMPLES_DIR, "elixir_advection_timeintegration_adaptive.jl")) @@ -68,14 +69,17 @@ end alg = SSPRK43(), tspan = (0.0, 10.0)) l2_expected, linf_expected = analysis_callback(sol) + # Perform a simulation restarting from an intermediate state println("═"^100) println(joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl")) - # Errors are exactly the same as in the elixir_advection_extended.jl - trixi_include(@__MODULE__, joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), + trixi_include(@__MODULE__, + joinpath(EXAMPLES_DIR, "elixir_advection_restart.jl"), alg = SSPRK43(), base_elixir = "elixir_advection_timeintegration_adaptive.jl") l2_actual, linf_actual = analysis_callback(sol) + # Check whether the errors are exactly the same as in the uninterrupted run + # using the same SSPRK method with error-based step size control. @test l2_actual == l2_expected @test linf_actual == linf_expected end