diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 38a475fb50..a8f03e6125 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -230,34 +230,6 @@ jobs: files: lcov.info token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false # or true if you want CI to fail when Codecov fails - integration: - timeout-minutes: 20 - name: Integration Tests - ${{ matrix.test }} - runs-on: ${{ matrix.os }} - env: - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - strategy: - fail-fast: false - matrix: - version: - - '1.10' - os: - - ubuntu-latest - test: - - DynamicExpressions - - Bijectors - steps: - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ matrix.version }} - - uses: julia-actions/cache@v2 - - uses: julia-actions/julia-buildpkg@v1 - - name: "Run tests" - run: | - julia --color=yes --project=test/integration/${{ matrix.test }} -e 'using Pkg; Pkg.develop([PackageSpec(; path) for path in (".", "lib/EnzymeCore")]); Pkg.instantiate()' - julia --color=yes --project=test/integration/${{ matrix.test }} --threads=auto --check-bounds=yes test/integration/${{ matrix.test }}/runtests.jl - shell: bash docs: timeout-minutes: 20 name: Documentation diff --git a/.github/workflows/Integration.yml b/.github/workflows/Integration.yml new file mode 100644 index 0000000000..3f7f216873 --- /dev/null +++ b/.github/workflows/Integration.yml @@ -0,0 +1,70 @@ +name: Integration +on: + pull_request: + paths: + - '.github/workflows/Integration.yml' + - 'ext/**' + - 'lib/**' + - 'src/**' + - 'test/**' + - 'Project.toml' + push: + branches: + - main + paths: + - release-* + - '.github/workflows/Integration.yml' + - 'ext/**' + - 'lib/**' + - 'src/**' + - 'test/**' + - 'Project.toml' + tags: '*' + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + integration: + timeout-minutes: 45 + name: Integration Tests - ${{ matrix.package }} - Julia ${{ matrix.version }} + runs-on: ${{ matrix.os }} + container: + image: ${{ (contains(matrix.os, 'linux') && 'ghcr.io/enzymead/reactant-docker-images@sha256:91e1edb7a7c869d5a70db06e417f22907be0e67ca86641d48adcea221fedc674' ) || '' }} + env: + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - '1.11' + os: + - linux-x86-n2-32 + package: + - Bijectors + - DifferentiationInterface + - DynamicExpressions + steps: + - uses: actions/checkout@v5 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + - name: "Install Dependencies" + run: | + julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 -e 'using Pkg; Pkg.develop([PackageSpec(; path) for path in (".", "lib/EnzymeCore")])' + shell: bash + if: ${{ matrix.version == '1.10' }} + - name: "Instantiate" + run: | + julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 -e 'using Pkg; Pkg.instantiate()' + shell: bash + - name: "Run tests" + run: | + julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 test/integration/${{ matrix.package }}/runtests.jl + shell: bash diff --git a/.gitignore b/.gitignore index 6f1a0e7a15..670cf4bbdb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,5 @@ lib/EnzymeCore/Manifest.toml /docs/Manifest.toml /docs/build/ /docs/src/generated/ - +test/integration/**/Manifest.toml .vscode diff --git a/test/integration/Bijectors/Project.toml b/test/integration/Bijectors/Project.toml index 2b8c2f46c2..b74b9f2e51 100644 --- a/test/integration/Bijectors/Project.toml +++ b/test/integration/Bijectors/Project.toml @@ -1,8 +1,14 @@ [deps] Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +[sources] +Enzyme = { path = "../../.." } +EnzymeCore = { path = "../../../lib/EnzymeCore" } + [compat] Bijectors = "=0.13.16" FiniteDifferences = "0.12.32" diff --git a/test/integration/Bijectors/runtests.jl b/test/integration/Bijectors/runtests.jl index 23e6136561..ccc894315c 100644 --- a/test/integration/Bijectors/runtests.jl +++ b/test/integration/Bijectors/runtests.jl @@ -131,7 +131,11 @@ end sum_b_binv_test_case(Bijectors.VecCholeskyBijector(:U), 3), sum_b_binv_test_case(Bijectors.VecCholeskyBijector(:U), 0), sum_b_binv_test_case(Bijectors.Coupling(Bijectors.Shift, Bijectors.PartitionMask(3, [1], [2])), 3), - sum_b_binv_test_case(Bijectors.InvertibleBatchNorm(3), (3, 3)), + sum_b_binv_test_case( + Bijectors.InvertibleBatchNorm(3), + (3, 3); + runtime_activity = (VERSION >= v"1.11" ? Both : Neither) + ), sum_b_binv_test_case(Bijectors.LeakyReLU(0.2), 3), sum_b_binv_test_case(Bijectors.Logit(0.1, 0.3), 3), sum_b_binv_test_case(Bijectors.PDBijector(), (3, 3)), @@ -186,6 +190,8 @@ end end, randn(rng, 7); name="PlanarLayer7", + # https://github.com/TuringLang/Bijectors.jl/issues/415 + broken = (VERSION >= v"1.11" ? Forward : Neither), ), TestCase( @@ -197,6 +203,7 @@ end end, randn(rng, 11); name="PlanarLayer11", + broken = (VERSION >= v"1.11" ? Forward : Neither), ), ] diff --git a/test/integration/DifferentiationInterface/Project.toml b/test/integration/DifferentiationInterface/Project.toml new file mode 100644 index 0000000000..678f007fc9 --- /dev/null +++ b/test/integration/DifferentiationInterface/Project.toml @@ -0,0 +1,18 @@ +[deps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[sources] +Enzyme = { path = "../../.." } +EnzymeCore = { path = "../../../lib/EnzymeCore" } + +[compat] +ADTypes = "1.17.0" +DifferentiationInterface = "=0.7.7" +DifferentiationInterfaceTest = "=0.10.2" +StaticArrays = "1.9" diff --git a/test/integration/DifferentiationInterface/README.md b/test/integration/DifferentiationInterface/README.md new file mode 100644 index 0000000000..f5984e7bca --- /dev/null +++ b/test/integration/DifferentiationInterface/README.md @@ -0,0 +1,35 @@ +# DI integration tests for Enzyme + +This folder contains tests to ensure that changes to Enzyme do not break integration with [DifferentiationInterface](https://github.com/JuliaDiff/DifferentiationInterface.jl) (DI). + +## Relevant source files + +The test utilities used here come from the sibling package [DifferentiationInterfaceTest](https://github.com/JuliaDiff/DifferentiationInterface.jl/tree/main/DifferentiationInterfaceTest) (DIT). +Correctness checking itself is implemented in [`src/tests/correctness_eval.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/tests/correctness_eval.jl), which is where you will see test errors originating. +Test scenarios are located in [`src/scenarios`](https://github.com/JuliaDiff/DifferentiationInterface.jl/tree/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios) (especially [`src/scenarios/default.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/default.jl) and [`src/scenarios/modify.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/modify.jl)) and in package extensions. +The structure of a `Scenario` is defined in [`src/scenarios/scenario.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/scenario.jl). + +Scenario generation relies on internals of DifferentiationInterfaceTest, which is why its version is pinned in the `Project.toml`. + +## Interpreting test errors + +The most common test error you will see looks like + +```julia +Correctness: Test Failed at .../src/tests/correctness_eval.jl:... + Expression: res1_out1_noval ≈ scen.res1 +``` + +Each test scenario `scen` contains a first-order result `res1` and a second-order result `res2`, which are the reference values we compare our autodiff results (i.e. the output of `DI.gradient` or `DI.jacobian`) against. +The suffixes of the left-hand term are defined as follows: + +- `_out` for the output of the operator, `_in` for the input if it is in-place +- `1` for the first call to the operator, `2` for the second call (which is used to check that the preparation object has not been altered and can safely be reused) +- `_val` if the operator also returns the value of the function (like `DI.value_and_gradient`), `_noval` otherwise + +As you can see, several variants of each operator are tested, so a single bug will give rise to many different errors. In addition, different preparation mechanisms are also cycled through. +The testset summary at the end of the CI log is probably the right place to start hunting down issues. + +## What to do if a test fails + +Open an issue on the DI repo with a link to the relevant PR or CI log. \ No newline at end of file diff --git a/test/integration/DifferentiationInterface/runtests.jl b/test/integration/DifferentiationInterface/runtests.jl new file mode 100644 index 0000000000..3af90111dd --- /dev/null +++ b/test/integration/DifferentiationInterface/runtests.jl @@ -0,0 +1,86 @@ +using ADTypes: AutoEnzyme +using DifferentiationInterface: DifferentiationInterface +using DifferentiationInterfaceTest: + default_scenarios, + sparse_scenarios, + static_scenarios, + test_differentiation, + function_place, + operator_place, + FIRST_ORDER, + SECOND_ORDER, + Scenario +using Enzyme: Enzyme +using EnzymeCore: Forward, Reverse, Const, Duplicated +using StaticArrays: StaticArrays +using Test + +logging = get(ENV, "CI", "false") == "false" + +function remove_matrices(scens::Vector{<:Scenario}) # TODO: remove + return filter(s -> s.x isa Union{Number, AbstractVector} && s.y isa Union{Number, AbstractVector}, scens) +end + +backends = [ + AutoEnzyme(; function_annotation = Const), + AutoEnzyme(; mode = Forward), + AutoEnzyme(; mode = Reverse), +] + +duplicated_backends = [ + AutoEnzyme(; mode = Forward, function_annotation = Duplicated), + AutoEnzyme(; mode = Reverse, function_annotation = Duplicated), +] + +@testset verbose = true "DifferentiationInterface integration" begin + test_differentiation( + backends, + default_scenarios(; include_constantified = true); + excluded = SECOND_ORDER, + logging, + testset_name = "Generic first order", + ) + + test_differentiation( + backends[1], + remove_matrices(default_scenarios(; include_constantified = true)); + excluded = FIRST_ORDER, + logging, + testset_name = "Generic second order", + ) + + test_differentiation( + backends[2], + remove_matrices( + default_scenarios(; + include_normal = false, + include_cachified = true, + include_constantorcachified = true, + use_tuples = true, + ) + ); + excluded = SECOND_ORDER, + logging, + testset_name = "Caches", + ) + + test_differentiation( + duplicated_backends, + remove_matrices(default_scenarios(; include_normal = false, include_closurified = true)); + excluded = SECOND_ORDER, + logging, + testset_name = "Closures", + ) + + filtered_static_scenarios = filter(remove_matrices(static_scenarios())) do s + operator_place(s) == :out && function_place(s) == :out + end + + test_differentiation( + backends[2:3], + filtered_static_scenarios; + excluded = SECOND_ORDER, + logging, + testset_name = "Static arrays", + ) +end diff --git a/test/integration/DynamicExpressions/Project.toml b/test/integration/DynamicExpressions/Project.toml index dd4a40af73..7654504251 100644 --- a/test/integration/DynamicExpressions/Project.toml +++ b/test/integration/DynamicExpressions/Project.toml @@ -1,5 +1,11 @@ [deps] DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + +[sources] +Enzyme = { path = "../../.." } +EnzymeCore = { path = "../../../lib/EnzymeCore" } [compat] DynamicExpressions = "=0.18.5"