diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22e9d7532..27df8adcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,7 @@ repos: additional_dependencies: - aiopg==1.4.0 - alabaster==1.0.0 + - alembic==1.18.4 - asgiref==3.11.0 - async-timeout==4.0.3 - attrs==25.4.0 @@ -53,6 +54,7 @@ repos: - idna==3.15 - imagesize==1.4.1 - jinja2==3.1.6 + - mako==1.3.12 - markupsafe==3.0.3 - packaging==26.0 - psycopg==3.3.2 diff --git a/docs/howto/production/migrations.md b/docs/howto/production/migrations.md index 8db7fb280..e3310dd91 100644 --- a/docs/howto/production/migrations.md +++ b/docs/howto/production/migrations.md @@ -22,6 +22,13 @@ ALTER TABLE procrastinate_jobs ADD COLUMN extra TEXT; The migration scripts are pure-SQL scripts, meaning that they may be applied to the database using any PostgreSQL client, including `psql` and `PGAdmin`. +Procrastinate also ships optional Alembic revisions that wrap the same SQL +migration scripts. Install them with: + +```console +pip install "procrastinate[alembic]" +``` + :::{note} If you use Django, instead of using the SQL migration scripts directly, you way want to rely on the Procrastinate Django app, and the Django database migration scripts @@ -37,6 +44,36 @@ $ procrastinate schema --migrations-path /home/me/my_venv/lib/python3.x/lib/site-packages/procrastinate/sql/migrations ``` +If your project already uses Alembic, add Procrastinate's packaged Alembic +versions directory to your Alembic `version_locations`, then run Alembic normally: + +```ini +[alembic] +version_locations = %(here)s/versions procrastinate:alembic/versions +``` + +The `procrastinate:alembic/versions` entry is resolved by Alembic from the +installed Python package, so it does not depend on where your virtual environment +or site-packages directory is located. + +The Procrastinate Alembic tree uses revision IDs prefixed with `procrastinate_` +and a `procrastinate` branch label, so it can live alongside your own revisions. +Most projects should keep the Procrastinate and application revision trees +independent. If your own schema changes must run after a specific Procrastinate +revision, your revision may set `down_revision` to that Procrastinate revision. + +Each Alembic revision wraps exactly one SQL migration script: the revision ID +(e.g. `procrastinate_0036`) is sequential, and each revision file records the +name of the SQL script it wraps. For blue-green deployments, upgrade to the +last `pre` revision of the target version before deploying, then to the last +`post` revision after: + +```console +alembic upgrade procrastinate_0036 +yoursystem/deploy procrastinate 3.4.0 +alembic upgrade procrastinate_0037 +``` + It's your responsibility to keep track of which migrations have been applied yet or not. Thankfully, the names of procrastinate migrations should help you: they follow a specific pattern: diff --git a/procrastinate/alembic/__init__.py b/procrastinate/alembic/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/procrastinate/alembic/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/procrastinate/alembic/versions/__init__.py b/procrastinate/alembic/versions/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/procrastinate/alembic/versions/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/procrastinate/alembic/versions/procrastinate_0000_initial.py b/procrastinate/alembic/versions/procrastinate_0000_initial.py new file mode 100644 index 000000000..650d993d5 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0000_initial.py @@ -0,0 +1,35 @@ +"""00.00.00 01 initial.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0000" +down_revision: str | None = None +branch_labels = ("procrastinate",) +depends_on = None + +MIGRATION_FILE = "00.00.00_01_initial.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0001_drop_started_at_column.py b/procrastinate/alembic/versions/procrastinate_0001_drop_started_at_column.py new file mode 100644 index 000000000..9fb975cf0 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0001_drop_started_at_column.py @@ -0,0 +1,35 @@ +"""00.05.00 01 drop started at column.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0001" +down_revision: str | None = "procrastinate_0000" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.05.00_01_drop_started_at_column.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0002_drop_started_at_column.py b/procrastinate/alembic/versions/procrastinate_0002_drop_started_at_column.py new file mode 100644 index 000000000..4e71179bc --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0002_drop_started_at_column.py @@ -0,0 +1,35 @@ +"""00.05.00 02 drop started at column.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0002" +down_revision: str | None = "procrastinate_0001" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.05.00_02_drop_started_at_column.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0003_drop_procrastinate_version_table.py b/procrastinate/alembic/versions/procrastinate_0003_drop_procrastinate_version_table.py new file mode 100644 index 000000000..b35e472a7 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0003_drop_procrastinate_version_table.py @@ -0,0 +1,35 @@ +"""00.05.00 03 drop procrastinate version table.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0003" +down_revision: str | None = "procrastinate_0002" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.05.00_03_drop_procrastinate_version_table.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0004_fix_procrastinate_fetch_job.py b/procrastinate/alembic/versions/procrastinate_0004_fix_procrastinate_fetch_job.py new file mode 100644 index 000000000..4382d3468 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0004_fix_procrastinate_fetch_job.py @@ -0,0 +1,35 @@ +"""00.06.00 01 fix procrastinate fetch job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0004" +down_revision: str | None = "procrastinate_0003" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.06.00_01_fix_procrastinate_fetch_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0005_fix_trigger_status_events_insert.py b/procrastinate/alembic/versions/procrastinate_0005_fix_trigger_status_events_insert.py new file mode 100644 index 000000000..96cc9c76f --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0005_fix_trigger_status_events_insert.py @@ -0,0 +1,35 @@ +"""00.07.01 01 fix trigger status events insert.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0005" +down_revision: str | None = "procrastinate_0004" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.07.01_01_fix_trigger_status_events_insert.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0006_add_queueing_lock_column.py b/procrastinate/alembic/versions/procrastinate_0006_add_queueing_lock_column.py new file mode 100644 index 000000000..bccd1f854 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0006_add_queueing_lock_column.py @@ -0,0 +1,35 @@ +"""00.08.01 01 add queueing lock column.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0006" +down_revision: str | None = "procrastinate_0005" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.08.01_01_add_queueing_lock_column.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0007_close_fetch_job_race_condition.py b/procrastinate/alembic/versions/procrastinate_0007_close_fetch_job_race_condition.py new file mode 100644 index 000000000..e6095a2d9 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0007_close_fetch_job_race_condition.py @@ -0,0 +1,35 @@ +"""00.10.00 01 close fetch job race condition.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0007" +down_revision: str | None = "procrastinate_0006" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.10.00_01_close_fetch_job_race_condition.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0008_add_defer_job_function.py b/procrastinate/alembic/versions/procrastinate_0008_add_defer_job_function.py new file mode 100644 index 000000000..886a50105 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0008_add_defer_job_function.py @@ -0,0 +1,35 @@ +"""00.10.00 02 add defer job function.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0008" +down_revision: str | None = "procrastinate_0007" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.10.00_02_add_defer_job_function.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0009_add_procrastinate_periodic_defers.py b/procrastinate/alembic/versions/procrastinate_0009_add_procrastinate_periodic_defers.py new file mode 100644 index 000000000..aaccaf5b7 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0009_add_procrastinate_periodic_defers.py @@ -0,0 +1,35 @@ +"""00.11.00 03 add procrastinate periodic defers.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0009" +down_revision: str | None = "procrastinate_0008" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.11.00_03_add_procrastinate_periodic_defers.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0010_add_foreign_key_index.py b/procrastinate/alembic/versions/procrastinate_0010_add_foreign_key_index.py new file mode 100644 index 000000000..22585624c --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0010_add_foreign_key_index.py @@ -0,0 +1,35 @@ +"""00.12.00 01 add foreign key index.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0010" +down_revision: str | None = "procrastinate_0009" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.12.00_01_add_foreign_key_index.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0011_add_locks_to_periodic_defer.py b/procrastinate/alembic/versions/procrastinate_0011_add_locks_to_periodic_defer.py new file mode 100644 index 000000000..f79acd4ce --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0011_add_locks_to_periodic_defer.py @@ -0,0 +1,35 @@ +"""00.14.00 01 add locks to periodic defer.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0011" +down_revision: str | None = "procrastinate_0010" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.14.00_01_add_locks_to_periodic_defer.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0012_fix_procrastinate_defer_periodic_job.py b/procrastinate/alembic/versions/procrastinate_0012_fix_procrastinate_defer_periodic_job.py new file mode 100644 index 000000000..2c326ba91 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0012_fix_procrastinate_defer_periodic_job.py @@ -0,0 +1,35 @@ +"""00.15.02 01 fix procrastinate defer periodic job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0012" +down_revision: str | None = "procrastinate_0011" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.15.02_01_fix_procrastinate_defer_periodic_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0013_add_finish_job_and_retry_job_functions.py b/procrastinate/alembic/versions/procrastinate_0013_add_finish_job_and_retry_job_functions.py new file mode 100644 index 000000000..bb96367ee --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0013_add_finish_job_and_retry_job_functions.py @@ -0,0 +1,35 @@ +"""00.16.00 01 add finish job and retry job functions.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0013" +down_revision: str | None = "procrastinate_0012" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.16.00_01_add_finish_job_and_retry_job_functions.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0014_add_trigger_on_job_deletion.py b/procrastinate/alembic/versions/procrastinate_0014_add_trigger_on_job_deletion.py new file mode 100644 index 000000000..3d1421d8c --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0014_add_trigger_on_job_deletion.py @@ -0,0 +1,35 @@ +"""00.17.00 01 add trigger on job deletion.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0014" +down_revision: str | None = "procrastinate_0013" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.17.00_01_add_trigger_on_job_deletion.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0015_delete_finished_jobs.py b/procrastinate/alembic/versions/procrastinate_0015_delete_finished_jobs.py new file mode 100644 index 000000000..7922420a7 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0015_delete_finished_jobs.py @@ -0,0 +1,35 @@ +"""00.17.00 02 delete finished jobs.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0015" +down_revision: str | None = "procrastinate_0014" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.17.00_02_delete_finished_jobs.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0016_add_checks_to_finish_job.py b/procrastinate/alembic/versions/procrastinate_0016_add_checks_to_finish_job.py new file mode 100644 index 000000000..25cb82c5f --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0016_add_checks_to_finish_job.py @@ -0,0 +1,35 @@ +"""00.17.00 03 add checks to finish job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0016" +down_revision: str | None = "procrastinate_0015" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.17.00_03_add_checks_to_finish_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0017_add_checks_to_retry_job.py b/procrastinate/alembic/versions/procrastinate_0017_add_checks_to_retry_job.py new file mode 100644 index 000000000..df618b6b9 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0017_add_checks_to_retry_job.py @@ -0,0 +1,35 @@ +"""00.17.00 04 add checks to retry job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0017" +down_revision: str | None = "procrastinate_0016" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.17.00_04_add_checks_to_retry_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0018_fix_finish_job_compat_issue.py b/procrastinate/alembic/versions/procrastinate_0018_fix_finish_job_compat_issue.py new file mode 100644 index 000000000..c53c34f87 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0018_fix_finish_job_compat_issue.py @@ -0,0 +1,35 @@ +"""00.18.01 01 fix finish job compat issue.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0018" +down_revision: str | None = "procrastinate_0017" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.18.01_01_fix_finish_job_compat_issue.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0019_add_index_on_procrastinate_jobs.py b/procrastinate/alembic/versions/procrastinate_0019_add_index_on_procrastinate_jobs.py new file mode 100644 index 000000000..690ea67b0 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0019_add_index_on_procrastinate_jobs.py @@ -0,0 +1,35 @@ +"""00.19.00 01 add index on procrastinate jobs.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0019" +down_revision: str | None = "procrastinate_0018" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.19.00_01_add_index_on_procrastinate_jobs.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0020_add_kwargs_to_defer_periodic_job.py b/procrastinate/alembic/versions/procrastinate_0020_add_kwargs_to_defer_periodic_job.py new file mode 100644 index 000000000..03cb86257 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0020_add_kwargs_to_defer_periodic_job.py @@ -0,0 +1,35 @@ +"""00.22.00 01 add kwargs to defer periodic job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0020" +down_revision: str | None = "procrastinate_0019" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.22.00_01_add_kwargs_to_defer_periodic_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0021_null_locks_excluded.py b/procrastinate/alembic/versions/procrastinate_0021_null_locks_excluded.py new file mode 100644 index 000000000..cc0172216 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0021_null_locks_excluded.py @@ -0,0 +1,35 @@ +"""00.23.00 01 null locks excluded.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0021" +down_revision: str | None = "procrastinate_0020" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "00.23.00_01_null_locks_excluded.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0022_remove_old_finish_job_function.py b/procrastinate/alembic/versions/procrastinate_0022_remove_old_finish_job_function.py new file mode 100644 index 000000000..ffc2caaf2 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0022_remove_old_finish_job_function.py @@ -0,0 +1,35 @@ +"""01.00.00 01 remove old finish job function.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0022" +down_revision: str | None = "procrastinate_0021" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "01.00.00_01_remove_old_finish_job_function.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0023_job_id_bigint.py b/procrastinate/alembic/versions/procrastinate_0023_job_id_bigint.py new file mode 100644 index 000000000..f18b2ce8f --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0023_job_id_bigint.py @@ -0,0 +1,35 @@ +"""01.01.01 01 job id bigint.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0023" +down_revision: str | None = "procrastinate_0022" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "01.01.01_01_job_id_bigint.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0024_add_job_priority.py b/procrastinate/alembic/versions/procrastinate_0024_add_job_priority.py new file mode 100644 index 000000000..b90e58fb4 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0024_add_job_priority.py @@ -0,0 +1,35 @@ +"""02.00.03 01 add job priority.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0024" +down_revision: str | None = "procrastinate_0023" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "02.00.03_01_add_job_priority.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0025_add_periodic_job_priority.py b/procrastinate/alembic/versions/procrastinate_0025_add_periodic_job_priority.py new file mode 100644 index 000000000..d7877dacf --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0025_add_periodic_job_priority.py @@ -0,0 +1,35 @@ +"""02.05.00 01 add periodic job priority.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0025" +down_revision: str | None = "procrastinate_0024" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "02.05.00_01_add_periodic_job_priority.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0026_add_cancel_states.py b/procrastinate/alembic/versions/procrastinate_0026_add_cancel_states.py new file mode 100644 index 000000000..bdccd29bf --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0026_add_cancel_states.py @@ -0,0 +1,35 @@ +"""02.06.00 01 add cancel states.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0026" +down_revision: str | None = "procrastinate_0025" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "02.06.00_01_add_cancel_states.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0027_add_additional_params_to_retry_job.py b/procrastinate/alembic/versions/procrastinate_0027_add_additional_params_to_retry_job.py new file mode 100644 index 000000000..c3b8ae6bc --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0027_add_additional_params_to_retry_job.py @@ -0,0 +1,35 @@ +"""02.08.00 01 add additional params to retry job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0027" +down_revision: str | None = "procrastinate_0026" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "02.08.00_01_add_additional_params_to_retry_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0028_add_indexes_for_fetch_job.py b/procrastinate/alembic/versions/procrastinate_0028_add_indexes_for_fetch_job.py new file mode 100644 index 000000000..f97fe527c --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0028_add_indexes_for_fetch_job.py @@ -0,0 +1,35 @@ +"""02.14.01 01 add indexes for fetch job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0028" +down_revision: str | None = "procrastinate_0027" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "02.14.01_01_add_indexes_for_fetch_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0029_pre_cancel_notification.py b/procrastinate/alembic/versions/procrastinate_0029_pre_cancel_notification.py new file mode 100644 index 000000000..168f84be7 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0029_pre_cancel_notification.py @@ -0,0 +1,35 @@ +"""03.00.00 01 pre cancel notification.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0029" +down_revision: str | None = "procrastinate_0028" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.00.00_01_pre_cancel_notification.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0030_post_cancel_notification.py b/procrastinate/alembic/versions/procrastinate_0030_post_cancel_notification.py new file mode 100644 index 000000000..f6e863196 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0030_post_cancel_notification.py @@ -0,0 +1,35 @@ +"""03.00.00 50 post cancel notification.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0030" +down_revision: str | None = "procrastinate_0029" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.00.00_50_post_cancel_notification.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0031_pre_add_heartbeat.py b/procrastinate/alembic/versions/procrastinate_0031_pre_add_heartbeat.py new file mode 100644 index 000000000..576c07348 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0031_pre_add_heartbeat.py @@ -0,0 +1,35 @@ +"""03.01.00 01 pre add heartbeat.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0031" +down_revision: str | None = "procrastinate_0030" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.01.00_01_pre_add_heartbeat.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0032_post_add_heartbeat.py b/procrastinate/alembic/versions/procrastinate_0032_post_add_heartbeat.py new file mode 100644 index 000000000..93a8014c4 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0032_post_add_heartbeat.py @@ -0,0 +1,35 @@ +"""03.01.00 50 post add heartbeat.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0032" +down_revision: str | None = "procrastinate_0031" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.01.00_50_post_add_heartbeat.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0033_pre_batch_defer_jobs.py b/procrastinate/alembic/versions/procrastinate_0033_pre_batch_defer_jobs.py new file mode 100644 index 000000000..7633913be --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0033_pre_batch_defer_jobs.py @@ -0,0 +1,35 @@ +"""03.02.00 01 pre batch defer jobs.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0033" +down_revision: str | None = "procrastinate_0032" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.02.00_01_pre_batch_defer_jobs.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0034_post_batch_defer_jobs.py b/procrastinate/alembic/versions/procrastinate_0034_post_batch_defer_jobs.py new file mode 100644 index 000000000..af9d811bd --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0034_post_batch_defer_jobs.py @@ -0,0 +1,35 @@ +"""03.02.00 50 post batch defer jobs.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0034" +down_revision: str | None = "procrastinate_0033" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.02.00_50_post_batch_defer_jobs.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0035_pre_priority_lock_fetch_job.py b/procrastinate/alembic/versions/procrastinate_0035_pre_priority_lock_fetch_job.py new file mode 100644 index 000000000..b88b0ef93 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0035_pre_priority_lock_fetch_job.py @@ -0,0 +1,35 @@ +"""03.03.00 01 pre priority lock fetch job.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0035" +down_revision: str | None = "procrastinate_0034" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.03.00_01_pre_priority_lock_fetch_job.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0036_pre_add_retry_failed_job_procedure.py b/procrastinate/alembic/versions/procrastinate_0036_pre_add_retry_failed_job_procedure.py new file mode 100644 index 000000000..11809cc63 --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0036_pre_add_retry_failed_job_procedure.py @@ -0,0 +1,35 @@ +"""03.04.00 01 pre add retry failed job procedure.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0036" +down_revision: str | None = "procrastinate_0035" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.04.00_01_pre_add_retry_failed_job_procedure.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/alembic/versions/procrastinate_0037_post_add_retry_failed_job_procedure.py b/procrastinate/alembic/versions/procrastinate_0037_post_add_retry_failed_job_procedure.py new file mode 100644 index 000000000..cf1c1ebbd --- /dev/null +++ b/procrastinate/alembic/versions/procrastinate_0037_post_add_retry_failed_job_procedure.py @@ -0,0 +1,35 @@ +"""03.04.00 50 post add retry failed job procedure.""" + +from __future__ import annotations + +from importlib import resources + +import sqlalchemy as sa +from alembic import op + +revision = "procrastinate_0037" +down_revision: str | None = "procrastinate_0036" +branch_labels = None +depends_on = None + +MIGRATION_FILE = "03.04.00_50_post_add_retry_failed_job_procedure.sql" + + +def _migration_sql() -> sa.TextClause: + sql = ( + resources.files("procrastinate.sql.migrations") + .joinpath(MIGRATION_FILE) + .read_text(encoding="utf-8") + ) + return sa.text(sql.replace(":", r"\:")) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.execute(_migration_sql()) + + +def downgrade() -> None: + raise NotImplementedError( + "Procrastinate Alembic revisions wrap irreversible SQL migrations." + ) diff --git a/procrastinate/schema.py b/procrastinate/schema.py index f393fae63..9b593ab8f 100644 --- a/procrastinate/schema.py +++ b/procrastinate/schema.py @@ -9,6 +9,7 @@ from procrastinate import connector as connector_module migrations_path = pathlib.Path(__file__).parent / "sql" / "migrations" +alembic_versions_path = pathlib.Path(__file__).parent / "alembic" / "versions" class SchemaManager: @@ -29,6 +30,10 @@ def get_schema() -> LiteralString: def get_migrations_path() -> str: return str(migrations_path) + @staticmethod + def get_alembic_versions_path() -> str: + return str(alembic_versions_path) + def apply_schema(self) -> None: queries = self.get_schema() queries = queries.replace("%", "%%") diff --git a/pyproject.toml b/pyproject.toml index 367c2f7f1..c45dce4ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ ] [project.optional-dependencies] +alembic = ["alembic"] django = ["django>=2.2"] sqlalchemy = ["sqlalchemy~=2.0"] aiopg = ["aiopg", "psycopg2-binary"] @@ -71,6 +72,7 @@ release = ["dunamai"] lint_format = ["ruff", "django-upgrade"] pg_implem = ["aiopg", "sqlalchemy", "psycopg2-binary", "psycopg[binary,pool]"] test = [ + "alembic", "pytest-asyncio", "pytest-benchmark", "pytest-cov", diff --git a/tests/conftest.py b/tests/conftest.py index 2f41d32ea..8ce045316 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -104,7 +104,7 @@ def cursor_execute(cursor, query, *identifiers): query = psycopg.sql.SQL(query).format( *(psycopg.sql.Identifier(identifier) for identifier in identifiers) ) - cursor.execute(query) + return cursor.execute(query) @contextlib.contextmanager diff --git a/tests/migration/test_migration.py b/tests/migration/test_migration.py index ecb50c6d6..af8f2ee50 100644 --- a/tests/migration/test_migration.py +++ b/tests/migration/test_migration.py @@ -1,8 +1,10 @@ from __future__ import annotations +import ast import contextlib import pathlib import subprocess +import textwrap import warnings import packaging.version @@ -68,6 +70,14 @@ def django_db(db): yield db +@pytest.fixture +def alembic_database(db_factory): + dbname = "procrastinate_alembic" + db_factory(dbname=dbname) + + return dbname + + def test_migration(schema_database, migrations_database, run_migrations): # apply the migrations on the migrations_database database run_migrations(migrations_database) @@ -92,6 +102,143 @@ def test_migration(schema_database, migrations_database, run_migrations): assert not m.statements +def make_alembic_config(tmp_path, dbname, *version_locations): + alembic_config = pytest.importorskip("alembic.config") + + script_location = tmp_path / "alembic" + script_location.mkdir() + (script_location / "env.py").write_text( + textwrap.dedent( + """ + from alembic import context + from sqlalchemy import engine_from_config, pool + + config = context.config + target_metadata = None + + + def run_migrations_online(): + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection) + with context.begin_transaction(): + context.run_migrations() + + + run_migrations_online() + """ + ), + encoding="utf-8", + ) + + config = alembic_config.Config() + config.set_main_option("script_location", str(script_location)) + config.set_main_option("sqlalchemy.url", f"postgresql+psycopg:///{dbname}") + config.set_main_option("path_separator", "space") + config.set_main_option( + "version_locations", " ".join(str(path) for path in version_locations) + ) + return config + + +def run_alembic_migrations(tmp_path, dbname, *version_locations): + alembic_command = pytest.importorskip("alembic.command") + + config = make_alembic_config(tmp_path, dbname, *version_locations) + alembic_command.upgrade(config, "heads") + + +def test_alembic_migration(schema_database, alembic_database, db_execute, tmp_path): + run_alembic_migrations( + tmp_path, + alembic_database, + schema.SchemaManager.get_alembic_versions_path(), + ) + + # Alembic's own bookkeeping table is not part of Procrastinate's schema. + with db_execute(alembic_database) as execute: + execute("DROP TABLE alembic_version") + + with contextlib.ExitStack() as stack: + schema_db_session = stack.enter_context( + S(f"postgresql:///{schema_database}", poolclass=NullPool) + ) + alembic_db_session = stack.enter_context( + S(f"postgresql:///{alembic_database}", poolclass=NullPool) + ) + m = Migration(alembic_db_session, schema_db_session) + m.set_safety(False) + m.add_all_changes() + + print(m.sql) + assert not m.statements + + +def test_alembic_multiple_version_locations(db_factory, db_execute, tmp_path): + dbname = "procrastinate_alembic_multi_base" + db_factory(dbname=dbname) + user_versions = tmp_path / "user_versions" + user_versions.mkdir() + (user_versions / "user_0001.py").write_text( + textwrap.dedent( + """ + from __future__ import annotations + + from alembic import op + + revision = "user_0001" + down_revision = None + branch_labels = ("user",) + depends_on = None + + + def upgrade() -> None: + op.execute("CREATE TABLE user_alembic_revision (id integer PRIMARY KEY)") + + + def downgrade() -> None: + op.execute("DROP TABLE user_alembic_revision") + """ + ), + encoding="utf-8", + ) + + run_alembic_migrations( + tmp_path, + dbname, + user_versions, + schema.SchemaManager.get_alembic_versions_path(), + ) + + procrastinate_head = sorted( + pathlib.Path(schema.SchemaManager.get_alembic_versions_path()).glob( + "procrastinate_*.py" + ) + )[-1] + module = ast.parse(procrastinate_head.read_text(encoding="utf-8")) + procrastinate_head_revision = next( + node.value.value + for node in module.body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "revision" + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ) + with db_execute(dbname) as execute: + execute("SELECT * FROM procrastinate_jobs") + execute("SELECT * FROM user_alembic_revision") + cursor = execute("SELECT version_num FROM alembic_version") + heads = {row[0] for row in cursor.fetchall()} + + assert heads == {procrastinate_head_revision, "user_0001"} + + def test_django_migrations_run_properly(django_db): # At this point, with the db fixture, we have all migrations applied with connection.cursor() as cursor: diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 8915c9e87..15f661cd9 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pathlib from collections import defaultdict @@ -11,6 +12,13 @@ def test_get_migrations_path(app): assert app.schema_manager.get_migrations_path().endswith("sql/migrations") +def test_get_alembic_versions_path(app): + path = pathlib.Path(app.schema_manager.get_alembic_versions_path()) + + assert path.name == "versions" + assert path.parent.name == "alembic" + + def test_apply_schema(app, connector): connector.reverse_queries = defaultdict(lambda: "apply_schema") connector.set_schema_version_run = lambda *a, **kw: None diff --git a/uv.lock b/uv.lock index 73fe82f04..86098c009 100644 --- a/uv.lock +++ b/uv.lock @@ -41,6 +41,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + [[package]] name = "anyio" version = "4.12.1" @@ -688,6 +703,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "mako" +version = "1.3.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/62/791b31e69ae182791ec67f04850f2f062716bbd205483d63a215f3e062d3/mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a", size = 400219, upload-time = "2026-04-28T19:01:08.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -974,6 +1001,9 @@ aiopg = [ { name = "aiopg" }, { name = "psycopg2-binary" }, ] +alembic = [ + { name = "alembic" }, +] django = [ { name = "django", version = "5.2.14", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "django", version = "6.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1029,6 +1059,7 @@ release = [ { name = "dunamai" }, ] test = [ + { name = "alembic" }, { name = "migra" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, @@ -1044,6 +1075,7 @@ types = [ [package.metadata] requires-dist = [ { name = "aiopg", marker = "extra == 'aiopg'" }, + { name = "alembic", marker = "extra == 'alembic'" }, { name = "asgiref" }, { name = "attrs" }, { name = "croniter" }, @@ -1057,7 +1089,7 @@ requires-dist = [ { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = "~=2.0" }, { name = "typing-extensions" }, ] -provides-extras = ["aiopg", "django", "psycopg2", "sphinx", "sqlalchemy"] +provides-extras = ["aiopg", "alembic", "django", "psycopg2", "sphinx", "sqlalchemy"] [package.metadata.requires-dev] dev = [ @@ -1092,6 +1124,7 @@ pg-implem = [ ] release = [{ name = "dunamai" }] test = [ + { name = "alembic" }, { name = "migra" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" },