diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34131270..61eb1ccd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,8 +30,8 @@ jobs: strategy: fail-fast: false matrix: - ruby: ["2.7", "3.0", "3.1", "3.2"] - rails: ["6.1", "7.0"] + ruby: ["3.1", "3.2", "3.3"] + rails: ["6.1", "7.0", "7.1", "main"] continue-on-error: [false] exclude: - ruby: "3.2" diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..e04626c3 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +Caleb Hearth +Derek Prior +Devon Estes diff --git a/Gemfile b/Gemfile index 5c270e1f..0f391c00 100644 --- a/Gemfile +++ b/Gemfile @@ -3,14 +3,13 @@ source "https://rubygems.org" # Specify your gem's dependencies in scenic.gemspec gemspec -rails_version = ENV.fetch("RAILS_VERSION", "6.1") +rails_version = ENV.fetch("RAILS_VERSION", "7.1") -rails_constraint = if rails_version == "master" +rails_constraint = if rails_version == "main" {github: "rails/rails"} else "~> #{rails_version}.0" end -gem "rails", rails_constraint -gem "sprockets", "< 4.0.0" -gem "pg", "~> 1.1" +gem "activerecord", rails_constraint +gem "railties", rails_constraint diff --git a/lib/scenic/adapters/postgres/connection.rb b/lib/scenic/adapters/postgres/connection.rb index c62f9d4f..463c67c9 100644 --- a/lib/scenic/adapters/postgres/connection.rb +++ b/lib/scenic/adapters/postgres/connection.rb @@ -46,6 +46,8 @@ def postgresql_version end end + def with_connection(&) = connection_pool.with_connection(&) + private def undecorated_connection diff --git a/lib/scenic/schema_dumper.rb b/lib/scenic/schema_dumper.rb index 68aab1b2..ebaf5232 100644 --- a/lib/scenic/schema_dumper.rb +++ b/lib/scenic/schema_dumper.rb @@ -3,6 +3,34 @@ module Scenic # @api private module SchemaDumper + # A hash to do topological sort + class TSortableHash < Hash + include TSort + + alias_method :tsort_each_node, :each_key + def tsort_each_child(node, &) + fetch(node).each(&) + end + end + + # Query for the dependencies between views + DEPENDENT_SQL = <<~SQL.freeze + SELECT distinct dependent_ns.nspname AS dependent_schema + , dependent_view.relname AS dependent_view + , source_ns.nspname AS source_schema + , source_table.relname AS source_table + FROM pg_depend + JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid + JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid + JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid + JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace + JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace + WHERE dependent_ns.nspname = ANY (current_schemas(false)) AND source_ns.nspname = ANY (current_schemas(false)) + AND source_table.relname != dependent_view.relname + AND source_table.relkind IN ('m', 'v') AND dependent_view.relkind IN ('m', 'v') + ORDER BY dependent_view.relname; + SQL + def tables(stream) super views(stream) @@ -22,11 +50,47 @@ def views(stream) private def dumpable_views_in_database - @dumpable_views_in_database ||= Scenic.database.views.reject do |view| - ignored?(view.name) + @ordered_dumpable_views_in_database ||= begin + existing_views = Scenic.database.views.reject do |view| + ignored?(view.name) + end + + tsorted_views(existing_views.map(&:name)).map do |view_name| + existing_views.find do |ev| + ev.name == view_name || ev.name == view_name.split(".").last + end + end.compact end end + # When dumping the views, their order must be topologically + # sorted to take into account dependencies + def tsorted_views(views_names) + views_hash = TSortableHash.new + + ::Scenic.database.execute(DEPENDENT_SQL).each do |relation| + source_v = [ + relation["source_schema"], + relation["source_table"] + ].compact.join(".") + dependent = [ + relation["dependent_schema"], + relation["dependent_view"] + ].compact.join(".") + views_hash[dependent] ||= [] + views_hash[source_v] ||= [] + views_hash[dependent] << source_v + views_names.delete(relation["source_table"]) + views_names.delete(relation["dependent_view"]) + end + + # after dependencies, there might be some views left + # that don't have any dependencies + views_names.sort.each { |v| views_hash[v] ||= [] } + + views_hash.tsort + end + unless ActiveRecord::SchemaDumper.private_instance_methods(false).include?(:ignored?) # This method will be present in Rails 4.2.0 and can be removed then. def ignored?(table_name) diff --git a/scenic.gemspec b/scenic.gemspec index edafe043..b440a214 100644 --- a/scenic.gemspec +++ b/scenic.gemspec @@ -22,15 +22,15 @@ Gem::Specification.new do |spec| spec.add_development_dependency "database_cleaner" spec.add_development_dependency "rake" spec.add_development_dependency "rspec", ">= 3.3" - spec.add_development_dependency "pg", "~> 0.19" - spec.add_development_dependency "pry" + spec.add_development_dependency "pg", "~> 1.1" spec.add_development_dependency "ammeter", ">= 1.1.3" spec.add_development_dependency "yard" spec.add_development_dependency "redcarpet" spec.add_development_dependency "standard" + spec.add_development_dependency "sprockets", "< 4.0.0" spec.add_dependency "activerecord", ">= 4.0.0" spec.add_dependency "railties", ">= 4.0.0" - spec.required_ruby_version = ">= 2.3.0" + spec.required_ruby_version = ">= 3.1" end diff --git a/spec/scenic/schema_dumper_spec.rb b/spec/scenic/schema_dumper_spec.rb index 02378110..67402673 100644 --- a/spec/scenic/schema_dumper_spec.rb +++ b/spec/scenic/schema_dumper_spec.rb @@ -7,14 +7,15 @@ class SearchInAHaystack < ActiveRecord::Base end describe Scenic::SchemaDumper, :db do - it "dumps a create_view for a view in the database" do - view_definition = "SELECT 'needle'::text AS haystack" - Search.connection.create_view :searches, sql_definition: view_definition + let(:output) do stream = StringIO.new - ActiveRecord::SchemaDumper.dump(Search.connection, stream) + stream.string + end - output = stream.string + it "dumps a create_view for a view in the database" do + view_definition = "SELECT 'needle'::text AS haystack" + Search.connection.create_view :searches, sql_definition: view_definition expect(output).to include 'create_view "searches", sql_definition: <<-SQL' expect(output).to include view_definition @@ -29,11 +30,7 @@ class SearchInAHaystack < ActiveRecord::Base it "accurately dumps create view statements with a regular expression" do view_definition = "SELECT 'needle'::text AS haystack WHERE 'a2z' ~ '\\d+'" Search.connection.create_view :searches, sql_definition: view_definition - stream = StringIO.new - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - - output = stream.string expect(output).to include "~ '\\\\d+'::text" Search.connection.drop_view :searches @@ -44,12 +41,8 @@ class SearchInAHaystack < ActiveRecord::Base it "dumps a create_view for a materialized view in the database" do view_definition = "SELECT 'needle'::text AS haystack" - Search.connection.create_view :searches, materialized: true, sql_definition: view_definition - stream = StringIO.new - - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - - output = stream.string + Search.connection.create_view :searches, + materialized: true, sql_definition: view_definition expect(output).to include 'create_view "searches", materialized: true, sql_definition: <<-SQL' expect(output).to include view_definition @@ -58,32 +51,33 @@ class SearchInAHaystack < ActiveRecord::Base context "with views in non public schemas" do it "dumps a create_view including namespace for a view in the database" do view_definition = "SELECT 'needle'::text AS haystack" - Search.connection.execute "CREATE SCHEMA scenic; SET search_path TO scenic, public" + Search.connection.execute "CREATE SCHEMA IF NOT EXISTS scenic; SET search_path TO scenic, public" Search.connection.create_view :"scenic.searches", sql_definition: view_definition - stream = StringIO.new - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - - output = stream.string expect(output).to include 'create_view "scenic.searches",' - - Search.connection.drop_view :"scenic.searches" end it "sorts dependency order when views exist in a non-public schema" do - Search.connection.execute("CREATE SCHEMA IF NOT EXISTS scenic; SET search_path TO public, scenic") Search.connection.execute("CREATE VIEW scenic.apples AS SELECT 1;") Search.connection.execute("CREATE VIEW scenic.bananas AS SELECT 2;") Search.connection.execute("CREATE OR REPLACE VIEW scenic.apples AS SELECT * FROM scenic.bananas;") - stream = StringIO.new - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - views = stream.string.lines.grep(/create_view/).map do |view_line| + views = output.lines.grep(/create_view/).map do |view_line| view_line.match('create_view "(?.*)"')[:name] end expect(views).to eq(%w[scenic.bananas scenic.apples]) + end - Search.connection.execute("DROP SCHEMA IF EXISTS scenic CASCADE; SET search_path TO public") + before(:each) do + Search.connection.execute( + "CREATE SCHEMA IF NOT EXISTS scenic; SET search_path TO public, scenic" + ) + end + + after(:each) do + Search.connection.execute( + "DROP SCHEMA IF EXISTS scenic CASCADE; SET search_path TO public" + ) end end @@ -91,11 +85,6 @@ class SearchInAHaystack < ActiveRecord::Base with_affixed_tables(prefix: "a_", suffix: "_z") do view_definition = "SELECT 'needle'::text AS haystack" Search.connection.create_view :a_searches_z, sql_definition: view_definition - stream = StringIO.new - - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - - output = stream.string expect(output).to include 'create_view "searches"' end @@ -104,11 +93,6 @@ class SearchInAHaystack < ActiveRecord::Base it "ignores tables internal to Rails" do view_definition = "SELECT 'needle'::text AS haystack" Search.connection.create_view :searches, sql_definition: view_definition - stream = StringIO.new - - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - - output = stream.string expect(output).to include 'create_view "searches"' expect(output).not_to include "pg_stat_statements_info" @@ -118,16 +102,13 @@ class SearchInAHaystack < ActiveRecord::Base context "with views using unexpected characters in name" do it "dumps a create_view for a view in the database" do view_definition = "SELECT 'needle'::text AS haystack" - Search.connection.create_view '"search in a haystack"', sql_definition: view_definition - stream = StringIO.new - - ActiveRecord::SchemaDumper.dump(Search.connection, stream) + Search.connection.create_view '"search in a haystack"', + sql_definition: view_definition - output = stream.string expect(output).to include 'create_view "\"search in a haystack\"",' expect(output).to include view_definition - Search.connection.drop_view :'"search in a haystack"' + Search.connection.drop_view :"\"search in a haystack\"" silence_stream($stdout) { eval(output) } # standard:disable Security/Eval @@ -139,23 +120,139 @@ class SearchInAHaystack < ActiveRecord::Base it "dumps a create_view for a view in the database" do view_definition = "SELECT 'needle'::text AS haystack" Search.connection.execute( - "CREATE SCHEMA scenic; SET search_path TO scenic, public" + "CREATE SCHEMA IF NOT EXISTS scenic; SET search_path TO scenic, public" ) Search.connection.create_view 'scenic."search in a haystack"', sql_definition: view_definition - stream = StringIO.new - - ActiveRecord::SchemaDumper.dump(Search.connection, stream) - output = stream.string expect(output).to include 'create_view "scenic.\"search in a haystack\"",' expect(output).to include view_definition - Search.connection.drop_view :'scenic."search in a haystack"' + if output.include? "create_schema" + # Rails 7.1+ seem to include the schema creation in the schema dump + Search.connection.execute "DROP SCHEMA IF EXISTS scenic CASCADE" + else + Search.connection.execute 'DROP VIEW IF EXISTS scenic."search in a haystack";' + end + Search.connection.execute "SET search_path TO scenic, public;" silence_stream($stdout) { eval(output) } # standard:disable Security/Eval expect(SearchInAHaystack.take.haystack).to eq "needle" end end + + context "with views ordered by name" do + it "sorts views without dependencies" do + Search.connection.create_view "cucumber_needles", + sql_definition: "SELECT 'kukumbas'::text AS needle" + Search.connection.create_view "vip_needles", + sql_definition: "SELECT 'vip'::text AS needle" + Search.connection.create_view "none_needles", + sql_definition: "SELECT 'none_needles'::text AS needle" + + # Same here, no dependencies among existing views, all views are sorted + sorted_views = %w[cucumber_needles vip_needles none_needles].sort + + expect(views_in_output_order).to eq(sorted_views) + end + + it "sorts according to dependencies" do + Search.connection.create_table(:tasks) { |t| t.integer :performer_id } + Search.connection.create_table(:notes) do |t| + t.text :title + t.integer :author_id + end + Search.connection.create_table(:users) { |t| t.text :nickname } + Search.connection.create_table(:roles) do |t| + t.text :name + t.integer :user_id + end + + Search.connection.create_view "recent_tasks", sql_definition: <<-SQL + SELECT id, performer_id FROM tasks WHERE id > 42 + SQL + Search.connection.create_view "old_roles", sql_definition: <<-SQL + SELECT id, name FROM roles WHERE id > 56 + SQL + Search.connection.create_view "nirvana_notes", sql_definition: <<-SQL + SELECT notes.id, notes.title, notes.author_id, old_roles.name + FROM notes + JOIN old_roles ON notes.author_id = old_roles.id + SQL + Search.connection.create_view "angry_zombies", sql_definition: <<-SQL + SELECT + users.nickname, + recent_tasks.id AS task_id, + nirvana_notes.title AS talk + FROM users + JOIN roles ON roles.user_id = users.id + JOIN nirvana_notes ON nirvana_notes.author_id = roles.id + JOIN recent_tasks ON recent_tasks.performer_id = roles.id + SQL + Search.connection.create_view "doctor_zombies", sql_definition: <<-SQL + SELECT id FROM old_roles WHERE name LIKE '%dr%' + SQL + Search.connection.create_view "xenomorphs", sql_definition: <<-SQL + SELECT id, name FROM roles WHERE name = 'xeno' + SQL + Search.connection.create_view "important_messages", sql_definition: <<-SQL + SELECT id, title FROM notes WHERE title LIKE '%important%' + SQL + + # converted with https://github.com/ggerganov/dot-to-ascii + # digraph { + # rankdir = "bt"; + # xenomorphs; + # important_messages; + # doctor_zombies -> old_roles; + # nirvana_notes -> old_roles; + # angry_zombies -> nirvana_notes; + # angry_zombies -> recent_tasks; + # } + # + # +--------------------+ + # | doctor_zombies | + # +--------------------+ + # | + # | + # v + # +--------------------+ + # | old_roles | + # +--------------------+ + # ^ + # | + # | + # +--------------------+ + # | nirvana_notes | + # +--------------------+ + # ^ + # | + # | + # +--------------------+ +--------------+ + # | angry_zombies | --> | recent_tasks | + # +--------------------+ +--------------+ + # +--------------------+ + # | important_messages | + # +--------------------+ + # +--------------------+ + # | xenomorphs | + # +--------------------+ + expect(views_in_output_order).to eq(%w[ + old_roles + nirvana_notes + recent_tasks + angry_zombies + doctor_zombies + important_messages + xenomorphs + ]) + end + + def views_in_output_order + output.lines.grep(/create_view/).map do |view_line| + view_line.match('create_view "(?.*)"')[:name] + end + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b3dfc3a9..7b3fc3dd 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -13,8 +13,6 @@ DatabaseCleaner.strategy = :transaction config.around(:each, db: true) do |example| - ActiveRecord::SchemaMigration.create_table - DatabaseCleaner.start example.run DatabaseCleaner.clean