Introduce acceptance testing

nickcharlton · nickcharlton · commit 32178f7fb72b · 2025-10-07T14:24:38.000+01:00
This is the culmination of several months work, and at least one side quest, in laying down the foundations of a test suite which will run Administrate completely from scratch. For each scenario, we create a new Rails application in a clean environment, install Administrate and run the generators. We can then test against this fresh application. Whilst we seem to have had a good track record when it comes to past and future compatibility, the hope is that by having such an outside perspective focused test suite, this will provide much broader confidence, especially where different frontend configurations are concerned. We'll also be able to test speculatively, such as against the `main` branch of Rails which isn't something we could've done with confidence in the past with so many different components involved. To do this, we introduce the `jet_black` gem, which helps with running "black box" tests for command line tools, and `sack_race`, which helps run and manage processes inside the test suite. To run the tests themselves, we use Capybara like we do elsewhere, only pointed against the Rails project we created as part of the test. It's not expected that we'd re-use our existing Capybara tests here, but rather create new ones for each scenario. As these tests are always going to be slow, we don't run these as usual when running `rspec`, so they excluded. We'll run them in GitHub Actions as standalone jobs. Over time, it's expected that we'd create acceptance tests for all supported configurations, covering different frontend tools, to projects with complex model configurations and beyond. https://github.com/odlp/jet_black https://github.com/nickcharlton/sack_race
diff --git a/.rspec b/.rspec
@@ -1,2 +1,3 @@
 --color
 --require spec_helper
+--exclude "**/black_box/*_spec.rb"
diff --git a/Gemfile b/Gemfile
@@ -36,7 +36,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,3 +1,9 @@
+GIT
+  remote: https://github.com/nickcharlton/sack_race.git
+  revision: 89b4e5fff452aa6e48498aa356da87f8d1b65ae9
+  specs:
+    sack_race (1.0.0.rc1)
+
 PATH
   remote: .
   specs:
@@ -174,6 +180,7 @@ GEM
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
+    jet_black (0.7.1)
     jsbundling-rails (1.3.1)
       railties (>= 6.0.0)
     json (2.13.2)
@@ -426,6 +433,7 @@ DEPENDENCIES
   globalid
   i18n-tasks (= 1.0.15)
   image_processing
+  jet_black
   jsbundling-rails (~> 1.3)
   kaminari-i18n
   launchy
@@ -434,6 +442,7 @@ DEPENDENCIES
   pundit
   rack-timeout
   redcarpet
+  sack_race!
   selenium-webdriver
   sentry-rails
   sentry-ruby
diff --git a/bin/bundle b/bin/bundle
diff --git a/gemfiles/pundit21.gemfile b/gemfiles/pundit21.gemfile
@@ -36,7 +36,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/gemfiles/rails60.gemfile b/gemfiles/rails60.gemfile
@@ -40,7 +40,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/gemfiles/rails61.gemfile b/gemfiles/rails61.gemfile
@@ -39,7 +39,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/gemfiles/rails70.gemfile b/gemfiles/rails70.gemfile
@@ -37,7 +37,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/gemfiles/rails80.gemfile b/gemfiles/rails80.gemfile
@@ -37,7 +37,9 @@ group :test do
   gem "capybara"
   gem "database_cleaner"
   gem "formulaic"
+  gem "jet_black"
   gem "launchy"
+  gem "sack_race", github: "nickcharlton/sack_race"
   gem "selenium-webdriver"
   gem "shoulda-matchers"
   gem "timecop"
diff --git a/spec/black_box/default_rails_app_spec.rb b/spec/black_box/default_rails_app_spec.rb
@@ -0,0 +1,33 @@
+require "rails_helper"
+
+RSpec.describe "with a Default Rails app" do
+  it "works" do
+    create_rails_application
+    setup_post_model
+    setup_administrate
+
+    app.start_and_wait_for_ready
+
+    visit("/admin")
+
+    expect(page).to have_css("header h1#page-title", text: "Posts")
+    click_on "New post"
+
+    fill_in "Body", with: "Some words!"
+    fill_in "Title", with: "A post with words"
+    click_button "Create Post"
+
+    expect(page).to have_content("Post was successfully created.")
+    expect(page).to have_content("Show Post #1")
+
+    app.stop
+  end
+
+  def session
+    @session ||= JetBlack::Session.new(options: {clean_bundler_env: true})
+  end
+
+  def app
+    @app ||= RailsProcess.new(session)
+  end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
@@ -1,4 +1,5 @@
-# http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require "jet_black/rspec"
+
 RSpec.configure do |config|
   config.expect_with :rspec do |expectations|
     expectations.syntax = :expect
diff --git a/spec/support/acceptance_helpers.rb b/spec/support/acceptance_helpers.rb
@@ -0,0 +1,65 @@
+module AcceptanceHelpers
+  def create_rails_application
+    session.create_file("Gemfile", <<~RUBY)
+      source "http://rubygems.org"
+
+      gem "rails"
+    RUBY
+
+    session.run("bundle install")
+
+    rails_new_cmd = [
+      "bundle exec rails new .",
+      "--skip-test",
+      "--skip-spring",
+      "--force"
+    ].join(" ")
+
+    expect(session.run(rails_new_cmd))
+      .to be_a_success.and have_stdout("force  Gemfile")
+  end
+
+  def gem_path
+    File.expand_path(Rails.root + "../../")
+  end
+
+  def setup_administrate
+    session.append_to_file("Gemfile", "gem \"administrate\", path: \"#{gem_path}\"")
+    session.run("bundle install")
+
+    generate_dashboards_run = session.run(
+      "bundle exec rails g administrate:install"
+    )
+
+    expect(generate_dashboards_run)
+      .to be_a_success.and have_stdout("create  app/dashboards/post_dashboard.rb")
+  end
+
+  def setup_post_model
+    new_post_cmd = session.run([
+      "bundle exec rails g model post",
+      "title:string",
+      "body:text",
+      "published_at:datetime"
+    ].join(" "))
+
+    expect(new_post_cmd)
+      .to be_a_success.and have_stdout("create    app/models/post.rb")
+
+    migrate_run = session.run("bundle exec rails db:migrate")
+
+    expect(migrate_run).to be_a_success.and have_stdout("CreatePosts: migrated")
+  end
+end
+
+RSpec.configure do |config|
+  config.include Capybara::DSL, type: :black_box
+  config.include Capybara::RSpecMatchers, type: :black_box
+  config.include AcceptanceHelpers, type: :black_box
+
+  config.before(:each, type: :black_box) do
+    Capybara.current_driver = :chrome
+    Capybara.run_server = false
+    Capybara.app_host = "http://localhost:3000"
+  end
+end
diff --git a/spec/support/rails_process.rb b/spec/support/rails_process.rb
@@ -0,0 +1,31 @@
+class RailsProcess
+  def initialize(session)
+    @session = session
+    @process =
+      Bundler.with_unbundled_env do
+        SackRace::Process.new(
+          "bundle exec rails server",
+          {
+            chdir: session.directory,
+            verbose: true
+          }
+        )
+      end
+  end
+
+  def start_and_wait_for_ready
+    Bundler.with_unbundled_env do
+      process.add_handler(:ready, "Use Ctrl-C to stop\n")
+      process.start
+      process.wait_for_handler(:ready)
+    end
+  end
+
+  def stop
+    process.stop
+  end
+
+  private
+
+  attr_reader :process, :session
+end

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`--color`
`2`	`2`	`--require spec_helper`
	`3`	`+--exclude "*/black_box/_spec.rb"`