Merge pull request #93 from sleepingkingstudios/feature/sandbox

Feature/Implement RSpec::SleepingKingStudios::Sandbox.

.rubocop.yml

@@ -16,6 +16,7 @@ AllCops:
     - lib/rspec/sleeping_king_studios/configuration.rb
     - lib/rspec/sleeping_king_studios/deferred.rb
     - lib/rspec/sleeping_king_studios/deferred/**/*
+    - lib/rspec/sleeping_king_studios/sandbox.rb
     - lib/rspec/sleeping_king_studios/version.rb
     - spec/rspec/configuration_spec.rb
     - spec/integration/concerns/**/*
@@ -24,10 +25,11 @@ AllCops:
     - spec/rspec/sleeping_king_studios/concerns/memoized_helpers_spec.rb
     - spec/rspec/sleeping_king_studios/configuration_spec.rb
     - spec/rspec/sleeping_king_studios/deferred/**/*
+    - lib/rspec/sleeping_king_studios/sandbox_spec.rb
+    - lib/rspec/sleeping_king_studios/sandbox/**/*
     - spec/rspec/sleeping_king_studios/support/shared_examples/deferred_call_examples.rb
     - spec/spec_helper.rb
     - spec/support/integration/**/*.rb
-    - spec/support/sandbox.rb
     - '*.thor'
   Exclude:
     - 'tmp/**/*'
@@ -38,6 +40,7 @@ RSpec:
     - '**/*_examples.rb'
     - '**/*_contract.rb'
     - '**/*_spec.rb'
+    - '**/*_spec.fixture.rb'
 
 Layout/ArgumentAlignment:
   EnforcedStyle: with_fixed_indentation

CHANGELOG.md

@@ -21,6 +21,10 @@ Implemented the `let?` memoized helper, which defines a helper method if there i
 
 Implemented deferred examples, which provide an alternative implementation for sharing specifications between projects.
 
+### Sandbox
+
+Implemented `RSpec::SleepingKingStudios::Sandbox` for running test files in an isolated environment.
+
 ## 2.7.0
 
 ### Concerns

README.md

@@ -1009,6 +1009,53 @@ Creates a value spy that watches the value of a method call or block. The spy al
 
 **Chaining:** None.
 
+## Sandbox
+
+The `RSpec::SleepingKingStudios::Sandbox` module allows for running a spec file or files in an isolated environment and capturing the results. This can be useful for testing code meant to enhance your tests, such as a custom RSpec matcher or a shared example group.
+
+First, define a spec file to run. As a recommended convention, spec files to be run in a sandbox should be given the `spec.fixture.rb` suffix to ensure they are not accidentally run with the main test suite.
+
+```ruby
+# frozen_string_literal: true
+
+# In spec/rocket_spec.fixture.rb:
+RSpec.describe Rocket do
+  describe '#launch' do
+    it 'should launch the rocket' do
+      expect { subject.launch }.to change(subject, :launched?).to be true
+    end
+  end
+end
+```
+
+Defining fixture files rather than generating temporary files is recommended for performance reasons, but both approaches are possible. Once the file is defined, it can be run in a sandbox:
+
+```ruby
+result = RSpec::SleepingKingStudios::Sandbox.run('spec/rocket_spec.fixture.rb')
+
+result.class #=> RSpec::SleepingKingStudios::Sandbox::Result
+result.output #=> """
+# Rocket
+#   #launch
+#     should launch the rocket
+#
+# 1 example, 0 failures
+# """
+result.status #=> 1
+result.summary #=> 1 example, 0 failures
+result.example_descriptions #=> [
+#   'Rocket#launch should launch the rocket'
+# ]
+```
+
+The `.run` method returns an instance of `RSpec::SleepingKingStudios::Sandbox::Result`, which wraps the result of running the specified spec files. It defines the following methods:
+
+- `#errors`: The output captured from STDERR when running the files.
+- `#example_descriptions`: The full description for each evaluated example.
+- `#json`: The json output from running the files.
+- `#output`: The output captured from STDOUT when running the files. The specs are run with the `--format=doc` flag, so this will include the individual examples as well as the summary.
+- `#summary`: The summary line for the tests.
+
 ## Deferred Examples
 
 `RSpec::SleepingKingStudios::Deferred` provides a mechanism for defining specifications that can be reused and shared between projects. For example, a library could use deferred examples to define an interface and test a reference implementation; projects that use that library could then use the published deferred examples to validate their own implementations of that interface.

lib/rspec/sleeping_king_studios.rb

@@ -7,6 +7,7 @@ module RSpec
   module SleepingKingStudios
     autoload :Concerns, 'rspec/sleeping_king_studios/concerns'
     autoload :Deferred, 'rspec/sleeping_king_studios/deferred'
+    autoload :Sandbox,  'rspec/sleeping_king_studios/sandbox'
 
     # @return [String] the path to the installed gem.
     def self.gem_path

lib/rspec/sleeping_king_studios/sandbox.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+require 'rspec/core/sandbox'
+
+require 'rspec/sleeping_king_studios'
+
+module RSpec::SleepingKingStudios
+  # Helper for running RSpec files in isolation.
+  #
+  # Sandboxed files can be used to test enhancements to RSpec itself, such as
+  # custom matchers or shared or deferred example groups.
+  module Sandbox
+    # Value class for the result of calling a sandboxed spec file.
+    Result = Struct.new(:output, :errors, :json, :status, keyword_init: true) do
+      # @return [Array<String>] the full description for each run example.
+      def example_descriptions
+        json['examples'].map { |hsh| hsh['full_description'] }
+      end
+
+      # @return [String] the summary of the sandboxed spec run.
+      def summary
+        json['summary_line']
+      end
+    end
+
+    class << self
+      # Runs the specified spec files in a sandbox.
+      #
+      # The examples and other RSpec code in the files will *not* be added to
+      # the current RSpec process.
+      #
+      # @param files [Array<String>] the file names or patterns for the spec
+      #   files to run.
+      #
+      # @return [RSpec::SleepingKingStudios::Result] the status and output of
+      #   the spec run.
+      def run(*files) # rubocop:disable Metrics/MethodLength
+        if files.empty?
+          raise ArgumentError, 'must specify at least one file or pattern'
+        end
+
+        err    = StringIO.new
+        out    = StringIO.new
+        status = nil
+        args   = format_args(*files)
+
+        RSpec::Core::Sandbox.sandboxed do |config|
+          config.filter_run_when_matching :focus
+
+          status = RSpec::Core::Runner.run(args, err, out)
+        end
+
+        build_result(err:, out:, status:)
+      end
+
+      private
+
+      def build_result(err:, out:, status:)
+        *output, raw_json = out.string.lines
+
+        Result.new(
+          output: output.join,
+          errors: err.string,
+          json:   JSON.parse(raw_json),
+          status:
+        )
+      end
+
+      def format_args(*files)
+        [
+          '--format=json',
+          '--format=doc',
+          '--order=defined',
+          "--pattern=#{files.join(',')}"
+        ]
+      end
+    end
+  end
+end

spec/integration/deferred/custom_spec.fixture.rb

@@ -3,7 +3,7 @@
 require 'support/integration/deferred/program_examples'
 require 'support/models/rocket'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   class << self
     alias_method :custom_example, :specify
 

spec/integration/deferred/custom_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/custom_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/example_groups_spec.fixture.rb

@@ -3,7 +3,7 @@
 require 'support/integration/deferred/launch_examples'
 require 'support/models/rocket'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   subject(:rocket) { described_class.new('Imp IV') }
 
   include Spec::Integration::Deferred::LaunchExamples

spec/integration/deferred/example_groups_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/example_groups_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/examples_spec.fixture.rb

@@ -6,7 +6,7 @@
 RSpec.describe Spec::Models::Rocket do
   subject(:rocket) { described_class.new('Imp IV') }
 
-  describe '#name' do
+  describe '#name' do # rubocop:disable RSpec/EmptyExampleGroup
     include Spec::Integration::Deferred::NamingExamples
   end
 end

spec/integration/deferred/examples_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/examples_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/helpers_spec.fixture.rb

@@ -2,7 +2,7 @@
 
 require 'support/integration/deferred/payload_examples'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   subject(:rocket) { described_class.new('Imp IV') }
 
   include Spec::Integration::Deferred::PayloadExamples

spec/integration/deferred/helpers_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/helpers_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/hooks_inheritance_spec.fixture.rb

@@ -2,6 +2,6 @@
 
 require 'support/integration/deferred/hooks_inheritance_examples'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   include Spec::Integration::Deferred::HooksInheritanceExamples
 end

spec/integration/deferred/hooks_inheritance_spec.rb

@@ -3,14 +3,13 @@
 require 'rspec/sleeping_king_studios/deferred/examples'
 
 require 'support/integration/deferred/hooks_inheritance_examples'
-require 'support/sandbox'
 
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/hooks_inheritance_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:recorded) do
     Spec::Integration::Deferred::HooksInheritanceExamples::Recorder

spec/integration/deferred/hooks_ordering_spec.fixture.rb

@@ -2,6 +2,6 @@
 
 require 'support/integration/deferred/hooks_ordering_examples'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   include Spec::Integration::Deferred::HooksOrderingExamples
 end

spec/integration/deferred/hooks_ordering_spec.rb

@@ -3,14 +3,13 @@
 require 'rspec/sleeping_king_studios/deferred/examples'
 
 require 'support/integration/deferred/hooks_ordering_examples'
-require 'support/sandbox'
 
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/hooks_ordering_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:recorded) do
     Spec::Integration::Deferred::HooksOrderingExamples::Recorder

spec/integration/deferred/inheritance_spec.fixture.rb

@@ -3,7 +3,7 @@
 require 'support/integration/deferred/rocket_examples'
 require 'support/models/rocket'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   subject(:rocket) { described_class.new('Imp IV') }
 
   let(:expected_type) { :rocket }

spec/integration/deferred/inheritance_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/inheritance_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/missing_spec.fixture.rb

@@ -3,7 +3,7 @@
 require 'support/integration/deferred/ordinal_examples'
 require 'support/models/rocket'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   class << self
     alias_method :custom_example, :specify
 

spec/integration/deferred/missing_spec.rb

@@ -2,14 +2,12 @@
 
 require 'rspec/sleeping_king_studios/deferred/examples'
 
-require 'support/sandbox'
-
 RSpec.describe RSpec::SleepingKingStudios::Deferred::Examples do
   let(:fixture_file) do
     %w[spec/integration/deferred/missing_spec.fixture.rb]
   end
   let(:result) do
-    Spec::Support::Sandbox.run(fixture_file)
+    RSpec::SleepingKingStudios::Sandbox.run(fixture_file)
   end
   let(:expected_examples) do
     <<~EXAMPLES.lines.map(&:strip)

spec/integration/deferred/optional_helpers_spec.fixture.rb

@@ -2,7 +2,7 @@
 
 require 'support/integration/deferred/orbit_examples'
 
-RSpec.describe Spec::Models::Rocket do
+RSpec.describe Spec::Models::Rocket do # rubocop:disable RSpec/EmptyExampleGroup
   subject(:rocket) { described_class.new('Imp IV') }
 
   include Spec::Integration::Deferred::OrbitExamples