Updates test logging and format for Elasticsearch YAML tests

elasticsearch-api/api-spec-testing/test_file/action.rb

@@ -141,24 +141,12 @@ def perform_action(method, args, client, test)
           @response = client.send(method, arguments)
           client
         when 'headers'
-          headers = prepare_arguments(args, test)
-          host = client.transport.instance_variable_get('@hosts')
           transport_options = client.transport.instance_variable_get('@options')&.dig(:transport_options) || {}
-          if ENV['QUIET'] == 'true'
-            # todo: create a method on Elasticsearch::Client that can clone the client with new options
-            Elasticsearch::Client.new(
-              host: host,
-              transport_options: transport_options.merge(headers: headers)
-            )
-          else
-            Elasticsearch::Client.new(
-              host: host,
-              tracer: Logger.new($stdout),
-              transport_options: transport_options.merge(headers: headers)
-            )
-          end
-        when 'catch', 'warnings', 'allowed_warnings', 'allowed_warnings_regex', 'warnings_regex'
-          client
+          Elasticsearch::Client.new(
+            hosts: client.transport.hosts,
+            tracer: client.transport.tracer || nil,
+            transport_options: transport_options.merge(headers: prepare_arguments(args, test))
+          )
         when 'put_trained_model_alias'
           args.merge!('reassign' => true) unless args['reassign'] === false
           @response = client.send(method, prepare_arguments(args, test))
@@ -187,6 +175,8 @@ def perform_action(method, args, client, test)
           end
           @response = client.send(method, prepare_arguments(args, test))
           client
+        when 'catch', 'warnings', 'allowed_warnings', 'allowed_warnings_regex', 'warnings_regex'
+          client
         else
           @response = client.send(method, prepare_arguments(args, test))
           client

elasticsearch-api/api-spec-testing/wipe_cluster.rb

@@ -15,9 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require_relative 'logging'
 require_relative 'custom_cleanup'
-include Elasticsearch::RestAPIYAMLTests::Logging
 
 module Elasticsearch
   module RestAPIYAMLTests
@@ -151,7 +149,7 @@ def check_for_unexpectedly_recreated_objects(client)
           unexpected_ilm_policies = client.index_lifecycle_management.get_lifecycle
           unexpected_ilm_policies.reject! { |k, _| preserve_policy?(k) }
           unless unexpected_ilm_policies.empty?
-            logger.info(
+            client.logger.info(
               "Expected no ILM policies after deletions, but found #{unexpected_ilm_policies.keys.join(',')}"
             )
           end
@@ -166,7 +164,7 @@ def check_for_unexpectedly_recreated_objects(client)
           unexpected_templates << legacy_templates.keys.reject { |t| platinum_template?(t) }
 
           unless unexpected_templates.reject(&:empty?).empty?
-            logger.info(
+            client.logger.info(
               "Expected no templates after deletions, but found #{unexpected_templates.join(',')}"
             )
           end
@@ -211,7 +209,7 @@ def wait_for_pending_rollup_tasks(client)
             end
             break unless count.positive? && Time.now.to_i < (start_time + 1)
           end
-          logger.debug("Waited for #{count} pending rollup tasks for #{Time.now.to_i - start_time}s.") if count.positive?
+          client.logger.debug("Waited for #{count} pending rollup tasks for #{Time.now.to_i - start_time}s.") if count.positive?
         end
 
         def delete_all_slm_policies(client)
@@ -247,7 +245,7 @@ def wipe_snapshots(client)
                 response = client.snapshot.get(repository: repository, snapshot: '_all', ignore_unavailable: true)
                 response['snapshots'].each do |snapshot|
                   if snapshot['state'] != 'SUCCESS'
-                    logger.debug("Found snapshot that did not succeed #{snapshot}")
+                    client.logger.debug("Found snapshot that did not succeed #{snapshot}")
                   end
                   client.snapshot.delete(repository: repository, snapshot: snapshot['snapshot'], ignore: 404)
                 end
@@ -261,7 +259,7 @@ def wipe_datastreams(client)
           begin
             client.indices.delete_data_stream(name: '*', expand_wildcards: 'all')
           rescue StandardError => e
-            logger.error "Caught exception attempting to delete data streams: #{e}"
+            client.logger.error "Caught exception attempting to delete data streams: #{e}"
             client.indices.delete_data_stream(name: '*')
           end
         end
@@ -290,14 +288,13 @@ def wipe_all_templates(client)
           end
 
           # Always check for legacy templates
-          templates = client.indices.get_template
-          templates.each do |name, _|
+          client.indices.get_template.each_key do |name|
             next if platinum_template?(name)
 
             begin
               client.indices.delete_template(name: name)
             rescue StandardError => e
-              logger.info("Unable to remove index template #{name}")
+              client.logger.info("Unable to remove index template #{name} - #{e}")
             end
           end
         end
@@ -331,7 +328,6 @@ def wait_for_cluster_tasks(client)
             end
             break unless count.positive? && Time.now.to_i < (start_time + 5)
           end
-          logger.debug("Waited for #{count} pending cluster tasks for #{Time.now.to_i - start_time}s.") if count.positive?
         end
 
         def skippable_task?(task)

elasticsearch-api/spec/rest_api/rest_api_tests_helper.rb

@@ -48,14 +48,21 @@
   transport_options = {}
 end
 
-ADMIN_CLIENT = Elasticsearch::Client.new(host: host, transport_options: transport_options)
+# DEBUG: For easier debugging, set LOG_STDOUT env variable to true
+output = if ENV['LOG_STDOUT'] == 'true'
+           $stdout
+         else
+           File.expand_path("../../tmp/tracer_log-#{ENV['TEST_SUITE']}-#{ENV['RUBY_VERSION']}.log", __dir__)
+         end
+logger = Logger.new(output)
 
+ADMIN_CLIENT = Elasticsearch::Client.new(host: host, transport_options: transport_options, tracer: logger)
 DEFAULT_CLIENT = if ENV['QUIET'] == 'true'
                    Elasticsearch::Client.new(host: host, transport_options: transport_options)
                  else
                    Elasticsearch::Client.new(
                      host: host,
-                     tracer: Logger.new($stdout),
+                     tracer: logger,
                      transport_options: transport_options
                    )
                  end

elasticsearch-api/spec/rest_api/rest_api_yaml_spec.rb

@@ -19,6 +19,9 @@
 require_relative 'rest_api_tests_helper'
 require_relative './run_rspec_matchers'
 
+# LOGGER logs to stdout since we want to see if there are any errors in the running of the tests.
+# The client has a tracer/logger defined in ./rest_api_tests_helper.rb, ENV['LOG_STDOUT'] set to
+# true will log to stdout instead of a log file:
 LOGGER = Logger.new($stdout)
 CLUSTER_FEATURES = ADMIN_CLIENT.features.get_features['features'].map { |f| f['name'] }
 
@@ -83,7 +86,7 @@ def skip_test?(test)
             run_rspec_matchers_on_task_group(task_group, test)
           end
         rescue StandardError => e
-          LOGGER.error e
+          LOGGER.error "#{context_name} - #{e}"
           raise e
         end
       end

elasticsearch-api/spec/spec_helper.rb

@@ -58,12 +58,14 @@ def self.included(context)
 
 RSpec.configure do |config|
   config.include(HelperModule)
-  config.add_formatter('documentation')
   config.filter_run_excluding skip: true
+  config.add_formatter('progress')
   if defined?(JRUBY_VERSION)
     config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-jruby-#{JRUBY_VERSION}.html")
+    config.add_formatter('documentation', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-jruby-#{JRUBY_VERSION}.log")
   else
     config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.html")
+    config.add_formatter('documentation', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.html")
   end
   if ENV['BUILDKITE']
     require_relative "./rspec_formatter.rb"