fix: fail job on tests failure and include test matrix url in GitHub notifier

lib/fastlane/plugin/firebase_test_lab_android/actions/firebase_test_lab_android_action.rb

@@ -14,17 +14,20 @@ def self.run(params)
 
         # Set target project
         Helper.config(params[:project_id])
+
         # Activate service account
         Helper.authenticate(params[:gcloud_service_key_file])
+
         # Run Firebase Test Lab
-        Helper.run_tests("--type #{params[:type]} "\
+        test_run_output = Helper.run_tests("--type #{params[:type]} "\
                   "--app #{params[:app_apk]} "\
                   "#{"--test #{params[:app_test_apk]} " unless params[:app_test_apk].nil?}"\
                   "#{"--use-orchestrator " if params[:type] == "instrumentation" && params[:use_orchestrator]}"\
                   "#{params[:devices].map { |d| "--device model=#{d[:model]},version=#{d[:version]},locale=#{d[:locale]},orientation=#{d[:orientation]} " }.join}"\
                   "--timeout #{params[:timeout]} "\
                   "--results-bucket #{results_bucket} "\
                   "--results-dir #{results_dir} "\
+                  "--num-uniform-shards=#{params[:num_shards]} "\
                   "#{params[:extra_options]} "\
                   "--format=json 1>#{Helper.if_need_dir(params[:console_log_file_name])}"
         )
@@ -42,14 +45,14 @@ def self.run(params)
         #     "test_details": "--"
         #   }
         # ]
-        json = JSON.parse(File.read(params[:console_log_file_name]))
-        UI.message("Test status: #{json}")
+        json_results = JSON.parse(File.read(params[:console_log_file_name]))
+        UI.message("Test status: #{json_results}")
 
         # Fetch results
         download_dir = params[:download_dir]
         if download_dir
           UI.message("Fetch results from Firebase Test Lab results bucket")
-          json.each do |status|
+          json_results.each do |status|
             axis = status["axis_value"]
             Helper.if_need_dir("#{download_dir}/#{axis}")
             Helper.copy_from_gcs("#{results_bucket}/#{results_dir}/#{axis}", download_dir)
@@ -59,7 +62,7 @@ def self.run(params)
 
         # Notify to Slack
         if params[:slack_url]
-          success, body = Helper.make_slack_text(json)
+          success, body = Helper.make_slack_text(json_results)
           SlackNotifier.notify(params[:slack_url], body, success)
         end
 
@@ -69,13 +72,20 @@ def self.run(params)
         pr_number = params[:github_pr_number]
         api_token = params[:github_api_token]
         if owner && repository && pr_number && api_token
-          prefix, comment = Helper.make_github_text(json, params[:project_id], results_bucket, results_dir, params[:type])
+          test_url = Helper.firebase_test_lab_histories_url(params[:project_id], test_run_output.to_s)
+          test_matrix_name = Helper.get_matrix_name(test_run_output)
+
+          prefix, comment = Helper.make_github_text(json_results, results_bucket, results_dir, params[:type], test_matrix_name, test_url)
           # Delete past comments
           GitHubNotifier.delete_comments(owner, repository, pr_number, prefix, api_token)
           GitHubNotifier.put_comment(owner, repository, pr_number, comment, api_token)
         end
 
         UI.message("Finishing...")
+
+        unless Helper.is_success(json_results)
+          raise "Tests Failed 🙈"
+        end
       end
 
       def self.description
@@ -220,7 +230,13 @@ def self.available_options
                                       description: "Target directory to download screenshots from firebase",
                                       type: String,
                                       optional: true,
-                                      default_value: nil)
+                                      default_value: nil),
+         FastlaneCore::ConfigItem.new(key: :num_shards,
+                                      env_name: "NUM_SHARDS",
+                                      description: "Number of shards to use when running the tests",
+                                      type: Integer,
+                                      optional: true,
+                                      default_value: 0)
         ]
       end
 

lib/fastlane/plugin/firebase_test_lab_android/helper/Helper.rb

@@ -18,8 +18,24 @@ def self.gcs_object_url(bucket, path)
       "https://storage.googleapis.com/#{bucket}/#{CGI.escape(path)}"
     end
 
-    def self.firebase_test_lab_histories_url(project_id)
-      "https://console.firebase.google.com/u/0/project/#{project_id}/testlab/histories/"
+    def self.firebase_test_lab_histories_url(project_id, test_run_output)
+      # "https://console.firebase.google.com/u/0/project/#{project_id}/testlab/histories/"
+      url = test_run_output.match("More details are available at (.*)")[1].sub('[','').sub('].','').to_s
+
+      #first_index = url.index('*')
+      #last_index = url.rindex('*')
+      #text_to_replace = url[first_index..last_index]
+
+      return url.sub("**************", project_id)
+    end
+
+    def self.get_matrix_name(test_run_output)
+      text_with_test_matrix = test_run_output.match("(.*) has been created in the Google Cloud.")[0]
+
+      start_index = text_with_test_matrix.index('[') + 1
+      end_index = text_with_test_matrix.index(']') - 1
+
+      return text_with_test_matrix[start_index..end_index]
     end
 
     def self.config(project_id)
@@ -34,7 +50,7 @@ def self.authenticate(gcloud_key_file)
 
     def self.run_tests(arguments)
       UI.message("Test running...")
-      Action.sh("set +e; gcloud firebase test android run #{arguments}; set -e")
+      return Action.sh("set +e; gcloud beta firebase test android run #{arguments}; set -e").to_s
     end
 
     def self.copy_from_gcs(bucket_and_path, copy_to)
@@ -105,7 +121,7 @@ def self.make_slack_text(json)
       return success, body
     end
 
-    def self.make_github_text(json, project_id, bucket, dir, test_type)
+    def self.make_github_text(json, bucket, dir, test_type, test_matrix_name, test_url)
       prefix = "<img src=\"https://github.com/cats-oss/fastlane-plugin-firebase_test_lab_android/blob/master/art/firebase_test_lab_logo.png?raw=true\" width=\"65%\" loading=\"lazy\" />"
       cells = json.map { |data|
         axis = data["axis_value"]
@@ -126,7 +142,7 @@ def self.make_github_text(json, project_id, bucket, dir, test_type)
         #{prefix}
 
         ### Results
-        Firebase console: [#{project_id}](#{Helper.firebase_test_lab_histories_url(project_id)}) 
+        Firebase console: [#{test_matrix_name}](#{test_url}) 
         Test results: [#{dir}](#{Helper.gcs_result_bucket_url(bucket, dir)})
 
         | :iphone: Device | :thermometer: Status | :memo: Message | :eyes: Logcat | :japan: Sitemap | 
@@ -135,5 +151,16 @@ def self.make_github_text(json, project_id, bucket, dir, test_type)
       EOS
       return prefix, comment
     end
+
+    def self.is_success(json)
+      success = true
+
+      json.each do |status|
+        success = !is_failure(status["outcome"])
+        break unless success
+      end
+
+      return success
+    end
   end
 end