Merge pull request #7 from ISS-SOA/feature-async

Feature async

.github/workflows/api.yml

@@ -21,6 +21,8 @@ jobs:
     strategy:
       # don't cancel other jobs if one fails
       fail-fast: false
+      # maximum number of jobs that can run simultaneously
+      max-parallel: 1
       matrix:
         os: [ubuntu, macos]
     runs-on: ${{ matrix.os }}-latest  # Runs on latest builds of matrix OSes
@@ -47,4 +49,10 @@ jobs:
           DB_FILENAME: ${{ secrets.DB_FILENAME }}
           REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }}
           API_HOST: ${{ secrets.API_HOST }}
-        run: bundle exec rake spec
+          CLONE_QUEUE_URL: ${{ secrets.CLONE_QUEUE_URL }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          rake worker:run:test &
+          bundle exec rake spec

Gemfile

@@ -45,6 +45,11 @@ gem 'hirb'
 # gem 'hirb-unicode' # incompatible with new rubocop
 gem 'sequel', '~> 5.0'
 
+# Asynchronicity
+gem 'concurrent-ruby', '~> 1.1'
+gem 'aws-sdk-sqs', '~> 1.0'
+gem 'shoryuken', '~> 5.0'
+
 group :development, :test do
   gem 'sqlite3', '~> 1.0'
 end

Gemfile.lock

@@ -4,6 +4,18 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.1014.0)
+    aws-sdk-core (3.214.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-sqs (1.89.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.10.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.2.0)
     bigdecimal (3.1.8)
     coderay (1.1.3)
@@ -90,6 +102,7 @@ GEM
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     ice_nine (0.11.2)
+    jmespath (1.6.2)
     json (2.8.2)
     language_server-protocol (3.17.0.3)
     listen (3.9.0)
@@ -142,7 +155,7 @@ GEM
       parser (~> 3.3.0)
       rainbow (>= 2.0, < 4.0)
       rexml (~> 3.1)
-    regexp_parser (2.9.2)
+    regexp_parser (2.9.3)
     representable (3.2.0)
       declarative (< 0.1.0)
       trailblazer-option (>= 0.1.1, < 0.2.0)
@@ -154,17 +167,17 @@ GEM
       representable (~> 3.1)
     roda (3.86.0)
       rack
-    rubocop (1.68.0)
+    rubocop (1.69.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.4, < 3.0)
-      rubocop-ast (>= 1.32.2, < 2.0)
+      rubocop-ast (>= 1.36.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.36.1)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.36.2)
       parser (>= 3.3.1.0)
     rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
@@ -180,6 +193,10 @@ GEM
     sequel (5.86.0)
       bigdecimal
     sexp_processor (4.17.3)
+    shoryuken (5.3.2)
+      aws-sdk-core (>= 2)
+      concurrent-ruby
+      thor
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -191,7 +208,9 @@ GEM
     thor (1.3.2)
     trailblazer-option (0.1.2)
     uber (0.1.0)
-    unicode-display_width (2.6.0)
+    unicode-display_width (3.1.2)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
     vcr (6.3.1)
       base64
     webmock (3.24.0)
@@ -205,6 +224,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aws-sdk-sqs (~> 1.0)
+  concurrent-ruby (~> 1.1)
   dry-monads (~> 1.4)
   dry-struct (~> 1.0)
   dry-transaction (~> 0.13)
@@ -237,6 +258,7 @@ DEPENDENCIES
   rubocop-rake
   rubocop-sequel
   sequel (~> 5.0)
+  shoryuken (~> 5.0)
   simplecov (~> 0.0)
   sqlite3 (~> 1.0)
   vcr (~> 6.0)

Procfile

@@ -1 +1,3 @@
+release: rake db:migrate; rake queues:create
 web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
+worker: bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml

Rakefile

@@ -9,6 +9,7 @@ end
 
 desc 'Run unit and integration tests'
 Rake::TestTask.new(:spec) do |t|
+  puts 'Make sure worker is running in separate process'
   t.pattern = 'spec/tests/**/*_spec.rb'
   t.warning = false
 end
@@ -153,6 +154,72 @@ namespace :cache do
   end
 end
 
+namespace :queues do
+  task :config do # rubocop:disable Rake/Desc
+    require 'aws-sdk-sqs'
+    require_relative 'config/environment' # load config info
+    @api = CodePraise::App
+    @sqs = Aws::SQS::Client.new(
+      access_key_id: @api.config.AWS_ACCESS_KEY_ID,
+      secret_access_key: @api.config.AWS_SECRET_ACCESS_KEY,
+      region: @api.config.AWS_REGION
+    )
+    @q_name = @api.config.CLONE_QUEUE
+    @q_url = @sqs.get_queue_url(queue_name: @q_name).queue_url
+
+    puts "Environment: #{@api.environment}"
+  end
+
+  desc 'Create SQS queue for worker'
+  task :create => :config do
+    @sqs.create_queue(queue_name: @q_name)
+
+    puts 'Queue created:'
+    puts "  Name: #{@q_name}"
+    puts "  Region: #{@api.config.AWS_REGION}"
+    puts "  URL: #{@q_url}"
+  rescue StandardError => e
+    puts "Error creating queue: #{e}"
+  end
+
+  desc 'Report status of queue for worker'
+  task :status => :config do
+    puts 'Queue info:'
+    puts "  Name: #{@q_name}"
+    puts "  Region: #{@api.config.AWS_REGION}"
+    puts "  URL: #{@q_url}"
+  rescue StandardError => e
+    puts "Error finding queue: #{e}"
+  end
+
+  desc 'Purge messages in SQS queue for worker'
+  task :purge => :config do
+    @sqs.purge_queue(queue_url: @q_url)
+    puts "Queue #{@q_name} purged"
+  rescue StandardError => e
+    puts "Error purging queue: #{e}"
+  end
+end
+
+namespace :worker do
+  namespace :run do
+    desc 'Run the background cloning worker in development mode'
+    task :dev => :config do
+      sh 'RACK_ENV=development bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_dev.yml'
+    end
+
+    desc 'Run the background cloning worker in testing mode'
+    task :test => :config do
+      sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_test.yml'
+    end
+
+    desc 'Run the background cloning worker in production mode'
+    task :production => :config do
+      sh 'RACK_ENV=production bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml'
+    end
+  end
+end
+
 desc 'Run application console'
 task :console do
   sh 'pry -r ./load_all'

app/application/controllers/app.rb

@@ -32,7 +32,9 @@ class App < Roda
           routing.on String, String do |owner_name, project_name|
             # GET /projects/{owner_name}/{project_name}[/folder_namepath/]
             routing.get do
-              response.cache_control public: true, max_age: 120
+              App.configure :production do
+                response.cache_control public: true, max_age: 300
+              end
 
               path_request = Request::ProjectPath.new(
                 owner_name, project_name, request

app/application/services/appraise_project.rb

@@ -8,8 +8,9 @@ module Service
     class AppraiseProject
       include Dry::Transaction
 
-      step :retrieve_remote_project
-      step :clone_remote
+      step :find_project_details
+      step :check_project_eligibility
+      step :request_cloning_worker
       step :appraise_contributions
 
       private
@@ -18,8 +19,10 @@ class AppraiseProject
       DB_ERR = 'Having trouble accessing the database'
       CLONE_ERR = 'Could not clone this project'
       NO_FOLDER_ERR = 'Could not find that folder'
+      SIZE_ERR = 'Project too large to analyze'
+      PROCESSING_MSG = 'Processing the summary request'
 
-      def retrieve_remote_project(input)
+      def find_project_details(input)
         input[:project] = Repository::For.klass(Entity::Project).find_full_name(
           input[:requested].owner_name, input[:requested].project_name
         )
@@ -33,13 +36,25 @@ def retrieve_remote_project(input)
         Failure(Response::ApiResult.new(status: :internal_error, message: DB_ERR))
       end
 
-      def clone_remote(input)
-        gitrepo = GitRepo.new(input[:project])
-        gitrepo.clone unless gitrepo.exists_locally?
+      def check_project_eligibility(input)
+        if input[:project].too_large?
+          Failure(Response::ApiResult.new(status: :bad_request, message: SIZE_ERR))
+        else
+          input[:gitrepo] = GitRepo.new(input[:project])
+          Success(input)
+        end
+      end
 
-        Success(input.merge(gitrepo:))
-      rescue StandardError => error
-        App.logger.error error.backtrace.join("\n")
+      def request_cloning_worker(input)
+        return Success(input) if input[:gitrepo].exists_locally?
+
+        Messaging::Queue
+          .new(App.config.CLONE_QUEUE_URL, App.config)
+          .send(Representer::Project.new(input[:project]).to_json)
+
+        Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG))
+      rescue StandardError => e
+        log_error(e)
         Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR))
       end
 
@@ -50,17 +65,21 @@ def appraise_contributions(input)
         appraisal = Response::ProjectFolderContributions.new(input[:project], input[:folder])
         Success(Response::ApiResult.new(status: :ok, message: appraisal))
       rescue StandardError
-        App.logger.error "Could not find: #{full_request_path(input)}"
+        # App.logger.error "Could not find: #{full_request_path(input)}"
         Failure(Response::ApiResult.new(status: :not_found, message: NO_FOLDER_ERR))
       end
 
-      # Helper methods
+      # Helper methods for steps
 
       def full_request_path(input)
         [input[:requested].owner_name,
          input[:requested].project_name,
          input[:requested].folder_name].join('/')
       end
+
+      def log_error(error)
+        App.logger.error [error.inspect, error.backtrace].flatten.join("\n")
+      end
     end
   end
 end

app/infrastructure/git/repositories/blame_reporter.rb

@@ -8,20 +8,20 @@ module CodePraise
   module Git
     # Git contributions report parsing and reporting services
     class BlameReporter
+      NOT_FOUND_ERROR_MSG = 'Folder not found'
+
       def initialize(gitrepo, folder_name)
         @local = gitrepo.local
         @folder_name = folder_name
         @folder_name = '' if @folder_name == '/'
       end
 
       def folder_report
-        raise('no files found in folder') if files.empty?
+        raise not_found_error unless folder_exists?
 
-        @local.in_repo do
-          files.map do |filename|
-            [filename, RepoFile.new(filename).blame]
-          end
-        end
+        filenames = @local.files.select { _1.start_with? @folder_name }
+
+        @local.in_repo { analyze_files_async(filenames) }
       end
 
       def files
@@ -32,6 +32,40 @@ def files
             @local.files.select { |file| file.start_with? "#{@folder_name}/" }
           end
       end
+
+      def folder_structure
+        @local.folder_structure
+      end
+
+      def file_report(filename)
+        Git::RepoFile.new(filename).blame
+      end
+
+      private
+
+      def folder_exists?
+        return true if @folder_name.empty?
+
+        @local.in_repo { Dir.exist? @folder_name }
+      end
+
+      def not_found_error
+        "#{NOT_FOUND_ERROR_MSG} (#{@folder_name})"
+      end
+
+      # synchronous reporting of a list of files
+      def analyze_files(filenames)
+        filenames.map { |fname| [fname, file_report(fname)] }
+      end
+
+      # asynchronous reporting of a list of files
+      def analyze_files_async(filenames)
+        filenames.map do |fname|
+          Concurrent::Promise
+            .execute { file_report(fname) }
+            .then { |freport| [fname, freport] }
+        end.map(&:value)
+      end
     end
   end
 end