Worker sends real-time progress events to client

- update appraisal service object to generate unique request id
- update API and worker to use Faye to mediate websocket messages

Gemfile

@@ -45,11 +45,6 @@ gem 'hirb'
 # gem 'hirb-unicode' # incompatible with new rubocop
 gem 'sequel', '~> 5.0'
 
-# Asynchronicity
-gem 'concurrent-ruby', '~> 1.1'
-gem 'aws-sdk-sqs', '~> 1.0'
-gem 'shoryuken', '~> 5.0'
-
 group :development, :test do
   gem 'sqlite3', '~> 1.0'
 end
@@ -58,6 +53,14 @@ group :production do
   gem 'pg', '~> 1.2'
 end
 
+# Asynchronicity
+gem 'concurrent-ruby', '~> 1.1'
+gem 'aws-sdk-sqs', '~> 1.48'
+
+# WORKER
+gem 'shoryuken', '~> 5.3'
+gem 'faye', '~> 1.4'
+
 # TESTING
 group :test do
   # API Unit/Integration/Acceptance Tests

Gemfile.lock

@@ -5,7 +5,7 @@ GEM
       public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.1014.0)
+    aws-partitions (1.1018.0)
     aws-sdk-core (3.214.0)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.992.0)
@@ -20,6 +20,7 @@ GEM
     bigdecimal (3.1.8)
     coderay (1.1.3)
     concurrent-ruby (1.3.4)
+    cookiejar (0.3.4)
     crack (1.0.0)
       bigdecimal
       rexml
@@ -79,6 +80,27 @@ GEM
       dry-initializer (~> 3.0)
       dry-schema (>= 1.12, < 2)
       zeitwerk (~> 2.6)
+    em-http-request (1.1.7)
+      addressable (>= 2.3.4)
+      cookiejar (!= 0.3.1)
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-socksify (0.3.3)
+      base64
+      eventmachine (>= 1.0.0.beta.4)
+    eventmachine (1.2.7)
+    faye (1.4.0)
+      cookiejar (>= 0.3.0)
+      em-http-request (>= 1.1.6)
+      eventmachine (>= 0.12.0)
+      faye-websocket (>= 0.11.0)
+      multi_json (>= 1.0.0)
+      rack (>= 1.0.0)
+      websocket-driver (>= 0.5.1)
+    faye-websocket (0.11.3)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.5.1)
     ffi (1.17.0-arm64-darwin)
     ffi (1.17.0-x86_64-linux-gnu)
     ffi-compiler (1.3.2)
@@ -98,22 +120,23 @@ GEM
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
       llhttp-ffi (~> 0.5.0)
-    http-cookie (1.0.7)
+    http-cookie (1.0.8)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
+    http_parser.rb (0.8.0)
     ice_nine (0.11.2)
     jmespath (1.6.2)
-    json (2.8.2)
+    json (2.9.0)
     language_server-protocol (3.17.0.3)
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     llhttp-ffi (0.5.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    logger (1.6.1)
+    logger (1.6.2)
     method_source (1.1.0)
-    minitest (5.25.2)
+    minitest (5.25.4)
     minitest-rg (5.3.0)
       minitest (~> 5.0)
     multi_json (1.15.0)
@@ -167,14 +190,14 @@ GEM
       representable (~> 3.1)
     roda (3.86.0)
       rack
-    rubocop (1.69.0)
+    rubocop (1.69.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.4, < 3.0)
-      rubocop-ast (>= 1.36.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.36.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
     rubocop-ast (1.36.2)
@@ -190,7 +213,7 @@ GEM
     ruby_parser (3.21.1)
       racc (~> 1.5)
       sexp_processor (~> 4.16)
-    sequel (5.86.0)
+    sequel (5.87.0)
       bigdecimal
     sexp_processor (4.17.3)
     shoryuken (5.3.2)
@@ -217,20 +240,24 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
     zeitwerk (2.7.1)
 
 PLATFORMS
   arm64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
-  aws-sdk-sqs (~> 1.0)
+  aws-sdk-sqs (~> 1.48)
   concurrent-ruby (~> 1.1)
   dry-monads (~> 1.4)
   dry-struct (~> 1.0)
   dry-transaction (~> 0.13)
   dry-types (~> 1.0)
   dry-validation (~> 1.7)
+  faye (~> 1.4)
   figaro (~> 1.2)
   flog
   hirb
@@ -258,7 +285,7 @@ DEPENDENCIES
   rubocop-rake
   rubocop-sequel
   sequel (~> 5.0)
-  shoryuken (~> 5.0)
+  shoryuken (~> 5.3)
   simplecov (~> 0.0)
   sqlite3 (~> 1.0)
   vcr (~> 6.0)

Rakefile

@@ -9,7 +9,6 @@ end
 
 desc 'Run unit and integration tests'
 Rake::TestTask.new(:spec) do |t|
-  puts 'Make sure worker is running in separate process'
   t.pattern = 'spec/tests/**/*_spec.rb'
   t.warning = false
 end
@@ -19,23 +18,23 @@ task :respec do
   sh "rerun -c 'rake spec' --ignore 'coverage/*' --ignore 'repostore/*'"
 end
 
-desc 'Run the webserver and application and restart if code changes'
-task :rerun do
-  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- rake run"
-end
-
 desc 'Run web app in default (dev) mode'
 task run: ['run:dev']
 
+desc 'Keep restarting web app in dev mode upon changes'
+task :rerun do
+  sh "rerun -c --ignore 'coverage/*' --ignore 'repostore/*' -- bundle exec puma -p 9090"
+end
+
 namespace :run do
   desc 'Run API in dev mode'
   task :dev do
-    sh 'bundle exec puma -p 9090'
+    sh 'RACK_ENV=development bundle exec puma config.ru -p 9090'
   end
 
   desc 'Run API in test mode'
   task :test do
-    sh 'RACK_ENV=test bundle exec puma -p 9090'
+    sh 'RACK_ENV=test bundle exec puma config.ru -p 9090'
   end
 end
 

app/application/controllers/app.rb

@@ -36,11 +36,17 @@ class App < Roda
                 response.cache_control public: true, max_age: 300
               end
 
+              request_id = [request.env, request.path, Time.now.to_f].hash
+
               path_request = Request::ProjectPath.new(
                 owner_name, project_name, request
               )
 
-              result = Service::AppraiseProject.new.call(requested: path_request)
+              result = Service::AppraiseProject.new.call(
+                requested: path_request,
+                request_id: request_id,
+                config: App.config
+              )
 
               if result.failure?
                 failed = Representer::HttpResponse.new(result.failure)

app/application/services/appraise_project.rb

@@ -20,8 +20,9 @@ class AppraiseProject
       CLONE_ERR = 'Could not clone this project'
       NO_FOLDER_ERR = 'Could not find that folder'
       SIZE_ERR = 'Project too large to analyze'
-      PROCESSING_MSG = 'Processing the summary request'
+      PROCESSING_MSG = 'Appraising the project'
 
+      # input hash keys expected: :project, :requested, :config
       def find_project_details(input)
         input[:project] = Repository::For.klass(Entity::Project).find_full_name(
           input[:requested].owner_name, input[:requested].project_name
@@ -40,19 +41,21 @@ def check_project_eligibility(input)
         if input[:project].too_large?
           Failure(Response::ApiResult.new(status: :bad_request, message: SIZE_ERR))
         else
-          input[:gitrepo] = GitRepo.new(input[:project])
+          input[:gitrepo] = GitRepo.new(input[:project], input[:config])
           Success(input)
         end
       end
 
       def request_cloning_worker(input)
         return Success(input) if input[:gitrepo].exists_locally?
 
-        Messaging::Queue
-          .new(App.config.CLONE_QUEUE_URL, App.config)
-          .send(Representer::Project.new(input[:project]).to_json)
+        Messaging::Queue.new(App.config.CLONE_QUEUE_URL, App.config)
+          .send(clone_request_json(input))
 
-        Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG))
+          Failure(Response::ApiResult.new(
+            status: :processing,
+            message: { request_id: input[:request_id], msg: PROCESSING_MSG }
+          ))
       rescue StandardError => e
         log_error(e)
         Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR))
@@ -80,6 +83,12 @@ def full_request_path(input)
       def log_error(error)
         App.logger.error [error.inspect, error.backtrace].flatten.join("\n")
       end
+
+      def clone_request_json(input)
+        Response::CloneRequest.new(input[:project], input[:request_id])
+          .then { Representer::CloneRequest.new(_1) }
+          .then(&:to_json)
+      end
     end
   end
 end

app/infrastructure/git/repositories/git_repo.rb

@@ -9,10 +9,10 @@ class Errors
       CannotOverwriteLocalGitRepo = Class.new(StandardError)
     end
 
-    def initialize(project, config = CodePraise::App.config)
+    def initialize(project, config)
       @project = project
       remote = Git::RemoteGitRepo.new(@project.http_url)
-      @local = Git::LocalGitRepo.new(remote)
+      @local = Git::LocalGitRepo.new(remote, config.REPOSTORE_PATH)
     end
 
     def local
@@ -27,7 +27,7 @@ def exists_locally?
       @local.exists?
     end
 
-    def clone
+    def clone_locally
       raise Errors::TooLargeToClone if @project.too_large?
       raise Errors::CannotOverwriteLocalGitRepo if exists_locally?
 

app/infrastructure/git/repositories/local_repo.rb

@@ -18,9 +18,9 @@ class LocalGitRepo
 
       attr_reader :git_repo_path
 
-      def initialize(remote)
+      def initialize(remote, repostore_path)
         @remote = remote
-        @git_repo_path = [ENV['REPOSTORE_PATH'], @remote.unique_id].join('/')
+        @git_repo_path = [repostore_path, @remote.unique_id].join('/')
       end
 
       def clone_remote

app/presentation/representers/clone_request_representer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'roar/decorator'
+require 'roar/json'
+require_relative 'project_representer'
+
+# Represents essential Repo information for API output
+module CodePraise
+  module Representer
+    # Representer object for project clone requests
+    class CloneRequest < Roar::Decorator
+      include Roar::JSON
+
+      property :project, extend: Representer::Project, class: OpenStruct
+      property :id
+    end
+  end
+end

app/presentation/representers/project_representer.rb

@@ -25,11 +25,15 @@ class Project < Roar::Decorator
       collection :contributors, extend: Representer::Member, class: OpenStruct
 
       link :self do
-        "#{App.config.API_HOST}/projects/#{project_name}/#{owner_name}"
+        "#{api_host_url}/api/v1/projects/#{project_name}/#{owner_name}"
       end
 
       private
 
+      def api_host_url
+        ENV.fetch('API_HOST', nil)
+      end
+
       def project_name
         represented.name
       end

app/presentation/responses/clone_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module CodePraise
+  module Response
+    CloneRequest = Struct.new :project, :id
+  end
+end

config.ru

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require 'faye'
 require_relative 'require_app'
 require_app
 
+use Faye::RackAdapter, mount: '/faye', timeout: 25
 run CodePraise::App.freeze.app

config/environment.rb

@@ -34,6 +34,9 @@ def self.config = Figaro.env
     end
 
     configure :production do
+      puts 'RUNNING IN PRODUCTION MODE'
+      # Set DATABASE_URL environment variable on production platform
+
       use Rack::Cache,
           verbose: true,
           metastore: "#{config.REDISCLOUD_URL}/0/metastore",

config/secrets_example.yml

@@ -10,7 +10,7 @@ development:
   AWS_ACCESS_KEY_ID: <aws credential>
   AWS_SECRET_ACCESS_KEY: <aws credential>
   AWS_REGION: <choose region>
-  CLONE_QUEUE: codepraise-clone-development
+  CLONE_QUEUE: soa-codepraise-clone-dev
   CLONE_QUEUE_URL: <set by AWS>
 
 app_test:
@@ -21,7 +21,7 @@ app_test:
   AWS_ACCESS_KEY_ID: <aws credential>
   AWS_SECRET_ACCESS_KEY: <aws credential>
   AWS_REGION: <choose region>
-  CLONE_QUEUE: codepraise-clone-development
+  CLONE_QUEUE: codepraise-clone-test
   CLONE_QUEUE_URL: <set by AWS>
 
 test:
@@ -33,7 +33,7 @@ test:
   AWS_ACCESS_KEY_ID: <aws credential>
   AWS_SECRET_ACCESS_KEY: <aws credential>
   AWS_REGION: <choose region>
-  CLONE_QUEUE: codepraise-clone-development
+  CLONE_QUEUE: codepraise-clone-test
   CLONE_QUEUE_URL: <set by AWS>
 
 production: