diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index 7738f60..791a00d 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -21,6 +21,8 @@ jobs: strategy: # don't cancel other jobs if one fails fail-fast: false + # maximum number of jobs that can run simultaneously + max-parallel: 1 matrix: os: [ubuntu, macos] runs-on: ${{ matrix.os }}-latest # Runs on latest builds of matrix OSes @@ -47,4 +49,10 @@ jobs: DB_FILENAME: ${{ secrets.DB_FILENAME }} REPOSTORE_PATH: ${{ secrets.REPOSTORE_PATH }} API_HOST: ${{ secrets.API_HOST }} - run: bundle exec rake spec + CLONE_QUEUE_URL: ${{ secrets.CLONE_QUEUE_URL }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + rake worker:run:test & + bundle exec rake spec \ No newline at end of file diff --git a/Gemfile b/Gemfile index 85219c5..6d6767a 100644 --- a/Gemfile +++ b/Gemfile @@ -45,6 +45,11 @@ gem 'hirb' # gem 'hirb-unicode' # incompatible with new rubocop gem 'sequel', '~> 5.0' +# Asynchronicity +gem 'concurrent-ruby', '~> 1.1' +gem 'aws-sdk-sqs', '~> 1.0' +gem 'shoryuken', '~> 5.0' + group :development, :test do gem 'sqlite3', '~> 1.0' end diff --git a/Gemfile.lock b/Gemfile.lock index ae4109d..fae0065 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,6 +4,18 @@ GEM addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) + aws-eventstream (1.3.0) + aws-partitions (1.1014.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-sqs (1.89.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) bigdecimal (3.1.8) coderay (1.1.3) @@ -90,6 +102,7 @@ GEM domain_name (~> 0.5) http-form_data (2.3.0) ice_nine (0.11.2) + jmespath (1.6.2) json (2.8.2) language_server-protocol (3.17.0.3) listen (3.9.0) @@ -142,7 +155,7 @@ GEM parser (~> 3.3.0) rainbow (>= 2.0, < 4.0) rexml (~> 3.1) - regexp_parser (2.9.2) + regexp_parser (2.9.3) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -154,17 +167,17 @@ GEM representable (~> 3.1) roda (3.86.0) rack - rubocop (1.68.0) + rubocop (1.69.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.4, < 3.0) - rubocop-ast (>= 1.32.2, < 2.0) + rubocop-ast (>= 1.36.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.36.1) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.36.2) parser (>= 3.3.1.0) rubocop-minitest (0.36.0) rubocop (>= 1.61, < 2.0) @@ -180,6 +193,10 @@ GEM sequel (5.86.0) bigdecimal sexp_processor (4.17.3) + shoryuken (5.3.2) + aws-sdk-core (>= 2) + concurrent-ruby + thor simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -191,7 +208,9 @@ GEM thor (1.3.2) trailblazer-option (0.1.2) uber (0.1.0) - unicode-display_width (2.6.0) + unicode-display_width (3.1.2) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) vcr (6.3.1) base64 webmock (3.24.0) @@ -205,6 +224,8 @@ PLATFORMS x86_64-linux DEPENDENCIES + aws-sdk-sqs (~> 1.0) + concurrent-ruby (~> 1.1) dry-monads (~> 1.4) dry-struct (~> 1.0) dry-transaction (~> 0.13) @@ -237,6 +258,7 @@ DEPENDENCIES rubocop-rake rubocop-sequel sequel (~> 5.0) + shoryuken (~> 5.0) simplecov (~> 0.0) sqlite3 (~> 1.0) vcr (~> 6.0) diff --git a/Procfile b/Procfile index 69286bd..7b12f85 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,3 @@ +release: rake db:migrate; rake queues:create web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development} +worker: bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml \ No newline at end of file diff --git a/Rakefile b/Rakefile index 385ae6c..f0db875 100644 --- a/Rakefile +++ b/Rakefile @@ -9,6 +9,7 @@ end desc 'Run unit and integration tests' Rake::TestTask.new(:spec) do |t| + puts 'Make sure worker is running in separate process' t.pattern = 'spec/tests/**/*_spec.rb' t.warning = false end @@ -153,6 +154,72 @@ namespace :cache do end end +namespace :queues do + task :config do # rubocop:disable Rake/Desc + require 'aws-sdk-sqs' + require_relative 'config/environment' # load config info + @api = CodePraise::App + @sqs = Aws::SQS::Client.new( + access_key_id: @api.config.AWS_ACCESS_KEY_ID, + secret_access_key: @api.config.AWS_SECRET_ACCESS_KEY, + region: @api.config.AWS_REGION + ) + @q_name = @api.config.CLONE_QUEUE + @q_url = @sqs.get_queue_url(queue_name: @q_name).queue_url + + puts "Environment: #{@api.environment}" + end + + desc 'Create SQS queue for worker' + task :create => :config do + @sqs.create_queue(queue_name: @q_name) + + puts 'Queue created:' + puts " Name: #{@q_name}" + puts " Region: #{@api.config.AWS_REGION}" + puts " URL: #{@q_url}" + rescue StandardError => e + puts "Error creating queue: #{e}" + end + + desc 'Report status of queue for worker' + task :status => :config do + puts 'Queue info:' + puts " Name: #{@q_name}" + puts " Region: #{@api.config.AWS_REGION}" + puts " URL: #{@q_url}" + rescue StandardError => e + puts "Error finding queue: #{e}" + end + + desc 'Purge messages in SQS queue for worker' + task :purge => :config do + @sqs.purge_queue(queue_url: @q_url) + puts "Queue #{@q_name} purged" + rescue StandardError => e + puts "Error purging queue: #{e}" + end +end + +namespace :worker do + namespace :run do + desc 'Run the background cloning worker in development mode' + task :dev => :config do + sh 'RACK_ENV=development bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_dev.yml' + end + + desc 'Run the background cloning worker in testing mode' + task :test => :config do + sh 'RACK_ENV=test bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken_test.yml' + end + + desc 'Run the background cloning worker in production mode' + task :production => :config do + sh 'RACK_ENV=production bundle exec shoryuken -r ./workers/git_clone_worker.rb -C ./workers/shoryuken.yml' + end + end +end + desc 'Run application console' task :console do sh 'pry -r ./load_all' diff --git a/app/application/controllers/app.rb b/app/application/controllers/app.rb index e269360..fe2e01f 100644 --- a/app/application/controllers/app.rb +++ b/app/application/controllers/app.rb @@ -32,7 +32,9 @@ class App < Roda routing.on String, String do |owner_name, project_name| # GET /projects/{owner_name}/{project_name}[/folder_namepath/] routing.get do - response.cache_control public: true, max_age: 120 + App.configure :production do + response.cache_control public: true, max_age: 300 + end path_request = Request::ProjectPath.new( owner_name, project_name, request diff --git a/app/application/services/appraise_project.rb b/app/application/services/appraise_project.rb index 6464724..f1f4331 100644 --- a/app/application/services/appraise_project.rb +++ b/app/application/services/appraise_project.rb @@ -8,8 +8,9 @@ module Service class AppraiseProject include Dry::Transaction - step :retrieve_remote_project - step :clone_remote + step :find_project_details + step :check_project_eligibility + step :request_cloning_worker step :appraise_contributions private @@ -18,8 +19,10 @@ class AppraiseProject DB_ERR = 'Having trouble accessing the database' CLONE_ERR = 'Could not clone this project' NO_FOLDER_ERR = 'Could not find that folder' + SIZE_ERR = 'Project too large to analyze' + PROCESSING_MSG = 'Processing the summary request' - def retrieve_remote_project(input) + def find_project_details(input) input[:project] = Repository::For.klass(Entity::Project).find_full_name( input[:requested].owner_name, input[:requested].project_name ) @@ -33,13 +36,25 @@ def retrieve_remote_project(input) Failure(Response::ApiResult.new(status: :internal_error, message: DB_ERR)) end - def clone_remote(input) - gitrepo = GitRepo.new(input[:project]) - gitrepo.clone unless gitrepo.exists_locally? + def check_project_eligibility(input) + if input[:project].too_large? + Failure(Response::ApiResult.new(status: :bad_request, message: SIZE_ERR)) + else + input[:gitrepo] = GitRepo.new(input[:project]) + Success(input) + end + end - Success(input.merge(gitrepo:)) - rescue StandardError => error - App.logger.error error.backtrace.join("\n") + def request_cloning_worker(input) + return Success(input) if input[:gitrepo].exists_locally? + + Messaging::Queue + .new(App.config.CLONE_QUEUE_URL, App.config) + .send(Representer::Project.new(input[:project]).to_json) + + Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG)) + rescue StandardError => e + log_error(e) Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR)) end @@ -50,17 +65,21 @@ def appraise_contributions(input) appraisal = Response::ProjectFolderContributions.new(input[:project], input[:folder]) Success(Response::ApiResult.new(status: :ok, message: appraisal)) rescue StandardError - App.logger.error "Could not find: #{full_request_path(input)}" + # App.logger.error "Could not find: #{full_request_path(input)}" Failure(Response::ApiResult.new(status: :not_found, message: NO_FOLDER_ERR)) end - # Helper methods + # Helper methods for steps def full_request_path(input) [input[:requested].owner_name, input[:requested].project_name, input[:requested].folder_name].join('/') end + + def log_error(error) + App.logger.error [error.inspect, error.backtrace].flatten.join("\n") + end end end end diff --git a/app/infrastructure/git/repositories/blame_reporter.rb b/app/infrastructure/git/repositories/blame_reporter.rb index 4c6fa67..458cb37 100644 --- a/app/infrastructure/git/repositories/blame_reporter.rb +++ b/app/infrastructure/git/repositories/blame_reporter.rb @@ -8,6 +8,8 @@ module CodePraise module Git # Git contributions report parsing and reporting services class BlameReporter + NOT_FOUND_ERROR_MSG = 'Folder not found' + def initialize(gitrepo, folder_name) @local = gitrepo.local @folder_name = folder_name @@ -15,13 +17,11 @@ def initialize(gitrepo, folder_name) end def folder_report - raise('no files found in folder') if files.empty? + raise not_found_error unless folder_exists? - @local.in_repo do - files.map do |filename| - [filename, RepoFile.new(filename).blame] - end - end + filenames = @local.files.select { _1.start_with? @folder_name } + + @local.in_repo { analyze_files_async(filenames) } end def files @@ -32,6 +32,40 @@ def files @local.files.select { |file| file.start_with? "#{@folder_name}/" } end end + + def folder_structure + @local.folder_structure + end + + def file_report(filename) + Git::RepoFile.new(filename).blame + end + + private + + def folder_exists? + return true if @folder_name.empty? + + @local.in_repo { Dir.exist? @folder_name } + end + + def not_found_error + "#{NOT_FOUND_ERROR_MSG} (#{@folder_name})" + end + + # synchronous reporting of a list of files + def analyze_files(filenames) + filenames.map { |fname| [fname, file_report(fname)] } + end + + # asynchronous reporting of a list of files + def analyze_files_async(filenames) + filenames.map do |fname| + Concurrent::Promise + .execute { file_report(fname) } + .then { |freport| [fname, freport] } + end.map(&:value) + end end end end diff --git a/app/infrastructure/git/repositories/git_repo.rb b/app/infrastructure/git/repositories/git_repo.rb index a95ec39..6406d3e 100644 --- a/app/infrastructure/git/repositories/git_repo.rb +++ b/app/infrastructure/git/repositories/git_repo.rb @@ -9,17 +9,16 @@ class Errors CannotOverwriteLocalGitRepo = Class.new(StandardError) end - def initialize(repo, config = CodePraise::App.config) - @repo = repo - remote = Git::RemoteGitRepo.new(@repo.http_url) - @local = Git::LocalGitRepo.new(remote, config.REPOSTORE_PATH) + def initialize(project, config = CodePraise::App.config) + @project = project + remote = Git::RemoteGitRepo.new(@project.http_url) + @local = Git::LocalGitRepo.new(remote) end def local exists_locally? ? @local : raise(Errors::NoGitRepoFound) end - # Deliberately :reek:MissingSafeMethod for file system changes def delete @local.delete end @@ -28,9 +27,8 @@ def exists_locally? @local.exists? end - # Deliberately :reek:MissingSafeMethod for file system changes def clone - raise Errors::TooLargeToClone if @repo.too_large? + raise Errors::TooLargeToClone if @project.too_large? raise Errors::CannotOverwriteLocalGitRepo if exists_locally? @local.clone_remote { |line| yield line if block_given? } diff --git a/app/infrastructure/git/repositories/local_repo.rb b/app/infrastructure/git/repositories/local_repo.rb index a37bfcc..f2f9c6a 100644 --- a/app/infrastructure/git/repositories/local_repo.rb +++ b/app/infrastructure/git/repositories/local_repo.rb @@ -18,9 +18,9 @@ class LocalGitRepo attr_reader :git_repo_path - def initialize(remote, repostore_path) + def initialize(remote) @remote = remote - @git_repo_path = [repostore_path, @remote.unique_id].join('/') + @git_repo_path = [ENV['REPOSTORE_PATH'], @remote.unique_id].join('/') end def clone_remote @@ -47,20 +47,19 @@ def exists? Dir.exist? @git_repo_path end - # Deliberately :reek:MissingSafeMethod delete def delete FileUtils.rm_rf(@git_repo_path) end + def wipe + FileUtils.rm_rf @git_repo_path + end + private def raise_unless_setup raise Errors::InvalidLocalGitRepo unless exists? end - - def wipe - FileUtils.rm_rf @git_repo_path - end end end end diff --git a/app/infrastructure/git/repositories/repo_store.rb b/app/infrastructure/git/repositories/repo_store.rb new file mode 100644 index 0000000..6ec2f4c --- /dev/null +++ b/app/infrastructure/git/repositories/repo_store.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module CodePraise + module Repository + # Collection of all local git repo clones + class RepoStore + def self.all_repos + Dir.glob(App.config.REPOSTORE_PATH + '/*') + .select { File.directory?(_1) } + end + + def self.wipe + all_repos.each { |dir| FileUtils.rm_r(dir) } + end + end + end +end diff --git a/app/infrastructure/messaging/queue.rb b/app/infrastructure/messaging/queue.rb new file mode 100644 index 0000000..8980d1b --- /dev/null +++ b/app/infrastructure/messaging/queue.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'aws-sdk-sqs' + +module CodePraise + module Messaging + ## Queue wrapper for AWS SQS + # Requires: AWS credentials loaded in ENV or through config file + class Queue + IDLE_TIMEOUT = 5 # seconds + + def initialize(queue_url, config) + @queue_url = queue_url + sqs = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + @queue = Aws::SQS::Queue.new(url: queue_url, client: sqs) + end + + ## Sends message to queue + # Usage: + # q = Messaging::Queue.new(App.config.CLONE_QUEUE_URL) + # q.send({data: "hello"}.to_json) + def send(message) + @queue.send_message(message_body: message) + end + + ## Polls queue, yielding each messge + # Usage: + # q = Messaging::Queue.new(App.config.CLONE_QUEUE_URL) + # q.poll { |msg| print msg.body.to_s } + def poll + poller = Aws::SQS::QueuePoller.new(@queue_url) + poller.poll(idle_timeout: IDLE_TIMEOUT) do |msg| + yield msg.body if block_given? + end + end + end + end +end diff --git a/app/presentation/representers/http_response_representer.rb b/app/presentation/representers/http_response_representer.rb index 08064dd..3af1990 100644 --- a/app/presentation/representers/http_response_representer.rb +++ b/app/presentation/representers/http_response_representer.rb @@ -20,7 +20,6 @@ class HttpResponse < Roar::Decorator ok: 200, created: 201, processing: 202, - no_content: 204, forbidden: 403, not_found: 404, diff --git a/app/presentation/responses/api_result.rb b/app/presentation/responses/api_result.rb index 6a273e4..7d4b7f9 100644 --- a/app/presentation/responses/api_result.rb +++ b/app/presentation/responses/api_result.rb @@ -3,7 +3,7 @@ module CodePraise module Response SUCCESS = Set.new( - %i[ok created processing no_content] + %i[ok created processing] ).freeze FAILURE = Set.new( diff --git a/config/secrets_example.yml b/config/secrets_example.yml index 942b655..410ee2c 100644 --- a/config/secrets_example.yml +++ b/config/secrets_example.yml @@ -7,12 +7,22 @@ development: REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: app_test: DB_FILENAME: db/local/test.db REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: test: DB_FILENAME: db/local/test.db @@ -20,6 +30,11 @@ test: REPOSTORE_PATH: repostore API_HOST: http://localhost:9090 REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE_URL: production: # - assign DATABASE_URL in production @@ -27,3 +42,8 @@ production: REPOSTORE_PATH: repostore API_HOST: https://codepraise2022-api REDISCLOUD_URL: url-assigned-by-Redis-provider-on-Heroku + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + CLONE_QUEUE: codepraise-clone-production + CLONE_QUEUE_URL: diff --git a/coverage/.resultset.json b/coverage/.resultset.json index e1cbec4..dcf57a3 100644 --- a/coverage/.resultset.json +++ b/coverage/.resultset.json @@ -16,7 +16,7 @@ 1, null, 1, - 54, + 57, null, null ] @@ -30,7 +30,8 @@ 1, 1, 1, - null, + 1, + 1, null, 1, null, @@ -47,6 +48,26 @@ null, 1, 0, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + null, + 1, + 0, 0, 0, null, @@ -721,11 +742,11 @@ 1, null, 1, - 4, + 7, null, null, 1, - 0, + 8, null, null, 1, @@ -737,6 +758,32 @@ null ] }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2024/api-codepraise2024/app/infrastructure/cache/redis_cache.rb": { + "lines": [ + null, + null, + 1, + null, + 1, + 1, + null, + 1, + 1, + 0, + null, + null, + 1, + 0, + null, + null, + 1, + 0, + null, + null, + null, + null + ] + }, "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2024/api-codepraise2024/app/infrastructure/database/orm/member_orm.rb": { "lines": [ null, @@ -807,7 +854,7 @@ null, null, 1, - 22, + 25, null, null, 1, @@ -835,9 +882,9 @@ null, null, 1, - 80, + 92, null, - 80, + 92, null, null, null, @@ -846,8 +893,8 @@ null, null, 1, - 20, - 60, + 23, + 69, null, null, null, @@ -878,12 +925,12 @@ null, null, null, - 21, + 24, null, null, null, null, - 21, + 24, null, null, 1, @@ -915,9 +962,9 @@ null, null, 1, - 47, + 50, null, - 20, + 23, null, null, null, @@ -963,16 +1010,16 @@ 1, null, 1, - 196, - 196, - 196, - 196, + 197, + 197, + 197, + 197, null, null, 1, - 1, - 1, - 1, + 2, + 2, + 2, null, null, 1, @@ -987,21 +1034,21 @@ null, null, 1, - 1, - 1, + 2, + 2, null, null, 1, - 1, - 1, + 2, + 2, null, null, 1, - 392, + 394, null, null, 1, - 196, + 197, null, null, null, @@ -1012,8 +1059,8 @@ null, null, 1, - 0, - 0, + 1, + 5, null, null, null, @@ -1266,6 +1313,8 @@ null, 1, 1, + null, + 1, 6, 6, 6, @@ -1274,19 +1323,51 @@ 1, 6, null, - 5, - 5, - 194, + 315, null, + 10, + null, + null, + 1, + 0, + 0, + 0, + null, + 0, null, null, null, 1, - 11, + 0, + null, + null, + 1, + 194, + null, + null, + 1, + null, + 1, 6, - 3, null, - 189, + 6, + null, + null, + 1, + 1, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 5, + 194, + 194, + 194, null, null, null, @@ -1333,30 +1414,28 @@ null, null, 1, - 6, - 6, - 6, + 10, + 10, + 10, null, null, 1, 6, null, null, - null, 1, 0, null, null, 1, - 12, - null, + 17, null, null, 1, - 0, - 0, + 1, + 1, null, - 0, + 6, null, null, null @@ -1385,33 +1464,32 @@ 1, null, 1, - 6, - 6, + 10, + 10, null, null, 1, - 0, - 0, + 6, + 1, null, null, 1, - 6, + 5, null, - 6, - 6, - 552, + 5, + 5, + 460, null, null, null, null, 1, - 11, - 22, + 13, + 26, null, null, 1, - 29, - null, + 35, null, null, 1, @@ -1419,13 +1497,13 @@ null, null, 1, + 0, null, - 1, - 17, null, + 1, null, 1, - 0, + 18, null, null, null, @@ -1450,19 +1528,40 @@ 1, null, 1, - 6, + 10, null, null, 1, - 6, + 10, null, null, 1, - 0, + 1, + null, + null, + null, null, null, null, null, + null + ] + }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2024/api-codepraise2024/app/infrastructure/git/repositories/repo_store.rb": { + "lines": [ + null, + null, + 1, + 1, + null, + 1, + 1, + 10, + 15, + null, + null, + 1, + 15, null, null, null, @@ -1677,6 +1776,52 @@ null ] }, + "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2024/api-codepraise2024/app/infrastructure/messaging/queue.rb": { + "lines": [ + null, + null, + 1, + null, + 1, + 1, + null, + null, + 1, + 1, + null, + 1, + 3, + 3, + null, + null, + null, + null, + 3, + null, + null, + null, + null, + null, + null, + 1, + 3, + null, + null, + null, + null, + null, + null, + 1, + 0, + 0, + 0, + null, + null, + null, + null, + null + ] + }, "/Users/soumyaray/Sync/Dropbox/ossdev/classes/SOA-class/projects/soa2024/api-codepraise2024/app/presentation/representers/contributor_representer.rb": { "lines": [ null, @@ -1866,9 +2011,8 @@ null, null, null, - null, 1, - 10, + 13, null, null, null, @@ -1968,17 +2112,17 @@ 1, null, 1, - 4, + 7, null, null, 1, null, 1, - 4, + 7, null, null, 1, - 4, + 7, null, null, null, @@ -2028,9 +2172,9 @@ null, 1, 1, - 27, + 30, null, - 26, + 29, null, null, null, @@ -2073,14 +2217,15 @@ 1, 1, 1, - 1, + null, + null, null, null, 1, - 10, + 13, null, null, - 10, + 13, 1, null, 1, @@ -2091,20 +2236,24 @@ 1, null, null, + 12, + 12, + 12, + null, 9, - 9, - 9, + 7, + 0, null, - 6, - 4, + null, + 7, null, null, null, - 4, + 7, null, - 4, - 2, - 2, + 7, + 5, + 5, null, null, 2, @@ -2152,7 +2301,6 @@ null, null, null, - null, null ] }, @@ -2246,16 +2394,16 @@ null, 1, 1, - 4, - 4, - 4, - 4, + 7, + 7, + 7, + 7, null, null, 1, null, 1, - 4, + 3, null, null, 1, @@ -2345,6 +2493,7 @@ 1, 1, 1, + 1, null, 1, null, @@ -2352,14 +2501,16 @@ 1, 1, 1, + 1, + 1, null, 1, - 6, + 9, null, null, null, - 6, - 4, + 9, + 7, null, 2, null, @@ -2368,10 +2519,22 @@ null, null, 1, - 4, - 4, + 7, + 0, + null, + 7, + 7, + null, null, - 4, + null, + 1, + 7, + null, + 3, + null, + null, + null, + 3, null, 0, 0, @@ -2384,16 +2547,20 @@ 3, 3, null, - 1, + null, 1, null, null, null, null, 1, - 1, + 0, + null, + null, null, null, + 1, + 0, null, null, null, @@ -2461,14 +2628,16 @@ 7, 7, 7, + 7, + 7, null, null, null, null, 1, 29, - 1709, - 1709, + 1859, + 1859, null, null, 29, @@ -2845,6 +3014,8 @@ null, null, 1, + 1, + 1, null, null, 1, @@ -3049,6 +3220,6 @@ ] } }, - "timestamp": 1732629655 + "timestamp": 1733028534 } } diff --git a/spec/helpers/vcr_helper.rb b/spec/helpers/vcr_helper.rb index b685e1c..aac161d 100644 --- a/spec/helpers/vcr_helper.rb +++ b/spec/helpers/vcr_helper.rb @@ -13,6 +13,8 @@ def self.setup_vcr vcr_config.cassette_library_dir = CASSETTES_FOLDER vcr_config.hook_into :webmock vcr_config.ignore_localhost = true # for acceptance tests + vcr_config.ignore_hosts 'sqs.us-east-1.amazonaws.com' + vcr_config.ignore_hosts 'sqs.ap-northeast-1.amazonaws.com' end end diff --git a/spec/tests/acceptance/api_spec.rb b/spec/tests/acceptance/api_spec.rb index 652578e..5b0c46f 100644 --- a/spec/tests/acceptance/api_spec.rb +++ b/spec/tests/acceptance/api_spec.rb @@ -17,6 +17,7 @@ def app before do VcrHelper.configure_vcr_for_github DatabaseHelper.wipe_database + CodePraise::Repository::RepoStore.wipe end after do @@ -35,11 +36,16 @@ def app end describe 'Appraise project folder route' do - it 'should be able to appraise a project folder' do + it 'should be able to appraise the root folder' do CodePraise::Service::AddProject.new.call( owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}" _(last_response.status).must_equal 200 appraisal = JSON.parse last_response.body @@ -58,6 +64,11 @@ def app owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/spec" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/spec" _(last_response.status).must_equal 200 appraisal = JSON.parse last_response.body @@ -76,6 +87,11 @@ def app owner_name: USERNAME, project_name: PROJECT_NAME ) + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/foobar" + _(last_response.status).must_equal 202 + + 5.times { sleep(1); print('_') } + get "/api/v1/projects/#{USERNAME}/#{PROJECT_NAME}/foobar" _(last_response.status).must_equal 404 _(JSON.parse(last_response.body)['status']).must_include 'not' diff --git a/spec/tests/integration/services/appraise_project_spec.rb b/spec/tests/integration/services/appraise_project_spec.rb index d4bac04..b4fc299 100644 --- a/spec/tests/integration/services/appraise_project_spec.rb +++ b/spec/tests/integration/services/appraise_project_spec.rb @@ -28,6 +28,8 @@ .new(GITHUB_TOKEN) .find(USERNAME, PROJECT_NAME) CodePraise::Repository::For.entity(gh_project).create(gh_project) + gitrepo = CodePraise::GitRepo.new(gh_project) + gitrepo.clone unless gitrepo.exists_locally? # WHEN: we request to appraise the project request = OpenStruct.new( diff --git a/workers/git_clone_worker.rb b/workers/git_clone_worker.rb new file mode 100644 index 0000000..59354bd --- /dev/null +++ b/workers/git_clone_worker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require_relative '../require_app' +require_app + +require 'figaro' +require 'shoryuken' + +# Shoryuken worker class to clone repos in parallel +class GitCloneWorker + # Environment variables setup + Figaro.application = Figaro::Application.new( + environment: ENV['RACK_ENV'] || 'development', + path: File.expand_path('config/secrets.yml') + ) + Figaro.load + def self.config() = Figaro.env + + Shoryuken.sqs_client = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + + include Shoryuken::Worker + shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true + + def perform(_sqs_msg, request) + project = CodePraise::Representer::Project + .new(OpenStruct.new).from_json(request) + CodePraise::GitRepo.new(project).clone + rescue CodePraise::GitRepo::Errors::CannotOverwriteLocalGitRepo + puts 'CLONE EXISTS -- ignoring request' + end +end diff --git a/workers/shoryuken.yml b/workers/shoryuken.yml new file mode 100644 index 0000000..2a460f0 --- /dev/null +++ b/workers/shoryuken.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.us-east-1.amazonaws.com/503315808870/soa-codepraise-clone-production diff --git a/workers/shoryuken_dev.yml b/workers/shoryuken_dev.yml new file mode 100644 index 0000000..35848dd --- /dev/null +++ b/workers/shoryuken_dev.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.ap-northeast-1.amazonaws.com/503315808870/soa-codepraise-clone-dev diff --git a/workers/shoryuken_test.yml b/workers/shoryuken_test.yml new file mode 100644 index 0000000..be9aa3c --- /dev/null +++ b/workers/shoryuken_test.yml @@ -0,0 +1,2 @@ +queues: + - https://sqs.ap-northeast-1.amazonaws.com/503315808870/soa-codepraise-clone-test