Skip to content

Commit

Permalink
Merge pull request #525 from slovensko-digital/GO-145/sync_from_page_…
Browse files Browse the repository at this point in the history 
…and_message_id

GO-145 Allow to sync Govbox::Folder starting with custom folder page & message ID
  • Loading branch information
@luciajanikova
luciajanikova authored Dec 9, 2024
2 parents 63f8e3f + 1b742c5 commit f8ea9d3
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 7 deletions.
7 changes: 6 additions & 1 deletion app/jobs/govbox/sync_folder_job.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ def perform(folder, upvs_client: UpvsEnvironment.upvs_client, batch_size: 1000)
new_messages_ids = []

sync_since = Date.parse(folder.box.settings['sync_since']) if folder.box.settings['sync_since'].present?
sync_from_page = folder.settings['sync_from_page'] || 0
sync_from_message_id = folder.settings['sync_from_message_id']

0.step do |k|
sync_from_page.step do |k|
response_status, raw_messages = edesk_api.fetch_messages(folder.edesk_folder_id, page: k + 1, count: batch_size)

raise "Unable to fetch folder messages" if response_status != 200
Expand All @@ -18,6 +20,9 @@ def perform(folder, upvs_client: UpvsEnvironment.upvs_client, batch_size: 1000)
next if sync_since && (Date.parse(raw_message['delivered_at']) < sync_since)

edesk_message_id = raw_message['id']

raise "MessageID out of order!" if sync_from_message_id && edesk_message_id < sync_from_message_id

old_folder_id = edesk_message_ids_to_folder_ids[edesk_message_id]

if old_folder_id.nil?
Expand Down
1 change: 1 addition & 0 deletions app/models/govbox/folder.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#
# id :bigint not null, primary key
# name :string not null
# settings :jsonb
# system :boolean not null
# created_at :datetime not null
# updated_at :datetime not null
Expand Down
5 changes: 5 additions & 0 deletions db/migrate/20241209184812_add_seetings_to_govbox_folders.rb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
class AddSeetingsToGovboxFolders < ActiveRecord::Migration[7.1]
def change
add_column :govbox_folders, :settings, :jsonb, default: {}
end
end
3 changes: 2 additions & 1 deletion db/schema.rb
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

157 changes: 152 additions & 5 deletions test/jobs/govbox/sync_folder_job_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,97 @@

class Govbox::SyncFolderJobTest < ActiveJob::TestCase
test "downloads all messages unless box sync_since value in box settings set" do
folder = govbox_folders(:ssd_one)

edesk_api_mock = Minitest::Mock.new
edesk_api_mock.expect :fetch_messages, [200, [
{
"id"=>4905707493,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e",
"correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2023-07-10T09:51:36.533Z"
},
{
"id"=>4906445420,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a",
"correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia",
"delivered_at"=>"2023-07-14T12:51:50.337Z"
},
{
"id"=>4906376126,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd",
"correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk",
"delivered_at"=>"2023-07-15T14:16:32.600Z"
},
{
"id"=>4905707496,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53",
"correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2023-08-10T09:51:36.560Z"
},
{
"id"=>4905707497,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53",
"correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2023-08-11T09:51:36.560Z"
},
]], [folder.edesk_folder_id], **{ page: 1, count: 5 }
edesk_api_mock.expect :fetch_messages, [200, [
{
"id"=>4905707493,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e",
"correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2023-08-12T09:51:36.533Z"
},
{
"id"=>4906445420,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a",
"correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia",
"delivered_at"=>"2023-08-14T12:51:50.337Z"
},
{
"id"=>4906376126,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd",
"correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk",
"delivered_at"=>"2023-11-15T14:16:32.600Z"
},
{
"id"=>4905707496,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53",
"correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2024-07-10T09:51:36.560Z"
}
]], [folder.edesk_folder_id], **{ page: 2, count: 5 }

::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do
Govbox::SyncFolderJob.new.perform(folder, batch_size: 5)
end

assert_enqueued_jobs 9
end

test "does not download older messages than box sync_since value in box settings" do
box = boxes(:ssd_main)
box.settings['sync_since'] = '2023-11-15'
box.save

folder = govbox_folders(:ssd_one)

Expand Down Expand Up @@ -46,15 +136,18 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase
Govbox::SyncFolderJob.new.perform(folder)
end

assert_enqueued_jobs 4
assert_enqueued_jobs 2
end

test "does not download older messages than box sync_since value in box settings" do
test "starts fetching messages from selected folder page and downloads only messages from selected ID (folder settings)" do
box = boxes(:ssd_main)
box.settings['sync_since'] = '2023-11-15'
box.save

folder = govbox_folders(:ssd_one)
folder.settings['sync_from_page'] = 9
folder.settings['sync_from_message_id'] = 4905707495
folder.save

edesk_api_mock = Minitest::Mock.new
edesk_api_mock.expect :fetch_messages, [200, [
Expand All @@ -67,15 +160,15 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase
"delivered_at"=>"2023-07-10T09:51:36.533Z"
},
{
"id"=>4906445420,
"id"=>4905707494,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a",
"correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia",
"delivered_at"=>"2023-11-14T12:51:50.337Z"
},
{
"id"=>4906376126,
"id"=>4905707495,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd",
"correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b",
Expand All @@ -90,12 +183,66 @@ class Govbox::SyncFolderJobTest < ActiveJob::TestCase
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2024-07-10T09:51:36.560Z"
},
]], [folder.edesk_folder_id], **{ page: 1, count: 1000 }
]], [folder.edesk_folder_id], **{ page: 10, count: 1000 }


::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do
Govbox::SyncFolderJob.new.perform(folder)
end

assert_enqueued_jobs 2
end

test "raises if downloading messages from selected ID and smaller ID is found (box settings)" do
box = boxes(:ssd_main)
box.settings['sync_since'] = '2023-11-15'
box.save

folder = govbox_folders(:ssd_one)
folder.settings['sync_from_page'] = 9
folder.settings['sync_from_message_id'] = 4906376126
folder.save

edesk_api_mock = Minitest::Mock.new
edesk_api_mock.expect :fetch_messages, [200, [
{
"id"=>4905707493,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"7cb378ef-9c76-493b-b41d-51f1e40dd68e",
"correlation_id"=>"6baa28c3-96e5-493b-a76c-9837a6d637b3",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2023-07-10T09:51:36.533Z"
},
{
"id"=>4906445420,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1fe26465-ff59-4fed-b948-630cc994507a",
"correlation_id"=>"5aeddd85-952e-4534-a9fd-9c73774049f3",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia",
"delivered_at"=>"2023-11-14T12:51:50.337Z"
},
{
"id"=>4906376126,
"class"=>"EGOV_DOCUMENT",
"message_id"=>"1d5f449c-7b8e-40d6-9662-64ff0af527cd",
"correlation_id"=>"7f2956fd-d8d8-4062-9d34-f9b0418c9f0b",
"subject"=>"Všeobecná agenda - rozhodnutie do vlastných rúk",
"delivered_at"=>"2023-11-15T14:16:32.600Z"
},
{
"id"=>4905707496,
"class"=>"ED_DELIVERY_NOTIFICATION",
"message_id"=>"14fe42a3-c3b7-419b-bb76-485f49543e53",
"correlation_id"=>"8c359d95-3d72-4e76-b0f2-ddeba11c3b5b",
"subject"=>"Notifikácia o doručení k \"Všeobecná agenda - rozhodnutie do vlastných rúk s fikciou doručenia\"",
"delivered_at"=>"2024-07-10T09:51:36.560Z"
},
]], [folder.edesk_folder_id], **{ page: 10, count: 1000 }

assert_raise('MessageID out of order!') do
::Upvs::GovboxApi::Edesk.stub :new, edesk_api_mock do
Govbox::SyncFolderJob.new.perform(folder)
end
end
end
end

0 comments on commit f8ea9d3

Please sign in to comment.