Skip to content

Commit

Permalink
scraper_event_check test
Browse files Browse the repository at this point in the history
  • Loading branch information
mikesndrs committed Oct 22, 2024
1 parent 41c9ade commit 875dd2b
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 8 deletions.
10 changes: 6 additions & 4 deletions lib/scraper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,17 @@ def scrape(source, user = get_user, index: 0)
end
end

def scraper_event_check(_data_sources)
return unless TeSS::Config&.scraper_event_check&.[]('enabled')
def scraper_event_check(data_sources)
result = false
return result unless TeSS::Config&.scraper_event_check&.[]('enabled')

data_sources.each do |_key, sources|
sources.each do |source|
event_check_rejected(source)
event_check_stale(source)
result ||= event_check_rejected(source)
result ||= event_check_stale(source)
end
end
result
end

def event_check_stale(source)
Expand Down
8 changes: 4 additions & 4 deletions test/unit/ingestors/scraper_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -310,27 +310,27 @@ def set_up_event_check
freeze_time(2019) do
set_up_event_check
with_settings({ scraper_event_check: { enabled: true, stale_threshold: 0.3 } }) do
assert_not @scraper.event_check_stale(@source)
assert_not @scraper.scraper_event_check({config: [@source]})
@source.content_provider.events.each do |event|
event.last_scraped = 10.days.freeze.ago
event.timezone = 'Amsterdam'
event.save!
event.reload
end
assert @scraper.event_check_stale(@source)
assert @scraper.scraper_event_check({config: [@source]})
end
end
end

test 'event_check_rejected' do
set_up_event_check
with_settings({ scraper_event_check: { enabled: true, rejected_threshold: 0.3 } }) do
assert_not @scraper.event_check_rejected(@source)
assert_not @scraper.scraper_event_check({config: [@source]})
@source.records_written = 10
@source.resources_rejected = 90
@source.save!
@source.reload
assert @scraper.event_check_rejected(@source)
assert @scraper.scraper_event_check({config: [@source]})
end
end

Expand Down

0 comments on commit 875dd2b

Please sign in to comment.