Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance Country Leaderboard Availability During Refresh #1099

Merged
merged 1 commit into from
Jan 16, 2024
+111 −63
Merged

Enhance Country Leaderboard Availability During Refresh #1099

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 111 additions & 63 deletions app/org/maproulette/jobs/SchedulerActor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,76 +486,116 @@ class SchedulerActor @Inject() (
val start = System.currentTimeMillis
logger.info(s"Scheduled Task '$action': Starting run")

db.withConnection { implicit c =>
// Clear TABLEs
SQL("DELETE FROM user_leaderboard WHERE country_code IS NOT NULL").executeUpdate()
SQL("DELETE FROM user_top_challenges WHERE country_code IS NOT NULL").executeUpdate()

val countryCodeMap = boundingBoxFinder.boundingBoxforAll()
for ((countryCode, boundingBox) <- countryCodeMap) {
SQL(
LeaderboardHelper.rebuildChallengesLeaderboardSQLCountry(
SchedulerActor.ONE_MONTH,
countryCode,
boundingBox,
config
)
).executeUpdate()
SQL(
LeaderboardHelper.rebuildChallengesLeaderboardSQLCountry(
SchedulerActor.THREE_MONTHS,
countryCode,
boundingBox,
config
)
).executeUpdate()
SQL(
LeaderboardHelper.rebuildChallengesLeaderboardSQLCountry(
SchedulerActor.SIX_MONTHS,
countryCode,
boundingBox,
config
)
).executeUpdate()
SQL(
LeaderboardHelper.rebuildChallengesLeaderboardSQLCountry(
SchedulerActor.TWELVE_MONTHS,
countryCode,
boundingBox,
config
)
).executeUpdate()
def deleteAndUpdateCountryLeaderboardForTimePeriod(
monthDuration: Int,
countryCode: String,
boundingBox: String
): Unit = {
logger.info(
s"Scheduled Task '$action': updating user_leaderboard monthDuration=$monthDuration countryCode=$countryCode"
)
db.withConnection { implicit c =>
// Delete the existing entries for the country and time period
SQL(
LeaderboardHelper.rebuildChallengesLeaderboardSQLCountry(
SchedulerActor.ALL_TIME,
countryCode,
boundingBox,
config
)
).executeUpdate()
s"DELETE FROM user_leaderboard WHERE country_code = {countryCode} AND month_duration = {monthDuration}"
).on(Symbol("countryCode") -> countryCode, Symbol("monthDuration") -> monthDuration)
.executeUpdate()
// Insert the new entries for the country and time period
SQL(
LeaderboardHelper.rebuildTopChallengesSQLCountry(
SchedulerActor.TWELVE_MONTHS,
countryCode,
boundingBox,
config
)
).executeUpdate()
SQL(
LeaderboardHelper.rebuildTopChallengesSQLCountry(
SchedulerActor.ALL_TIME,
countryCode,
boundingBox,
config
)
LeaderboardHelper
.rebuildChallengesLeaderboardSQLCountry(monthDuration, countryCode, boundingBox, config)
).executeUpdate()
}

val totalTime = System.currentTimeMillis - start
logger.info(
s"Scheduled Task '$action': Finished run. Time spent: ${String.format("%1d", totalTime)}ms"
s"Scheduled Task '$action': finished updating user_leaderboard monthDuration=$monthDuration countryCode=$countryCode"
)
}

// TODO(ljdelight): If the loop order is inverted where each country loops over the monthDuration, will this be faster?
// The database may be able to more effectively cache the per-country results vs the time-based outer loop.
SchedulerActor.MONTH_DURATIONS.foreach(monthDuration => {
val countryCodeMap = boundingBoxFinder.boundingBoxforAll()
for ((countryCode, boundingBox) <- countryCodeMap) {
try {
deleteAndUpdateCountryLeaderboardForTimePeriod(monthDuration, countryCode, boundingBox)
} catch {
// If an exception occurs, log it and continue to the next country
case e: Exception =>
logger.error(
s"Scheduled Task '$action': Failed to update user_leaderboard monthDuration=$monthDuration countryCode=$countryCode",
e
)
}
}
})

db.withConnection { implicit c =>
val countryCodeMap = boundingBoxFinder.boundingBoxforAll()
for ((countryCode, boundingBox) <- countryCodeMap) {
try {
logger.info(
s"Scheduled Task '$action': updating user_top_challenges monthDuration=12 countryCode=$countryCode"
)
// Delete the existing entries for the country and time period
SQL(
"DELETE FROM user_top_challenges WHERE country_code = {countryCode} AND month_duration = {monthDuration}"
).on(
Symbol("countryCode") -> countryCode,
Symbol("monthDuration") -> 12
)
.executeUpdate()

SQL(
LeaderboardHelper.rebuildTopChallengesSQLCountry(
SchedulerActor.TWELVE_MONTHS,
countryCode,
boundingBox,
config
)
).executeUpdate()
} catch {
case e: Exception =>
logger.error(
s"Scheduled Task '$action': Error updating user_top_challenges for monthDuration=12 countryCode=$countryCode",
e
)
}

try {
logger.info(
s"Scheduled Task '$action': updating user_top_challenges monthDuration=-1 countryCode=$countryCode"
)
// Delete the existing entries for the country and time period
SQL(
"DELETE FROM user_top_challenges WHERE country_code = {countryCode} AND month_duration = {monthDuration}"
).on(
Symbol("countryCode") -> countryCode,
Symbol("monthDuration") -> -1
)
.executeUpdate()

SQL(
LeaderboardHelper.rebuildTopChallengesSQLCountry(
SchedulerActor.ALL_TIME,
countryCode,
boundingBox,
config
)
).executeUpdate()
} catch {
case e: Exception =>
logger.error(
s"Scheduled Task '$action': Error updating user_top_challenges for monthDuration=12 countryCode=$countryCode",
e
)
}
}
}

val totalTime = System.currentTimeMillis - start
logger.info(
s"Scheduled Task '$action': Finished run. Time spent: ${String.format("%1d", totalTime)}ms"
)
}

def sendImmediateNotificationEmails(action: String) = {
Expand Down Expand Up @@ -846,6 +886,14 @@ object SchedulerActor {
private val TWELVE_MONTHS = 12
private val ALL_TIME = -1

private val MONTH_DURATIONS = List(
SchedulerActor.ONE_MONTH,
SchedulerActor.THREE_MONTHS,
SchedulerActor.SIX_MONTHS,
SchedulerActor.TWELVE_MONTHS,
SchedulerActor.ALL_TIME
).sorted

def props = Props[SchedulerActor]()

case class RunJob(name: String, action: String = "")
Expand Down