Skip to content

Commit

Permalink
Merge branch 'ko3n1g/ci/retry-download' into 'main'
Browse files Browse the repository at this point in the history
ci: Retry download assets

See merge request ADLR/megatron-lm!2357
  • Loading branch information
ko3n1g committed Nov 18, 2024
2 parents 6c88bfc + 06c67b4 commit 5438d15
Showing 1 changed file with 2 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -222,19 +222,17 @@ def main(
while n_download_attempt < 3:
try:
jet_log = main_job.get_logs()
logs = extract_logs_to_string(logs=jet_log)
download_job_assets(logs=jet_log, iteration=n_iteration)
break
except requests.exceptions.ConnectionError as e:
print(e)
time.sleep((3**n_download_attempt) * 60)
n_download_attempt += 1

logs = extract_logs_to_string(logs=jet_log)

concat_logs = "\n".join(logs)
print(f"Logs:\n{concat_logs}")

download_job_assets(logs=jet_log, iteration=n_iteration)

if test_type != "release":
success = pipeline.get_status() == PipelineStatus.SUCCESS

Expand Down

0 comments on commit 5438d15

Please sign in to comment.