Skip to content
This repository has been archived by the owner on Nov 11, 2019. It is now read-only.

Commit

Permalink
shipit_code_coverage: Use args and kwargs parameters of executor.…
Browse files Browse the repository at this point in the history
…submit instead of defining intermediate functions (#884)
  • Loading branch information
gabriel-v authored and marco-c committed Feb 28, 2018
1 parent a56c2a4 commit bdec1d7
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 18 deletions.
5 changes: 1 addition & 4 deletions src/shipit_code_coverage/shipit_code_coverage/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,8 @@ def download_all(self):
if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status']['state']]:
download_tasks[(chunk_name, platform_name)] = test_task

def download_task(test_task):
return lambda: self.download(test_task)

with ThreadPoolExecutorResult() as executor:
for test_task in test_tasks:
executor.submit(download_task(test_task))
executor.submit(self.download, test_task)

logger.info('Code coverage artifacts downloaded')
24 changes: 10 additions & 14 deletions src/shipit_code_coverage/shipit_code_coverage/codecov.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,9 @@ def generate_suite_report(suite):

logger.info('Suite report generated', suite=suite)

def generate_suite_report_task(suite):
return lambda: generate_suite_report(suite)

with ThreadPoolExecutor(max_workers=2) as executor:
for suite in self.suites:
executor.submit(generate_suite_report_task(suite))
executor.submit(generate_suite_report, suite)

def generate_zero_coverage_report(self):
report = self.generate_info(self.revision, out_format='coveralls+')
Expand Down Expand Up @@ -209,22 +206,21 @@ def generate_files_list(self, covered=True, platform=None, chunk=None):
return files.splitlines()

def generate_chunk_mapping(self):
def get_files_task(platform, chunk):
return lambda: (platform, chunk, self.generate_files_list(True, platform=platform, chunk=chunk))

with ThreadPoolExecutor(max_workers=4) as executor:
futures = []
futures = {}
for platform in ['linux', 'windows']:
for chunk in self.artifactsHandler.get_chunks():
futures.append(executor.submit(get_files_task(platform, chunk)))
future = executor.submit(self.generate_files_list, True, platform=platform, chunk=chunk)
futures[future] = (platform, chunk)

with sqlite3.connect('chunk_mapping.sqlite') as conn:
c = conn.cursor()
c.execute('CREATE TABLE file_to_chunk (path text, platform text, chunk text)')
c.execute('CREATE TABLE chunk_to_test (platform text, chunk text, path text)')

for future in concurrent.futures.as_completed(futures):
(platform, chunk, files) = future.result()
(platform, chunk) = futures[future]
files = future.result()
c.executemany('INSERT INTO file_to_chunk VALUES (?,?,?)', ((f, platform, chunk) for f in files))

try:
Expand Down Expand Up @@ -259,10 +255,10 @@ def get_files_task(platform, chunk):
def go(self):
with ThreadPoolExecutorResult(max_workers=2) as executor:
# Thread 1 - Download coverage artifacts.
executor.submit(lambda: self.artifactsHandler.download_all())
executor.submit(self.artifactsHandler.download_all)

# Thread 2 - Clone mozilla-central.
executor.submit(lambda: self.clone_mozilla_central(self.revision))
executor.submit(self.clone_mozilla_central, self.revision)

if self.from_pulse:
self.githubUtils.update_geckodev_repo()
Expand All @@ -276,8 +272,8 @@ def go(self):
logger.info('Report generated successfully')

with ThreadPoolExecutorResult(max_workers=2) as executor:
executor.submit(lambda: uploader.coveralls(output))
executor.submit(lambda: uploader.codecov(output, commit_sha))
executor.submit(uploader.coveralls, output)
executor.submit(uploader.codecov, output, commit_sha)

logger.info('Waiting for build to be ingested by Codecov...')
# Wait until the build has been ingested by Codecov.
Expand Down

0 comments on commit bdec1d7

Please sign in to comment.