diff --git a/.travis_scripts/create_feedstocks.py b/.travis_scripts/create_feedstocks.py index bf5f788803a42..aec486bb27bfc 100755 --- a/.travis_scripts/create_feedstocks.py +++ b/.travis_scripts/create_feedstocks.py @@ -2,7 +2,9 @@ """ Convert all recipes into feedstocks. -This script is to be run in a TravisCI context, with all secret environment variables defined (BINSTAR_TOKEN, GH_TOKEN) +This script is to be run in a TravisCI context, with all secret environment +variables defined (BINSTAR_TOKEN, GH_TOKEN) + Such as: export GH_TOKEN=$(cat ~/.conda-smithy/github.token) @@ -28,6 +30,8 @@ recipe_directory_name = 'recipes' + + def list_recipes(): if os.path.isdir(recipe_directory_name): recipes = os.listdir(recipe_directory_name) @@ -84,9 +88,11 @@ def print_rate_limiting_info(gh, user): print("GitHub API Rate Limit Info:") print("---------------------------") print("token: ", user) - print("Currently remaining {remaining} out of {total}.".format(remaining=gh_api_remaining, total=gh_api_total)) + print("Currently remaining {remaining} out of {total}.".format( + remaining=gh_api_remaining, total=gh_api_total)) print("Will reset in {time}.".format(time=gh_api_reset_time)) print("") + return gh_api_remaining def sleep_until_reset(gh): @@ -116,7 +122,10 @@ def sleep_until_reset(gh): if __name__ == '__main__': exit_code = 0 - is_merged_pr = (os.environ.get('TRAVIS_BRANCH') == 'master' and os.environ.get('TRAVIS_PULL_REQUEST') == 'false') + is_merged_pr = ( + os.environ.get('TRAVIS_BRANCH') == 'master' and + os.environ.get('TRAVIS_PULL_REQUEST') == 'false' + ) smithy_conf = os.path.expanduser('~/.conda-smithy') if not os.path.exists(smithy_conf): @@ -134,20 +143,25 @@ def write_token(name, token): if 'DRONE_TOKEN' in os.environ: write_token('drone', os.environ['DRONE_TOKEN']) + gh_drone = Github(os.environ['GH_DRONE_TOKEN']) + gh_drone_remaining = print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') + gh = None if 'GH_TOKEN' in os.environ: write_token('github', os.environ['GH_TOKEN']) gh = Github(os.environ['GH_TOKEN']) # Get our initial rate limit info. - print_rate_limiting_info(gh, 'GH_TOKEN') + gh_remaining = print_rate_limiting_info(gh, 'GH_TOKEN') # if we are out, exit early # if sleep_until_reset(gh): # sys.exit(1) - gh_drone = Github(os.environ['GH_DRONE_TOKEN']) - print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') + # try the other token maybe? + # if gh_remaining < gh_drone_remaining and gh_remaining < 100: + # write_token('github', os.environ['GH_DRONE_TOKEN']) + # gh = Github(os.environ['GH_DRONE_TOKEN']) owner_info = ['--organization', 'conda-forge'] @@ -158,34 +172,53 @@ def write_token(name, token): feedstock_dir = os.path.join(feedstocks_dir, name + '-feedstock') print('Making feedstock for {}'.format(name)) try: - subprocess.check_call(['conda', 'smithy', 'init', recipe_dir, - '--feedstock-directory', feedstock_dir]) + subprocess.check_call( + ['conda', 'smithy', 'init', recipe_dir, + '--feedstock-directory', feedstock_dir]) except subprocess.CalledProcessError: traceback.print_exception(*sys.exc_info()) continue if not is_merged_pr: - # We just want to check that conda-smithy is doing its thing without having any metadata issues. + # We just want to check that conda-smithy is doing its + # thing without having any metadata issues. continue feedstock_dirs.append([feedstock_dir, name, recipe_dir]) - subprocess.check_call(['git', 'remote', 'add', 'upstream_with_token', - 'https://conda-forge-manager:{}@github.com/conda-forge/{}-feedstock'.format(os.environ['GH_TOKEN'], - name)], - cwd=feedstock_dir) + subprocess.check_call([ + 'git', 'remote', 'add', 'upstream_with_token', + 'https://conda-forge-manager:{}@github.com/' + 'conda-forge/{}-feedstock'.format( + os.environ['GH_TOKEN'], + name + ) + ], + cwd=feedstock_dir + ) print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') - # Sometimes we already have the feedstock created. We need to deal with that case. + # Sometimes we already have the feedstock created. We need to + # deal with that case. if repo_exists(gh, 'conda-forge', name + '-feedstock'): - subprocess.check_call(['git', 'fetch', 'upstream_with_token'], cwd=feedstock_dir) - subprocess.check_call(['git', 'branch', '-m', 'master', 'old'], cwd=feedstock_dir) + subprocess.check_call( + ['git', 'fetch', 'upstream_with_token'], cwd=feedstock_dir) + subprocess.check_call( + ['git', 'branch', '-m', 'master', 'old'], cwd=feedstock_dir) try: - subprocess.check_call(['git', 'checkout', '-b', 'master', 'upstream_with_token/master'], cwd=feedstock_dir) + subprocess.check_call( + [ + 'git', 'checkout', '-b', 'master', + 'upstream_with_token/master' + ], + cwd=feedstock_dir) except subprocess.CalledProcessError: - # Sometimes, we have a repo, but there are no commits on it! Just catch that case. - subprocess.check_call(['git', 'checkout', '-b' 'master'], cwd=feedstock_dir) + # Sometimes, we have a repo, but there are no commits on + # it! Just catch that case. + subprocess.check_call( + ['git', 'checkout', '-b' 'master'], cwd=feedstock_dir) print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') - subprocess.check_call(['conda', 'smithy', 'register-github', feedstock_dir] + owner_info) + subprocess.check_call( + ['conda', 'smithy', 'register-github', feedstock_dir] + owner_info) print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') from conda_smithy.ci_register import drone_sync @@ -195,7 +228,8 @@ def write_token(name, token): time.sleep(100) # actually wait print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN') - # Break the previous loop to allow the TravisCI registering to take place only once per function call. + # Break the previous loop to allow the TravisCI registering + # to take place only once per function call. # Without this, intermittent failures to synch the TravisCI repos ensue. # Hang on to any CI registration errors that occur and raise them at the end. for num, (feedstock_dir, name, recipe_dir) in enumerate(feedstock_dirs): @@ -209,9 +243,12 @@ def write_token(name, token): # we fail the build so that people are aware that things did not clear. try: # Stop registering on appveyor temporarily - subprocess.check_call(['conda', 'smithy', 'register-ci', '--without-appveyor', - '--without-webservice', '--feedstock_directory', feedstock_dir] + owner_info) - subprocess.check_call(['conda', 'smithy', 'rerender'], cwd=feedstock_dir) + subprocess.check_call( + ['conda', 'smithy', 'register-ci', '--without-appveyor', + '--without-webservice', '--feedstock_directory', + feedstock_dir] + owner_info) + subprocess.check_call( + ['conda', 'smithy', 'rerender'], cwd=feedstock_dir) except subprocess.CalledProcessError: exit_code = 1 traceback.print_exception(*sys.exc_info()) @@ -221,26 +258,37 @@ def write_token(name, token): for i in range(1, 13): time.sleep(10) print("Waiting for registration: {i} s".format(i=i*10)) - subprocess.check_call(['git', 'commit', '--allow-empty', '-am', "Re-render the feedstock after CI registration."], cwd=feedstock_dir) + subprocess.check_call( + ['git', 'commit', '--allow-empty', '-am', + "Re-render the feedstock after CI registration."], cwd=feedstock_dir) for i in range(5): try: # Capture the output, as it may contain the GH_TOKEN. - out = subprocess.check_output(['git', 'push', 'upstream_with_token', 'HEAD:master'], cwd=feedstock_dir, - stderr=subprocess.STDOUT) + out = subprocess.check_output( + ['git', 'push', 'upstream_with_token', 'HEAD:master'], + cwd=feedstock_dir, + stderr=subprocess.STDOUT) break except subprocess.CalledProcessError: pass # Likely another job has already pushed to this repo. # Place our changes on top of theirs and try again. - out = subprocess.check_output(['git', 'fetch', 'upstream_with_token', 'master'], cwd=feedstock_dir, - stderr=subprocess.STDOUT) + out = subprocess.check_output( + ['git', 'fetch', 'upstream_with_token', 'master'], + cwd=feedstock_dir, + stderr=subprocess.STDOUT) try: - subprocess.check_call(['git', 'rebase', 'upstream_with_token/master', 'master'], cwd=feedstock_dir) + subprocess.check_call( + ['git', 'rebase', 'upstream_with_token/master', 'master'], + cwd=feedstock_dir) except subprocess.CalledProcessError: # Handle rebase failure by choosing the changes in `master`. - subprocess.check_call(['git', 'checkout', 'master', '--', '.'], cwd=feedstock_dir) - subprocess.check_call(['git', 'rebase', '--continue'], cwd=feedstock_dir) + subprocess.check_call( + ['git', 'checkout', 'master', '--', '.'], + cwd=feedstock_dir) + subprocess.check_call( + ['git', 'rebase', '--continue'], cwd=feedstock_dir) # Remove this recipe from the repo. if is_merged_pr: @@ -258,27 +306,33 @@ def write_token(name, token): # Finish quietly. pass - # Parse `git status --porcelain` to handle some merge conflicts and generate the removed recipe list. - changed_files = subprocess.check_output(['git', 'status', '--porcelain', recipe_directory_name], - universal_newlines=True) + # Parse `git status --porcelain` to handle some merge conflicts and + # generate the removed recipe list. + changed_files = subprocess.check_output( + ['git', 'status', '--porcelain', recipe_directory_name], + universal_newlines=True) changed_files = changed_files.splitlines() - # Add all files from AU conflicts. They are new files that we weren't tracking previously. + # Add all files from AU conflicts. They are new files that we + # weren't tracking previously. # Adding them resolves the conflict and doesn't actually add anything to the index. new_file_conflicts = filter(lambda _: _.startswith("AU "), changed_files) - new_file_conflicts = map(lambda _ : _.replace("AU", "", 1).lstrip(), new_file_conflicts) + new_file_conflicts = map( + lambda _: _.replace("AU", "", 1).lstrip(), new_file_conflicts) for each_new_file in new_file_conflicts: subprocess.check_call(['git', 'add', each_new_file]) # Generate a fresh listing of recipes removed. # # * Each line we get back is a change to a file in the recipe directory. - # * We narrow the list down to recipes that are staged for deletion (ignores examples). + # * We narrow the list down to recipes that are staged for deletion + # (ignores examples). # * Then we clean up the list so that it only has the recipe names. removed_recipes = filter(lambda _: _.startswith("D "), changed_files) - removed_recipes = map(lambda _ : _.replace("D", "", 1).lstrip(), removed_recipes) - removed_recipes = map(lambda _ : os.path.relpath(_, recipe_directory_name), removed_recipes) - removed_recipes = map(lambda _ : _.split(os.path.sep)[0], removed_recipes) + removed_recipes = map(lambda _: _.replace("D", "", 1).lstrip(), removed_recipes) + removed_recipes = map( + lambda _: os.path.relpath(_, recipe_directory_name), removed_recipes) + removed_recipes = map(lambda _: _.split(os.path.sep)[0], removed_recipes) removed_recipes = sorted(set(removed_recipes)) # Commit any removed packages. @@ -290,14 +344,17 @@ def write_token(name, token): msg += ' [ci skip]' if is_merged_pr: # Capture the output, as it may contain the GH_TOKEN. - out = subprocess.check_output(['git', 'remote', 'add', 'upstream_with_token', - 'https://conda-forge-manager:{}@github.com/conda-forge/staged-recipes'.format(os.environ['GH_TOKEN'])], - stderr=subprocess.STDOUT) + out = subprocess.check_output( + ['git', 'remote', 'add', 'upstream_with_token', + 'https://conda-forge-manager:{}@github.com/' + 'conda-forge/staged-recipes'.format(os.environ['GH_TOKEN'])], + stderr=subprocess.STDOUT) subprocess.check_call(['git', 'commit', '-m', msg]) # Capture the output, as it may contain the GH_TOKEN. branch = os.environ.get('TRAVIS_BRANCH') - out = subprocess.check_output(['git', 'push', 'upstream_with_token', 'HEAD:%s' % branch], - stderr=subprocess.STDOUT) + out = subprocess.check_output( + ['git', 'push', 'upstream_with_token', 'HEAD:%s' % branch], + stderr=subprocess.STDOUT) else: print('Would git commit, with the following message: \n {}'.format(msg)) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e14b7619292fb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length=88