From 497454786885c4a7d77bc09d03dad37ca46022f2 Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 3 Jan 2018 14:58:25 -0500 Subject: [PATCH] reverting --- submit50.py | 765 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 695 insertions(+), 70 deletions(-) diff --git a/submit50.py b/submit50.py index a5cc9eb..2f64abc 100755 --- a/submit50.py +++ b/submit50.py @@ -1,83 +1,708 @@ -from glob import glob -from os import walk -from os.path import isfile, join, splitext -from setuptools import setup -from setuptools.command.develop import develop -from setuptools.command.install import install -from subprocess import call -from sys import platform, version_info - -def create_mo_files(): - """Compiles .po files in local/LANG to .mo files and returns them as array of data_files""" - - mo_files=[] - for prefix in glob("locale/*"): - for _,_,files in walk(prefix): - for file in files: - if file.endswith(".po"): - po_file = join(prefix, file) - mo_file = splitext(po_file)[0] + ".mo" - call(["msgfmt", "-o", mo_file, po_file]) - mo_files.append((join("submit50", prefix, "LC_MESSAGES"), [mo_file])) +from __future__ import print_function + +import argparse +import atexit +import datetime +import distutils +import gettext +import itertools +import json +import os +import pexpect +import pipes +import platform +import re +import readline +import requests +import select +import shlex +import shutil +import signal +import subprocess +import sys +import termcolor +import tempfile +import textwrap +import time +import traceback + +from backports.shutil_get_terminal_size import get_terminal_size +from backports.shutil_which import which +from distutils.spawn import find_executable +from distutils.version import StrictVersion +from pkg_resources import get_distribution, parse_version +from six.moves import urllib +from threading import Thread + +# Internationalization +gettext.bindtextdomain("messages", os.path.join(sys.prefix, "submit50/locale")) +gettext.textdomain("messages") +_ = gettext.gettext + +# globals +# require python 2.7+ +if sys.version_info < (2, 7): + sys.exit(_("You have an old version of python. Install version 2.7 or higher.")) +if sys.version_info < (3, 0): + input = raw_input +if not hasattr(shlex, "quote"): + shlex.quote = pipes.quote + +timestamp = None + + +class Error(Exception): + """Exception raised for errors.""" + pass - return mo_files -def install_certs(cmd): +class _Getch: """ - Decorator for classes subclassing one of setuptools commands. + Get a single character from standard input. - Installs certificates before installing the package when running - Python >= 3.6 on Mac OS. + https://stackoverflow.com/a/510364 """ - orig_run = cmd.run - def run(self): - if platform == "darwin" and version_info >= (3, 6): - INSTALL_CERTS = "/Applications/Python 3.6/Install Certificates.command" - if not isfile(INSTALL_CERTS) or call(INSTALL_CERTS) != 0: - raise RuntimeError("Error installing certificates.") - orig_run(self) + class _GetchUnix: + def __init__(self): + import tty + import sys - cmd.run = run - return cmd + def __call__(self): + import sys + import termios + import tty + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + class _GetchWindows: + def __init__(self): + import msvcrt -@install_certs -class CustomDevelop(develop): - pass + def __call__(self): + import msvcrt + return msvcrt.getch() + def __init__(self): + try: + self.impl = _Getch._GetchWindows() + except ImportError: + self.impl = _Getch._GetchUnix() -@install_certs -class CustomInstall(install): - pass + def __call__(self): + return self.impl() + + +getch = _Getch() + + +# submit50 +def main(): + + # listen for ctrl-c + signal.signal(signal.SIGINT, handler) + + # clean up on normal exit + atexit.register(teardown) + + # define command-line arguments + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--verbose", + action="store_true", + help=_("show commands being executed")) + parser.add_argument("slug", help=_("prescribed identifier of work to submit")) + args = vars(parser.parse_args()) + + # submit50 -v + # submit50 --verbose + if args["verbose"]: + run.verbose = True + + # submit50 slug + submit("submit50", args["slug"]) + + # kthxbai + sys.exit(0) + + +def authenticate(org): + """Authenticate user.""" + + # cache credentials in ~/.git-credential-cache/:org + cache = os.path.expanduser("~/.git-credential-cache") + try: + os.mkdir(cache, 0o700) + except BaseException: + pass + authenticate.SOCKET = os.path.join(cache, "submit50") + + spawn = pexpect.spawn if sys.version_info < (3, 0) else pexpect.spawnu + child = spawn( + "git -c credential.helper='cache --socket {}' credential fill".format(authenticate.SOCKET)) + child.sendline("") + if child.expect(["Username:", "Password:", pexpect.EOF]) == 2: + clear_credentials() + username, password = re.search( + "username=([^\r]+)\r\npassword=([^\r]+)", child.before, re.MULTILINE).groups() + else: + try: + username = run("git config --global credential.https://github.com/submit50.username", quiet=True) + except Error: + username = None + password = None + child.close() + + def rlinput(prompt, prefill=""): + """ + Input function that uses a prefill value and advanced line editing. + + https://stackoverflow.com/a/2533142 + """ + readline.set_startup_hook(lambda: readline.insert_text(prefill)) + try: + return input(prompt) + finally: + readline.set_startup_hook() + + # prompt for credentials + progress(False) # because not using cprint herein + if not password: + + # prompt for username, prefilling if possible + progress(False) + try: + username = rlinput(_("GitHub username: "), username).strip() + except EOFError: + print() + + # prompt for password + print(_("GitHub password: "), end="") + sys.stdout.flush() + password = str() + while True: + ch = getch() + if ch in ["\n", "\r"]: # Enter + print() + break + elif ch == "\177": # DEL + if len(password) > 0: + password = password[:-1] + print("\b \b", end="") + sys.stdout.flush() + elif ch == "\3": # ctrl-c + print("^C", end="") + os.kill(os.getpid(), signal.SIGINT) + elif ch == "\4": # ctrl-d + print() + break + else: + password += ch + print("*", end="") + sys.stdout.flush() + + # authenticate user + email = "{}@users.noreply.github.com".format(username) + res = requests.get("https://api.github.com/user", + auth=(username, password)) + + # check for 2-factor authentication + # http://github3.readthedocs.io/en/develop/examples/oauth.html?highlight=token + if "X-GitHub-OTP" in res.headers: + password = two_factor(org, username, password) + res = requests.get("https://api.github.com/user", + auth=(username, password)) + + # check if incorrect password + if res.status_code == 401: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") + raise Error(_("Invalid username and/or password.")) + + # check for other error + elif res.status_code != 200: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") + raise Error(_("Could not authenticate user.")) + + # canonicalize (capitalization of) username, + # especially if user logged in via email address + username = res.json()["login"] + + # cache credentials for 1 week + timeout = int(datetime.timedelta(weeks=1).total_seconds()) + run("git -c credential.helper='cache --socket {} --timeout {}' " + "-c credentialcache.ignoresighup=true credential approve".format(authenticate.SOCKET, timeout), + lines=["username={}".format(username), "password={}".format(password), "", ""], + quiet=True) + + # return credentials + return (username, password, email) + + +authenticate.SOCKET = None + + +def clear_credentials(): + """Clear git credential cache """ + run("git credential-cache --socket {} exit".format(authenticate.SOCKET)) + if platform.system() == "Darwin": + try: + run("git credential-osxkeychain erase", lines=["host=github.com", "protocol=https", ""]) + except Error: + pass + + +def cprint(text="", color=None, on_color=None, attrs=None, **kwargs): + """Colorizes text (and wraps to terminal's width).""" + + # update progress + progress(False) + + # assume 80 in case not running in a terminal + columns, lines = get_terminal_size() + if columns == 0: + columns = 80 # because get_terminal_size's default fallback doesn't work in pipes + + # print text + termcolor.cprint(textwrap.fill(text, columns, drop_whitespace=False), + color=color, on_color=on_color, attrs=attrs, **kwargs) + + +def excepthook(type, value, tb): + """Report an exception.""" + excepthook.ignore = False + progress(False) + teardown() + if type is Error and str(value): + cprint(str(value), "yellow") + elif type is requests.exceptions.ConnectionError: + cprint(_("Could not connect to GitHub."), "yellow") + else: + if run.verbose: + traceback.print_exception(type, value, tb) + cprint(_("Sorry, something's wrong! Let sysadmins@cs50.harvard.edu know!"), "yellow") + if authenticate.SOCKET: # not set when using SSH + try: + clear_credentials() + except Exception: + pass + cprint(_("Submission cancelled."), "red") + + +sys.excepthook = excepthook + + +def handler(number, frame): + """Handle SIGINT.""" + os.system("stty sane 2> {}".format(os.devnull)) # in case signalled from input_with_prefill + if progress.progressing: + progress(False) + else: + cprint() + try: + clear_credentials() + except Exception: + pass + teardown() + cprint(_("{} cancelled.".format(handler.type.capitalize())), "red") + os._exit(0) + +handler.type = "submission" + +def run(command, cwd=None, env=None, lines=[], password=None, quiet=False, timeout=None): + """Run a command.""" + + # echo command + if run.verbose: + cprint(command, attrs=["bold"]) + + # include GIT_DIR and GIT_WORK_TREE in env + if not env: + env = { + "GIT_DIR": run.GIT_DIR, + "GIT_WORK_TREE": run.GIT_WORK_TREE, + "HOME": os.path.expanduser("~") + } + if os.getenv("SSH_AGENT_PID"): + env["SSH_AGENT_PID"] = os.getenv("SSH_AGENT_PID") + if os.getenv("SSH_AUTH_SOCK"): + env["SSH_AUTH_SOCK"] = os.getenv("SSH_AUTH_SOCK") + + # spawn command + if sys.version_info < (3, 0): + child = pexpect.spawn(command, cwd=cwd, env=env, ignore_sighup=True, timeout=timeout) + else: + child = pexpect.spawnu( + command, + cwd=cwd, + encoding="utf-8", + env=env, + ignore_sighup=True, + timeout=timeout) + + # send output of command to stdout only if run with --verbose (and not quieted by caller) + if run.verbose and not quiet: + child.logfile_read = sys.stdout + + # wait for prompt, send password + if password: + i = child.expect(["Password for '.*': ", pexpect.EOF]) + if i == 0: + child.sendline(password) + + # send lines of input + for line in lines: + child.sendline(line) + + # read output, check status + command_output = child.read().strip() + child.close() + if child.signalstatus is None and child.exitstatus != 0: + raise Error() + return command_output + + +run.GIT_DIR = tempfile.mkdtemp() +run.GIT_WORK_TREE = os.getcwd() +run.verbose = False + + +def progress(message=""): + """Display a progress bar as dots.""" + + # don't show in verbose mode + if run.verbose: + if message: + print(message + "...") + return + + # stop progressing if already progressing + if progress.progressing: + progress.progressing = False + progress.thread.join() + sys.stdout.write("\n") + sys.stdout.flush() + + # display dots if message passed + if message: + def progress_helper(): + sys.stdout.write(message + "...") + sys.stdout.flush() + while progress.progressing: + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(0.5) + progress.progressing = True + progress.thread = Thread(target=progress_helper) + progress.thread.start() + + +progress.progressing = False + + +def submit(org, branch): + """Submit work.""" + + # check announcements + res = requests.get("https://cs50.me/status/submit50") + if res.status_code == 200 and res.text.strip(): + raise Error(res.text.strip()) + + # require git 2.7+, so that credential-cache--daemon ignores SIGHUP + # https://github.com/git/git/blob/v2.7.0/credential-cache--daemon.c + if not which("git"): + raise Error(_("You don't have git. Install git, then re-run {}!".format(org))) + version = subprocess.check_output(["git", "--version"]).decode("utf-8") + matches = re.search(r"^git version (\d+\.\d+\.\d+).*$", version) + if not matches or StrictVersion(matches.group(1)) < StrictVersion("2.7.0"): + raise Error( + _("You have an old version of git. Install version 2.7 or later, then re-run {}!".format(org))) + + # update progress + progress("Connecting") + + # compute timestamp + global timestamp + headers = requests.get("https://api.github.com/").headers + timestamp = datetime.datetime.strptime(headers["Date"], "%a, %d %b %Y %H:%M:%S %Z") + timestamp = timestamp.strftime("%Y%m%dT%H%M%SZ") + + # check version + res = requests.get("https://cs50.me/versions/submit50") + if res.status_code != 200: + raise Error(_("You have an unknown version of submit50. " + "Email sysadmins@cs50.harvard.edu!")) + version_required = res.text.strip() + if parse_version(version_required) > parse_version(get_distribution("submit50").version): + raise Error(_("You have an old version of submit50. " + "Run update50, then re-run {}!".format(org))) + + # separate branch into slug and repo + check_repo = "@cs50/checks" + branch = branch if not branch.endswith(check_repo) else branch[:-len(check_repo)] + try: + slug, src = branch.split("@") + except ValueError: + slug, src = branch, "cs50/checks" + + # ensure slug exists + file, submit.EXCLUDE = tempfile.mkstemp() + url = "https://github.com/{}/raw/master/{}/submit50/exclude".format(src, slug) + try: + urllib.request.urlretrieve(url, filename=submit.EXCLUDE) + lines = open(submit.EXCLUDE) + except Exception as e: + if run.verbose: + cprint(str(e)) + e = Error(_("Invalid slug. Did you mean to submit something else?")) + e.__cause__ = None + raise e + if slug.startswith("/") and slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(slug.strip("/")))) + elif slug.startswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading slash?".format(slug.strip("/")))) + elif slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the trailing slash?".format(slug.strip("/")))) + + # check for missing files + missing = [] + for line in lines: + matches = re.match(r"^\s*#\s*([^\s]+)\s*$", line) + if matches: + pattern = matches.group(1) + if pattern[:-1] == "/": + if not os.path.isdir(pattern): + missing.append(pattern) + elif not os.path.isfile(pattern): + missing.append(pattern) + if missing: + cprint(_("You seem to be missing these files:")) + for pattern in missing: + cprint(" {}".format(pattern)) + raise Error(_("Ensure you have the required files before submitting.")) + + # update progress + progress(_("Authenticating")) + + # authenticate user via SSH + try: + + # require ssh + assert which("ssh") + + # require GitHub username in ~/.gitconfig + username, password = run( + "git config --global credential.https://github.com/submit50.username", quiet=True), None + email = "{}@users.noreply.github.com".format(username) + repo = "git@github.com:{}/{}.git".format(org, username) + progress(False) + + # require ssh-agent + child = pexpect.spawn("ssh git@github.com") + i = child.expect(["Enter passphrase for key", "Are you sure you want to continue connecting", pexpect.EOF]) + child.close() + assert i == 2 + + # authenticate user via HTTPS + except BaseException: + username, password, email = authenticate(org) + repo = "https://{}@github.com/{}/{}".format(username, org, username) + + # update progress + progress(_("Preparing")) + + # clone repository + try: + run("git clone --bare {} {}".format(shlex.quote(repo), + shlex.quote(run.GIT_DIR)), password=password) + except BaseException: + if password: + e = Error(_("Looks like {} isn't enabled for your account yet. " + "Go to https://cs50.me/authorize and make sure you accept any pending invitations!".format(org, org))) + else: + e = Error(_("Looks like you have the wrong username in ~/.gitconfig or {} isn't yet enabled for your account. " + "Double-check ~/.gitconfig and then log into https://cs50.me/ in a browser, " + "click \"Authorize application\" if prompted, and re-run {} here.".format(org, org))) + e.__cause__ = None + raise e + + # check out .gitattributes, if any, temporarily shadowing student's, if any + if os.path.isfile(".gitattributes"): + submit.ATTRIBUTES = ".gitattributes.{}".format(round(time.time())) + os.rename(".gitattributes", submit.ATTRIBUTES) + try: + run("git checkout --force {} .gitattributes".format(branch)) + except Exception: + pass + + # set options + tag = "{}@{}".format(branch, timestamp) + run("git config user.email {}".format(shlex.quote(email))) + run("git config user.name {}".format(shlex.quote(username))) + run("git symbolic-ref HEAD refs/heads/{}".format(shlex.quote(branch))) + + # patterns of file names to exclude + run("git config core.excludesFile {}".format(shlex.quote(submit.EXCLUDE))) + + # blocklist for git-lfs + # https://github.com/git-lfs/git-lfs/blob/master/commands/command_track.go + with open("{}/info/exclude".format(run.GIT_DIR), "w") as file: + file.write(".git*\n") + file.write(".lfs*\n") + + # adds, modifies, and removes index entries to match the working tree + run("git add --all") + + # get file lists + files = run("git ls-files").splitlines() + others = run("git ls-files --exclude-from={}/info/exclude --other".format(run.GIT_DIR)).splitlines() + + # unescape any octal codes in lists + # https://stackoverflow.com/a/46650050/5156190 + def unescape(s): + if s.startswith('"') and s.endswith('"'): + return ( + s.replace('"', '') + .encode("latin1") + .decode("unicode-escape") + .encode("latin1") + .decode("utf8") + ) + return s + files = [unescape(file) for file in files] + others = [unescape(other) for other in others] + + # hide .gitattributes, if any, from output + if ".gitattributes" in files: + files.remove(".gitattributes") + + # check for large files > 100 MB (and huge files > 2 GB) + # https://help.github.com/articles/conditions-for-large-files/ + # https://help.github.com/articles/about-git-large-file-storage/ + larges, huges = [], [] + for file in files: + size = os.path.getsize(file) + if size > (100 * 1024 * 1024): + larges.append(file) + elif size > (2 * 1024 * 1024 * 1024): + huges.append(file) + if len(huges) > 0: + raise Error(_("These files are too large to be submitted:\n{}\n" + "Remove these files from your directory " + "and then re-run {}!").format("\n".join(huges), org)) + elif len(larges) > 0: + if not which("git-lfs"): + raise Error(_("These files are too large to be submitted:\n{}\n" + "Install git-lfs (or remove these files from your directory) " + "and then re-run {}!").format("\n".join(larges), org)) + run("git lfs install --local") + run("git config credential.helper cache") # for pre-push hook + for large in larges: + run("git rm --cached {}".format(large)) + run("git lfs track {}".format(large)) + run("git add {}".format(large)) + run("git add --force .gitattributes") + + # files that will be submitted + if len(files) == 0: + raise Error(_("No files in this directory are expected for submission.")) + + # prompts for submit50 + if org == "submit50": + if files: + cprint(_("Files that will be submitted:"), "green") + for file in files: + cprint("./{}".format(file), "green") + + # files that won't be submitted + if others: + cprint(_("Files that won't be submitted:"), "yellow") + for other in others: + cprint("./{}".format(other), "yellow") + + # prompt for honesty + readline.clear_history() + try: + answer = input(_("Keeping in mind the course's policy on academic honesty, " + "are you sure you want to submit these files (yes/no)? ")) + except EOFError: + answer = None + print() + if not answer or not re.match("^\s*(?:y|yes)\s*$", answer, re.I): + raise Error(_("No files were submitted.")) + + # update progress + if org == "submit50": + progress(_("Submitting")) + else: + progress(_("Uploading")) + + # push branch + run("git commit --allow-empty --message='{}'".format(timestamp)) + commit_hash = run("git rev-parse HEAD") + run("git push origin 'refs/heads/{}'".format(branch), password=password) + + # successful submission + if org == "submit50": + cprint(_("Submitted {}! See https://cs50.me/submissions.").format(branch), + "green") + progress(False) + return username, commit_hash + + +submit.ATTRIBUTES = None +submit.EXCLUDE = None + + +def teardown(): + """Delete temporary directory and temporary file, restore any attributes.""" + if os.path.isfile(".gitattributes"): + try: + os.remove(".gitattributes") + except Exception: + pass + if submit.ATTRIBUTES: + try: + os.rename(submit.ATTRIBUTES, ".gitattributes") + except Exception: + pass + shutil.rmtree(run.GIT_DIR, ignore_errors=True) + if submit.EXCLUDE: + try: + os.remove(submit.EXCLUDE) + except Exception: + pass + + +def two_factor(org, username, password): + """Get one-time authentication code.""" + # send authentication request + requests.post("https://api.github.com/authorizations", + auth=(username, password)) + while True: + cprint("Authentication code:", end=" ") + sys.stdout.flush() + code = input() + if code: + break + data = json.dumps({"scopes": ["repo", "user"], "note": "{} {}".format(org, timestamp)}) + res = requests.post("https://api.github.com/authorizations", + auth=(username, password), + data=data, + headers={"X-GitHub-OTP": str(code)}) + if res.status_code == 201 and "token" in res.json(): + return res.json()["token"] + else: + raise Error(_("Could not complete two-factor authentication.")) -setup( - author="CS50", - author_email="sysadmins@cs50.harvard.edu", - classifiers=[ - "Intended Audience :: Education", - "Programming Language :: Python :: 3", - "Topic :: Education", - "Topic :: Utilities" - ], - description="This is submit50, with which you can submit solutions to \ -problems for CS50.", - install_requires=[ - "backports.shutil_get_terminal_size", "backports.shutil_which", - "pexpect>=4.0", "requests", "six", "termcolor" - ], - keywords=["submit", "submit50"], - name="submit50", - py_modules=["submit50"], - cmdclass={ - "develop": CustomDevelop, - "install": CustomInstall - }, - entry_points={ - "console_scripts": ["submit50=submit50:main"] - }, - data_files=create_mo_files(), - url="https://github.com/cs50/submit50", - version="2.4.7" -) +if __name__ == "__main__": + main()