diff --git a/.gitignore b/.gitignore index 4847247c..6ecb0de7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ pip-selfcheck.json # IDEs .idea +.vscode + +# Backup files +*.bak + diff --git a/README.md b/README.md index 297d675b..0824d7ac 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,10 @@ Please feel free to use Tango at your school/organization. If you run into any p 3. [Read the documentation for the VMMS API](https://github.com/autolab/Tango/wiki/Tango-VMMS-API). 4. [Test whether Tango is set up properly and can process jobs](https://github.com/autolab/Tango/wiki/Testing-Tango). -## Python 3 Upgrade -We are in the process of porting Tango from Python 2 to Python 3. The current working branch for the update is `python3-upgrade`. +## Python 2 Support +Tango now runs on Python 3. However, there is a legacy branch [master-python2](https://github.com/autolab/Tango/tree/master-python2) which is a snapshot of the last Python 2 Tango commit for legacy reasons. You are strongly encouraged to upgrade to the current Python 3 version of Tango if you are still on the Python 2 version, as future enhancements and bug fixes will be focused on the current master. + +We will not be backporting new features from `master` to `master-python2`. ## Contributing to Tango diff --git a/clients/tango-cli.py b/clients/tango-cli.py index ab345164..2708e0c2 100755 --- a/clients/tango-cli.py +++ b/clients/tango-cli.py @@ -4,6 +4,12 @@ # tango-cli.py - Command line client for the RESTful Tango. # +from __future__ import print_function +from builtins import range +from future import standard_library +standard_library.install_aliases() +from builtins import map +from builtins import str import os import sys @@ -12,7 +18,7 @@ import argparse import requests import json -import urllib +import urllib.request, urllib.parse, urllib.error # # @@ -95,35 +101,35 @@ def checkKey(): if (args.key is None): - print "Key must be specified with -k" + print("Key must be specified with -k") return -1 return 0 def checkCourselab(): if (args.courselab is None): - print "Courselab must be specified with -l" + print("Courselab must be specified with -l") return -1 return 0 def checkFilename(): if (args.filename is None): - print "Filename must be specified with --filename" + print("Filename must be specified with --filename") return -1 return 0 def checkInfiles(): if (args.infiles is None): - print "Input files must be specified with --infiles" + print("Input files must be specified with --infiles") return -1 return 0 def checkDeadjobs(): if (args.deadJobs is None): - print "Deadjobs must be specified with --deadJobs" + print("Deadjobs must be specified with --deadJobs") return -1 return 0 @@ -139,11 +145,11 @@ def tango_open(): response = requests.get( 'http://%s:%d/open/%s/%s/' % (args.server, args.port, args.key, args.courselab)) - print "Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) - print response.content + print("Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab) + print("Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)) print (str(err)) sys.exit(0) @@ -170,11 +176,11 @@ def tango_upload(): data=f.read(), headers=header) f.close() - print "Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) - print response.content + print("Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename) + print("Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)) print (str(err)) sys.exit(0) @@ -208,11 +214,11 @@ def tango_addJob(): args.key, args.courselab), data=json.dumps(requestObj)) - print "Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) - print response.content + print("Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)) + print("Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))) print (str(err)) sys.exit(0) @@ -231,13 +237,13 @@ def tango_poll(): args.port, args.key, args.courselab, - urllib.quote( + urllib.parse.quote( args.outputFile))) - print "Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) - print response.content + print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile)) + print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile))) print (str(err)) sys.exit(0) @@ -252,11 +258,11 @@ def tango_info(): response = requests.get( 'http://%s:%d/info/%s/' % (args.server, args.port, args.key)) - print "Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key) - print response.content + print("Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key)) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key) + print("Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key)) print (str(err)) sys.exit(0) @@ -272,11 +278,11 @@ def tango_jobs(): response = requests.get( 'http://%s:%d/jobs/%s/%d/' % (args.server, args.port, args.key, args.deadJobs)) - print "Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) - print response.content + print("Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs) + print("Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)) print (str(err)) sys.exit(0) @@ -291,11 +297,11 @@ def tango_pool(): response = requests.get('http://%s:%d/pool/%s/%s/' % (args.server, args.port, args.key, args.image)) - print "Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) - print response.content + print("Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image) + print("Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)) print (str(err)) sys.exit(0) @@ -321,11 +327,11 @@ def tango_prealloc(): args.image, args.num), data=json.dumps(vmObj)) - print "Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) - print response.content + print("Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) + print(response.text) except Exception as err: - print "Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)) + print("Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))) print (str(err)) sys.exit(0) @@ -343,31 +349,31 @@ def file_to_dict(file): def tango_runJob(): if args.runJob is None: - print "Invalid usage: [runJob]" + print("Invalid usage: [runJob]") sys.exit(0) dir = args.runJob infiles = [file for file in os.listdir( dir) if os.path.isfile(os.path.join(dir, file))] files = [os.path.join(dir, file) for file in infiles] - args.infiles = map(file_to_dict, infiles) + args.infiles = list(map(file_to_dict, infiles)) args.jobname += "-0" args.outputFile += "-0" - for i in xrange(1, args.numJobs + 1): - print "----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------" - print "----------- OPEN" + for i in range(1, args.numJobs + 1): + print("----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------") + print("----------- OPEN") tango_open() - print "----------- UPLOAD" + print("----------- UPLOAD") for file in files: args.filename = file tango_upload() - print "----------- ADDJOB" + print("----------- ADDJOB") length = len(str(i - 1)) args.jobname = args.jobname[:-length] + str(i) args.outputFile = args.outputFile[:-length] + str(i) tango_addJob() - print "--------------------------------------------------------------------------------------------------\n" + print("--------------------------------------------------------------------------------------------------\n") def router(): @@ -403,7 +409,7 @@ def router(): try: response = requests.get('http://%s:%d/' % (args.server, args.port)) except: - print 'Tango not reachable on %s:%d!\n' % (args.server, args.port) + print('Tango not reachable on %s:%d!\n' % (args.server, args.port)) sys.exit(0) router() diff --git a/config.template.py b/config.template.py index 5e62e592..bb7d3f22 100644 --- a/config.template.py +++ b/config.template.py @@ -2,12 +2,13 @@ # config.py - Global configuration constants and runtime info # +from builtins import object import logging, time # Config - defines -class Config: +class Config(object): ##### # Part 1: Tango constants for developers # diff --git a/jobManager.py b/jobManager.py index 7ec31aee..5f24bc33 100644 --- a/jobManager.py +++ b/jobManager.py @@ -1,3 +1,4 @@ +from __future__ import print_function # # JobManager - Thread that assigns jobs to worker threads # @@ -9,6 +10,10 @@ # is launched that will handle things from here on. If anything goes # wrong, the job is made dead with the error. # +from builtins import object +from future import standard_library +standard_library.install_aliases() +from builtins import str import threading, logging, time, copy from datetime import datetime @@ -20,7 +25,7 @@ from tangoObjects import TangoQueue from config import Config -class JobManager: +class JobManager(object): def __init__(self, queue): self.daemon = True @@ -62,9 +67,16 @@ def __manage(self): if id: job = self.jobQueue.get(id) + + # job could no longer exist if it was completed by someone else + if job == None: + continue + if not job.accessKey and Config.REUSE_VMS: id, vm = self.jobQueue.getNextPendingJobReuse(id) job = self.jobQueue.get(id) + if job == None: + continue try: # Mark the job assigned diff --git a/jobQueue.py b/jobQueue.py index ad43e3b9..c64484a6 100644 --- a/jobQueue.py +++ b/jobQueue.py @@ -7,6 +7,9 @@ # JobManager: Class that creates a thread object that looks for new # work on the job queue and assigns it to workers. # +from builtins import range +from builtins import object +from builtins import str import threading, logging, time from datetime import datetime @@ -28,7 +31,7 @@ # -class JobQueue: +class JobQueue(object): def __init__(self, preallocator): self.liveJobs = TangoDictionary("liveJobs") @@ -53,7 +56,7 @@ def _getNextID(self): keys = self.liveJobs.keys() if (str(id) in keys): id = -1 - for i in xrange(1, Config.MAX_JOBID + 1): + for i in range(1, Config.MAX_JOBID + 1): if (str(i) not in keys): id = i break @@ -191,7 +194,7 @@ def getNextPendingJob(self): Called by JobManager when Config.REUSE_VMS==False """ self.queueLock.acquire() - for id, job in self.liveJobs.iteritems(): + for id, job in self.liveJobs.items(): if job.isNotAssigned(): self.queueLock.release() return id @@ -203,7 +206,7 @@ def getNextPendingJobReuse(self, target_id=None): Called by JobManager when Config.REUSE_VMS==True """ self.queueLock.acquire() - for id, job in self.liveJobs.iteritems(): + for id, job in self.liveJobs.items(): # if target_id is set, only interested in this id if target_id and target_id != id: continue diff --git a/preallocator.py b/preallocator.py index 026c09f5..8eb3f190 100644 --- a/preallocator.py +++ b/preallocator.py @@ -1,6 +1,8 @@ # # preallocator.py - maintains a pool of active virtual machines # +from builtins import object +from builtins import range import threading, logging, time, copy from tangoObjects import TangoDictionary, TangoQueue, TangoIntValue @@ -17,7 +19,7 @@ # -class Preallocator: +class Preallocator(object): def __init__(self, vmms): self.machines = TangoDictionary("machines") diff --git a/requirements.txt b/requirements.txt index 618987ec..57f86313 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,9 @@ -backports.ssl-match-hostname==3.4.0.2 -boto==2.27.0 -futures==2.2.0 -plumbum==1.4.2 -pyflakes==0.8.1 -redis==2.10.3 -requests==2.2.1 -rpyc==3.3.0 -wsgiref==0.1.2 -tornado==4.1 +backports.ssl-match-hostname==3.7.0.1 +boto==2.49.0 # used only by ec2SSH.py +plumbum==1.6.9 +pyflakes==2.1.1 +redis==3.4.1 +requests==2.23.0 +rpyc==4.1.4 +tornado==4.5.3 +future==0.18.2 diff --git a/restful-tango/server.py b/restful-tango/server.py index 4f67e8ef..afd4e273 100755 --- a/restful-tango/server.py +++ b/restful-tango/server.py @@ -1,5 +1,7 @@ +from future import standard_library +standard_library.install_aliases() import tornado.web -import urllib +import urllib.request, urllib.parse, urllib.error import sys import os from tempfile import NamedTemporaryFile @@ -67,7 +69,7 @@ def prepare(self): """ set up the temporary file""" tempdir="%s/tmp" % (Config.COURSELABS,) if not os.path.exists(tempdir): - os.mkdir(tempdir, 0700) + os.mkdir(tempdir, 0o700) if os.path.exists(tempdir) and not os.path.isdir(tempdir): tangoREST.log("Cannot process uploads, %s is not a directory" % (tempdir,)) return self.send_error() @@ -104,7 +106,7 @@ class PollHandler(tornado.web.RequestHandler): def get(self, key, courselab, outputFile): """ get - Handles the get request to poll.""" self.set_header('Content-Type', 'application/octet-stream') - return tangoREST.poll(key, courselab, urllib.unquote(outputFile)) + return tangoREST.poll(key, courselab, urllib.parse.unquote(outputFile)) class InfoHandler(tornado.web.RequestHandler): diff --git a/restful-tango/tangoREST.py b/restful-tango/tangoREST.py index 59d02365..2e3d581f 100644 --- a/restful-tango/tangoREST.py +++ b/restful-tango/tangoREST.py @@ -1,9 +1,12 @@ +from __future__ import print_function # tangoREST.py # # Implements open, upload, addJob, and poll to be used for the RESTful # interface of Tango. # +from builtins import object +from builtins import str import sys import os import inspect @@ -22,7 +25,7 @@ from config import Config -class Status: +class Status(object): def __init__(self): self.found_dir = self.create(0, "Found directory") @@ -53,7 +56,7 @@ def create(self, id, msg): return result -class TangoREST: +class TangoREST(object): COURSELABS = Config.COURSELABS OUTPUT_FOLDER = Config.OUTPUT_FOLDER @@ -109,7 +112,7 @@ def checkFileExists(self, directory, filename, fileMD5): for elem in os.listdir(directory): if elem == filename: try: - body = open("%s/%s" % (directory, elem)).read() + body = open("%s/%s" % (directory, elem)).read().encode('utf-8') md5hash = hashlib.md5(body).hexdigest() return md5hash == fileMD5 except IOError: @@ -288,6 +291,9 @@ def upload(self, key, courselab, file, tempfile, fileMD5): os.unlink(tempfile) return self.status.wrong_courselab except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + print(exc_type, fname, exc_tb.tb_lineno) self.log.error("upload request failed: %s" % str(e)) os.unlink(tempfile) return self.status.create(-1, str(e)) diff --git a/tango.py b/tango.py index 058c9930..1d2d7b56 100755 --- a/tango.py +++ b/tango.py @@ -34,6 +34,8 @@ # the pool, the preallocator creates another instance and adds it # to the pool. (preallocator.py) +from builtins import object +from builtins import str import threading, logging, time, stat, re, os from datetime import datetime @@ -45,7 +47,7 @@ from config import Config -class TangoServer: +class TangoServer(object): """ TangoServer - Implements the API functions that the server accepts """ @@ -233,7 +235,7 @@ def resetTango(self, vmms): self.log.warning("Killed these %s VMs on restart: %s" % (vmms_name, namelist)) - for _, job in self.jobQueue.liveJobs.iteritems(): + for _, job in self.jobQueue.liveJobs.items(): if not job.isNotAssigned(): job.makeUnassigned() self.log.debug("job: %s, assigned: %s" % diff --git a/tangoObjects.py b/tangoObjects.py index 17e4130f..ec389a3e 100644 --- a/tangoObjects.py +++ b/tangoObjects.py @@ -2,9 +2,14 @@ # # Implements objects used to pass state within Tango. # +from builtins import range +from builtins import object +from future import standard_library +standard_library.install_aliases() +from builtins import str import redis import pickle -import Queue +import queue from config import Config redisConnection = None @@ -19,7 +24,7 @@ def getRedisConnection(): return redisConnection -class InputFile(): +class InputFile(object): """ InputFile - Stores pointer to the path on the local machine and the @@ -35,7 +40,7 @@ def __repr__(self): self.destFile) -class TangoMachine(): +class TangoMachine(object): """ TangoMachine - A description of the Autograding Virtual Machine @@ -62,7 +67,7 @@ def __repr__(self): return "TangoMachine(image: %s, vmms: %s)" % (self.image, self.vmms) -class TangoJob(): +class TangoJob(object): """ TangoJob - A job that is to be run on a TangoMachine @@ -156,7 +161,7 @@ def TangoIntValue(object_name, obj): return TangoNativeIntValue(object_name, obj) -class TangoRemoteIntValue(): +class TangoRemoteIntValue(object): def __init__(self, name, value, namespace="intvalue"): """The default connection parameters are: host='localhost', port=6379, db=0""" @@ -176,7 +181,7 @@ def set(self, val): return self.__db.set(self.key, val) -class TangoNativeIntValue(): +class TangoNativeIntValue(object): def __init__(self, name, value, namespace="intvalue"): self.key = '%s:%s' % (namespace, name) @@ -198,10 +203,10 @@ def TangoQueue(object_name): if Config.USE_REDIS: return TangoRemoteQueue(object_name) else: - return Queue.Queue() + return queue.Queue() -class TangoRemoteQueue(): +class TangoRemoteQueue(object): """Simple Queue with Redis Backend""" @@ -263,7 +268,7 @@ def TangoDictionary(object_name): return TangoNativeDictionary() -class TangoRemoteDictionary(): +class TangoRemoteDictionary(object): def __init__(self, object_name): self.r = getRedisConnection() @@ -279,7 +284,7 @@ def set(self, id, obj): return str(id) def get(self, id): - if str(id) in self.r.hkeys(self.hash_name): + if self.r.hexists(self.hash_name, str(id)): unpickled_obj = self.r.hget(self.hash_name, str(id)) obj = pickle.loads(unpickled_obj) return obj @@ -287,7 +292,8 @@ def get(self, id): return None def keys(self): - return self.r.hkeys(self.hash_name) + keys = map(lambda key : key.decode(), self.r.hkeys(self.hash_name)) + return list(keys) def values(self): vals = self.r.hvals(self.hash_name) @@ -304,11 +310,11 @@ def _clean(self): # only for testing self.r.delete(self.hash_name) - def iteritems(self): - return iter([(i, self.get(i)) for i in xrange(1,Config.MAX_JOBID+1) + def items(self): + return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) -class TangoNativeDictionary(): +class TangoNativeDictionary(object): def __init__(self): self.dict = {} @@ -317,23 +323,23 @@ def set(self, id, obj): self.dict[str(id)] = obj def get(self, id): - if str(id) in self.dict.keys(): + if str(id) in self.dict: return self.dict[str(id)] else: return None def keys(self): - return self.dict.keys() + return list(self.dict.keys()) def values(self): - return self.dict.values() + return list(self.dict.values()) def delete(self, id): - if str(id) in self.dict.keys(): + if str(id) in list(self.dict.keys()): del self.dict[str(id)] - def iteritems(self): - return iter([(i, self.get(i)) for i in xrange(1,Config.MAX_JOBID+1) + def items(self): + return iter([(i, self.get(i)) for i in range(1,Config.MAX_JOBID+1) if self.get(i) != None]) def _clean(self): diff --git a/tests/testJobQueue.py b/tests/testJobQueue.py index 0b6a970c..6bd65266 100644 --- a/tests/testJobQueue.py +++ b/tests/testJobQueue.py @@ -1,3 +1,6 @@ +from __future__ import print_function +from builtins import range +from builtins import str import unittest import redis @@ -55,7 +58,7 @@ def test_job(self): self.assertTrue(job.isNotAssigned()) self.job1.makeAssigned() - print "Checkout:" + print("Checkout:") self.assertFalse(self.job1.isNotAssigned()) self.assertFalse(job.isNotAssigned()) @@ -127,7 +130,7 @@ def test_makeDead(self): def test__getNextID(self): init_id = self.jobQueue.nextID - for i in xrange(1, Config.MAX_JOBID + 100): + for i in range(1, Config.MAX_JOBID + 100): id = self.jobQueue._getNextID() self.assertNotEqual(str(id), self.jobId1) diff --git a/tests/validate.py b/tests/validate.py index 28b08db5..6f047acb 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1,3 +1,4 @@ +from __future__ import print_function # # Recursively validates all python files with pyflakes that were modified # since the last validation, and provides basic stats. Ignores hidden @@ -8,8 +9,8 @@ import pyflakes pyflakes # avoid unused warning when validating self! except ImportError: - print 'Validate requires pyflakes. Please install '\ - 'with: pip install pyflakes' + print('Validate requires pyflakes. Please install '\ + 'with: pip install pyflakes') exit() import argparse @@ -46,7 +47,7 @@ line_count = 0 -print '\n---- Validating all files ----' +print('\n---- Validating all files ----') for dirname, dirnames, filenames in os.walk('.'): for filename in filenames: @@ -77,16 +78,16 @@ if args.stats: line_count += sum(1 for line in open(path)) if validated_issue_count == 0: - print 'ALL OKAY' -print '\n---- Validation summary ----' -print 'Files with validation issues: %i' % validated_issue_count -print 'Validated files: %i' % validated_count -print 'Total python files: %i' % file_count + print('ALL OKAY') +print('\n---- Validation summary ----') +print('Files with validation issues: %i' % validated_issue_count) +print('Validated files: %i' % validated_count) +print('Total python files: %i' % file_count) # Print stats if args.stats: - print '\n---- Stats ----' - print 'Total python line count: %i' % line_count + print('\n---- Stats ----') + print('Total python line count: %i' % line_count) # Finish -print '' +print('') diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 8ea8def2..c773409f 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -8,6 +8,8 @@ # `domain_name` attribtue of TangoMachine. # +from builtins import object +from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import tempfile import socket @@ -64,7 +66,7 @@ def timeoutWithReturnStatus(command, time_out, returnValue = 0): stderr=subprocess.STDOUT) return ret -class DistDocker: +class DistDocker(object): _SSH_FLAGS = ["-q", "-o", "BatchMode=yes" ] _SSH_AUTH_FLAGS = [ "-i", os.path.join(os.path.dirname(__file__), "id_rsa"), @@ -223,7 +225,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("Lost persistent SSH connection") return ret - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab &> output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) @@ -335,7 +337,7 @@ def getVMs(self): volumes = subprocess.check_output(["ssh"] + DistDocker._SSH_FLAGS + DistDocker._SSH_AUTH_FLAGS + ["%s@%s" % (self.hostUser, host), - "(ls %s)" % volumePath]).split('\n') + "(ls %s)" % volumePath]).decode('utf-8').split('\n') for volume in volumes: if re.match("%s-" % config.Config.PREFIX, volume): machine = TangoMachine() @@ -373,7 +375,7 @@ def getImages(self): o = subprocess.check_output(["ssh"] + DistDocker._SSH_FLAGS + DistDocker._SSH_AUTH_FLAGS + ["%s@%s" % (self.hostUser, host), - "(docker images)"]) + "(docker images)"]).decode('utf-8') o_l = o.split('\n') o_l.pop() o_l.reverse() diff --git a/vmms/ec2SSH.py b/vmms/ec2SSH.py index 303bf381..4f9eeb0f 100644 --- a/vmms/ec2SSH.py +++ b/vmms/ec2SSH.py @@ -8,6 +8,10 @@ # Ec2Exception - EC2 raises this if it encounters any problem # ec2CallError - raised by ec2Call() function # +# TODO: this currently probably does not work on Python 3 yet + +from builtins import object +from builtins import str import subprocess import os import re @@ -81,7 +85,7 @@ class ec2CallError(Exception): pass -class Ec2SSH: +class Ec2SSH(object): _SSH_FLAGS = ["-i", config.Config.SECURITY_KEY_PATH, "-o", "StrictHostKeyChecking no", "-o", "GSSAPIAuthentication no"] @@ -341,7 +345,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.instanceName(vm.id, vm.name)) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab &> output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab > output 2>&1 " % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, maxOutputFileSize) @@ -368,7 +372,7 @@ def copyOut(self, vm, destFile): self.ssh_flags + [ "%s@%s" % (config.Config.EC2_USER_NAME, domain_name), - 'cat time.out']).rstrip('\n') + 'cat time.out']).decode('utf-8').rstrip('\n') # If the output is empty, then ignore it (timing info wasn't # collected), otherwise let's log it! diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 52c030b7..1c89f5c9 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -2,6 +2,8 @@ # localDocker.py - Implements the Tango VMMS interface to run Tango jobs in # docker containers. In this context, VMs are docker containers. # +from builtins import object +from builtins import str import random, subprocess, re, time, logging, threading, os, sys, shutil import config from tangoObjects import TangoMachine @@ -60,7 +62,7 @@ def timeoutWithReturnStatus(command, time_out, returnValue = 0): # User defined exceptions # -class LocalDocker: +class LocalDocker(object): def __init__(self): """ Checks if the machine is ready to run docker containers. @@ -137,7 +139,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): args = args + [vm.image] args = args + ['sh', '-c'] - autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab &> output/feedback' % \ + autodriverCmd = 'autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1' % \ (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) @@ -227,7 +229,7 @@ def getImages(self): """ result = set() cmd = "docker images" - o = subprocess.check_output("docker images", shell=True) + o = subprocess.check_output("docker images", shell=True).decode('utf-8') o_l = o.split('\n') o_l.pop() o_l.reverse() diff --git a/vmms/tashiSSH.py b/vmms/tashiSSH.py index ea05e114..57db80e7 100644 --- a/vmms/tashiSSH.py +++ b/vmms/tashiSSH.py @@ -8,6 +8,9 @@ # TashiException - Tashi raises this if it encounters any problem # tashiCallError - raised by tashiCall() function # +# TODO: this currently probably does not work on Python 3 yet +from builtins import object +from builtins import str import random import subprocess import os @@ -93,7 +96,7 @@ class tashiCallError(Exception): pass -class TashiSSH: +class TashiSSH(object): _SSH_FLAGS = ["-q", "-i", os.path.dirname(__file__) + "/id_rsa", "-o", "StrictHostKeyChecking=no", "-o", "GSSAPIAuthentication=no"] @@ -278,7 +281,7 @@ def runJob(self, vm, runTimeout, maxOutputFileSize): self.log.debug("runJob: Running job on VM %s" % domain_name) # Setting ulimits for VM and running job runcmd = "/usr/bin/time --output=time.out autodriver -u %d -f %d -t \ - %d -o %d autolab &> output" % (config.Config.VM_ULIMIT_USER_PROC, + %d -o %d autolab > output 2>&1 " % (config.Config.VM_ULIMIT_USER_PROC, config.Config.VM_ULIMIT_FILE_SIZE, runTimeout, config.Config.MAX_OUTPUT_FILE_SIZE) @@ -307,7 +310,7 @@ def copyOut(self, vm, destFile): [ 'autolab@%s' % (domain_name), - 'cat time.out']).rstrip('\n') + 'cat time.out']).decode('utf-8').rstrip('\n') # If the output is empty, then ignore it (timing info wasn't # collected), otherwise let's log it! diff --git a/worker.py b/worker.py index e7ffec25..dc9ca674 100644 --- a/worker.py +++ b/worker.py @@ -1,6 +1,7 @@ # # worker.py - Thread that shepherds a job through it execution sequence # +from builtins import str import threading import time import logging @@ -119,7 +120,7 @@ def catFiles(self, f1, f2): """ self.appendMsg(f1, "Here is the output from the autograder:\n---") (wfd, tmpname)=tempfile.mkstemp(dir=os.path.dirname(f2)) - wf=os.fdopen(wfd, "a") + wf=os.fdopen(wfd, "ab") with open(f1, "rb") as f1fd: shutil.copyfileobj(f1fd, wf) # f2 may not exist if autograder failed @@ -138,7 +139,7 @@ def notifyServer(self, job): outputFileName = job.outputFile.split( "/")[-1] # get filename from path fh = open(job.outputFile, 'rb') - files = {'file': unicode(fh.read(), errors='ignore')} + files = {'file': str(fh.read(), errors='ignore')} hdrs = {'Filename': outputFileName} self.log.debug("Sending request to %s" % job.notifyURL) response = requests.post(