Merge pull request #90 from ska-sa/py3ize

Some py3 changes

.gitignore

@@ -5,4 +5,4 @@ MANIFEST
 dist/
 .venv*/
 pyxis.log
-
+.vscode

.travis.yml

@@ -6,6 +6,7 @@ matrix:
   - env: TARGET=pep8
   - env: TARGET=mypy
   allow_failures:
+  - env: TARGET=py3
   - env: TARGET=pep8
   - env: TARGET=mypy
 sudo: required
@@ -14,7 +15,7 @@ services:
 install:
  - true
 script:
-- docker build . -f .travis/${TARGET}.docker
+- travis_wait 30 docker build . -f .travis/${TARGET}.docker
 deploy:
   provider: pypi
   user: gijzelaerr

.travis/py2.docker

@@ -1,13 +1,39 @@
-FROM kernsuite/base:4
+FROM kernsuite/base:5
 RUN docker-apt-install \
+    casacore-dev \
+    casacore-tools \
+    casarest \
     meqtrees-timba \
+    libqdbm-dev \
+    owlcat \
+    python-astro-tigger \
+    python-gdbm \
+    python-pip \
     python-astropy \
-    python-matplotlib \
-    python-casacore \
     python-numpy \
-    python-pip
+    python-scipy \
+    python-astlib \
+    python-casacore \
+    python-casacore \
+    python-qt4 \
+    python-qwt \
+    build-essential \
+    libboost-all-dev \
+    makems
 
-ADD . /code
-WORKDIR /code
+# Test specific external software
+RUN docker-apt-install \
+    casalite \
+    wsclean
+
+# check basic pyxis installation
+ADD . /code/pyxis
+WORKDIR /code/pyxis
 RUN pip install .
-RUN pyxis
+WORKDIR /
+RUN pyxis
+
+# run test when built
+RUN pip install nose
+WORKDIR /usr/local/lib/python2.7/dist-packages/Pyxis/recipies
+RUN python2.7 -m "nose"

.travis/py3.docker

@@ -1,13 +1,35 @@
-FROM kernsuite/base:4
+FROM kernsuite/base:5
 RUN docker-apt-install \
+    casacore-dev \
+    casacore-tools \
+    casarest \
     meqtrees-timba \
+    libqdbm-dev \
+    owlcat \
+    python-astro-tigger \
+    python3-gdbm \
+    python3-pip \
     python3-astropy \
-    python3-matplotlib \
-    python3-casacore \
     python3-numpy \
-    python3-pip
+    python3-scipy \
+    python3-astlib \
+    python3-casacore \
+    python3-casacore \
+    python3-pyqt4 \
+    python3-qwt \
+    build-essential \
+    libboost-all-dev \
+    makems \
+    wget
 
-ADD . /code
-WORKDIR /code
+# check basic pyxis installation
+ADD . /code/pyxis
+WORKDIR /code/pyxis
 RUN pip3 install .
+WORKDIR /
 RUN pyxis
+
+# run test when built
+RUN pip3 install nose
+WORKDIR /usr/local/lib/python3.6/dist-packages/Pyxis/recipies
+RUN python3.6 -m "nose"

MANIFEST.in

@@ -0,0 +1,14 @@
+include pyxis *.py
+include Pyxides *.py
+include Pyxides/im *.py
+include Pyxides/utils *.py
+
+# any datafiles here
+recursive-include Pyxis/recipies/meqtrees-batch-test/WSRT_ANTENNA *
+recursive-include Pyxis/recipies/imagers-test/WSRT_ANTENNA *
+include Pyxis/recipies/meqtrees-batch-test/test-lsm1.txt.reference
+recursive-include Pyxis/recipies/ *.py 
+recursive-include Pyxis/recipies/ *.txt 
+recursive-include Pyxis/recipies/ *.fits 
+recursive-include Pyxis/recipies/ *.conf
+recursive-include Pyxis/recipies/ *.cfg

Pyxides/im/__init__.py

@@ -61,7 +61,7 @@
 # import moresane
 
 def make_image (*args,**kw):
-  imager = kw.get('imager',IMAGER).lower();
+  imager = kw.get('imager', IMAGER).lower();
   imgmod = "im."+imager;
   try:
     mod = getattr(__import__(imgmod),imager);

Pyxides/im/casa.py

@@ -4,7 +4,7 @@
 import im
 import subprocess
 import tempfile
-import im.argo
+import  argo
 import pyrap.images as Images
 
 # register ourselves with Pyxis and define the superglobals
@@ -102,7 +102,7 @@ def _run(path='${im.CASA_PATH}',clean=False,makepsf=False,**kw):
 
     path = interpolate_locals('path')
     # map some options to casapy equivalents
-    global cell,imsize,weighting
+    global cell,imsize,weighting,_casa_args
     cell,imsize,weighting = cellsize,npix,weight
 
     # make dict of imager arguments that have been specified globally or locally

Pyxides/im/lwimager.py

@@ -5,7 +5,7 @@
 import os
 import subprocess
 from astropy.io import fits as pyfits
-import argo
+from im import argo
 
 import ms
 import std
@@ -90,6 +90,8 @@ def _run (convert_output_to_fits=True,lwimager_path="$LWIMAGER_PATH",**kw):
   # look up lwimager
   lwimager_path = interpolate_locals("lwimager_path");
   lwimager_path = argo.findImager(lwimager_path)
+  if not lwimager_path:
+    raise RuntimeError("Failed to find lwimager")
   # make dict of imager arguments that have been specified globally or locally
   args = dict([ (arg,globals()[arg]) for arg in _lwimager_args if arg in globals() and globals()[arg] is not None ]);
   args.update([ (arg,kw[arg]) for arg in _lwimager_args if arg in kw ]);
@@ -120,6 +122,7 @@ def _run (convert_output_to_fits=True,lwimager_path="$LWIMAGER_PATH",**kw):
     velo = kw.get('velocity') or velocity;
     for arg in _fileargs:
       if arg in fitsfiles:
+        error("FITS::" + args[arg])
         _im = pyrap.images.image(args[arg]);
         if fs and fs != 1:
           _im.putdata(fs*_im.getdata());
@@ -141,9 +144,9 @@ def lwimager_version (path="$LWIMAGER_PATH"):
     vstr = subprocess.Popen([path,"--version"],stderr=subprocess.PIPE).stderr.read().strip().split()[-1];
   except:
     return 0,"";
-  if '.' in vstr:
-    major,minor,patch = vstr.split('.')
-    patch,tail = patch.split("-",1) if "-" in patch else (patch,"");
+  if b'.' in vstr:
+    major,minor,patch = vstr.split(b'.')
+    patch,tail = patch.split(b"-",1) if b"-" in patch else (patch,"");
   else:
     major,minor,patch,tail = 1,3,0,vstr;
   try:

Pyxides/im/moresane.py

@@ -7,7 +7,6 @@
 import math
 import argo
 import im
-
 # Standard options
 gain = 0.1
 sigmalevel = 3.0

Pyxides/imager.py

@@ -1,4 +1,4 @@
-import im.argo
+from Pyxides.im import argo
 
 from Pyxis.ModSupport import *
 

Pyxides/lsm.py

@@ -4,7 +4,7 @@
 
 from astropy.io import fits as pyfits
 import Tigger
-import im.argo
+from im import argo
 
 register_pyxis_module(superglobals="OUTFILE");
 

Pyxides/ms.py

@@ -8,8 +8,8 @@
 from Pyxis.ModSupport import *
 
 import std
-import _utils.casa_scripts
-import im.argo
+from utils import casa_scripts
+
 
 # register ourselves with Pyxis, and define the superglobals
 register_pyxis_module();
@@ -61,7 +61,7 @@ def ms (msname="$MS",subtable=None,write=False):
   """Opens the MS or a subtable (read-only by default), returns table object."""
   msname = interpolate_locals("msname");
   if not msname:
-    raise ValueError("'msname' or global MS variable must be set");
+    raise ValueError("'msname' or global MS variable must be set and valid");
   if subtable:
     msname = table(msname,ack=False).getkeyword(subtable);
   tab = table(msname,readonly=not write,ack=False);

Pyxis/Commands.py

@@ -115,6 +115,10 @@ def _verbose (level,*msg,**kw):
     verb = int(Pyxis.Context['VERBOSE']);
   except:
     verb = 1;
+  try:
+    level = int(level)
+  except:
+    level = 1;
   if level <= verb:
     _message(_timestamp(),"PYXIS:",*msg,**kw);
 
@@ -453,7 +457,7 @@ def add (self,obj):
     """Adds an object to the safelist, in an MP-safe manner""";
     if isinstance(obj,str):
       obj = _I(obj,2);
-    ff = file(self.filename,"ab");
+    ff = open(self.filename,"a");
     fcntl.flock(ff,fcntl.LOCK_EX);
     try:
       pickle.dump(obj,ff);
@@ -464,7 +468,7 @@ def read (self):
     """Reads all objects accumulated in the safelist, in an MP-safe manner""";
     ret = [];
     if os.path.exists(self.filename):
-      ff = file(self.filename);
+      ff = open(self.filename);
       fcntl.flock(ff,fcntl.LOCK_EX);
       try:
         while True:

Pyxis/Internals.py

@@ -63,6 +63,12 @@ def init (context):
   global _abort
   global _verbose
   global _warn
+
+  import six
+  global _ispy3
+  global _ispy2
+  _ispy3 = six.PY3
+  _ispy2 = six.PY2
   from Pyxis.Commands import _debug,_info,_abort,_verbose,_warn
   # set default output dir
   context.setdefault("OUTDIR",".");
@@ -232,8 +238,13 @@ def sh (self,*args,**kws):
     # if stdout/stderr is not a file (as is the case under ipython notebook, then
     # subprocess.Popen() fails. Therefore, in these cases, or if get_output is true, we
     # pipe the output into here via communicate()
-    stdout = subprocess.PIPE if self.get_output or type(sys.stdout) is not file else sys.stdout;
-    stderr = subprocess.PIPE if type(sys.stderr) is not file else sys.stderr;
+    if _ispy2:
+      stdout = subprocess.PIPE if self.get_output or type(sys.stdout) is not file else sys.stdout;
+      stderr = subprocess.PIPE if type(sys.stderr) is not file else sys.stderr;
+    else:
+      from _io import TextIOWrapper
+      stdout = subprocess.PIPE if self.get_output or type(sys.stdout) is not TextIOWrapper else sys.stdout;
+      stderr = subprocess.PIPE if type(sys.stderr) is not TextIOWrapper else sys.stderr;
     po = subprocess.Popen(["/bin/bash","-c"]+list(commands), preexec_fn=_on_parent_exit('SIGTERM'),
         shell=False,stdout=stdout,stderr=stderr);
     # if piping either output stream, capture it here
@@ -714,15 +725,15 @@ def set_logfile (filename,quiet=False):
       sys.stdout,sys.stderr = sys.__stdout__,sys.__stderr__;
       _current_logobj = None;
     else:
-      mode = "a";
+      mode = "wa";
       # append to file if name starts with +, or if file has already been used as a log this session, or if flush is off
       if Pyxis.Context.get("LOG_FLUSH"):
         mode = "w";
         if filename[0] == '+':
           filename = filename[1:];
-          mode = "a";
+          mode = "wa";
         if filename in _visited_logfiles:
-          mode = "a";
+          mode = "wa";
       Pyxis.ModSupport.makedir(os.path.dirname(filename),no_interpolate=True);
       _current_logobj = sys.stdout = sys.stderr = open(filename,mode);
       hdr = Pyxis.Context.get("LOG_HEADER");
@@ -811,7 +822,7 @@ def load_package (pkgname,filename,chdir=True,report=True):
     if dirname not in oldpath:
       sys.path.append(dirname);
     try:
-      exec(file(filename),Pyxis.Context);
+      exec(open(filename),Pyxis.Context);
     finally:
       sys.path = oldpath;
   except SystemExit:
@@ -908,22 +919,32 @@ def _call_exec (path,args,kws1={},**kws):
     _verbose(verbose,"executing '%s':"%(" ".join(args)));
     stdout,stderr = sys.stdout,sys.stderr;
     if quiet:
-      stdout = stderr = file('/dev/null','w')
+      stdout = stderr = open('/dev/null','wb')
     # if stdout/stderr is not a file (as is the case under ipython notebook, then
     # subprocess.Popen() fails. Therefore, in these cases, or if get_output is true, we
     # pipe the output into here via communicate()
-    if get_output or type(stdout) is not file or type(stderr) is not file:
+    if _ispy2:
+      is_not_std_file = type(stdout) is not file or type(stderr) is not file
+    else:
+      from _io import TextIOWrapper
+      is_not_std_file = type(stdout) is not TextIOWrapper or type(stderr) is not TextIOWrapper
+    if get_output or is_not_std_file:
       stdout = stderr = subprocess.PIPE;
-    type(stdout) is file and stdout.flush();
-    type(stderr) is file and stderr.flush();
+    if _ispy2:
+      type(stdout) is file and stdout.flush();
+      type(stderr) is file and stderr.flush();
+    else:
+      from _io import TextIOWrapper
+      type(stdout) is TextIOWrapper and stdout.flush();
+      type(stderr) is TextIOWrapper and stderr.flush();
     po = subprocess.Popen(args,preexec_fn=_on_parent_exit('SIGTERM'),
       stdout=stdout,stderr=stderr);
     if stdout is subprocess.PIPE:
       output,err_output = po.communicate();
       if not get_output:
-        sys.stdout.write(output);
+        sys.stdout.write(output.decode('ascii'));
         output = None;
-      sys.stderr.write(err_output);
+      sys.stderr.write(err_output.decode('ascii'));
     else:
       po.wait();
       output = po.returncode;

Pyxis/__init__.py

@@ -18,9 +18,21 @@
 
 if not _initialized:
   """Initializes Pyxis with a global context dict.""";
-  # initialize Pyxis using the globals of whoever imported the module  
-  _context = inspect.currentframe().f_back.f_globals;
-
+  # initialize Pyxis using the globals of whoever imported the module
+  ## OMS: this was enough in Python2. Not in Py3
+  #_context = inspect.currentframe().f_back.f_globals;
+  ## in Py3:
+  frame = inspect.currentframe().f_back
+  while frame:
+    _context = frame.f_globals
+    if "PYXIS_ROOT_NAMESPACE" in _context:
+      print("root namespace is",_context.get("__name__"),_context.get("__path__"))
+      break
+    frame = frame.f_back
+
+  if not frame:
+    raise RuntimeError("Unable to find root namespace. Set PYXIS_ROOT_NAMESPACE=True before importing pyxis.")
+
   Pyxis.Internals.init(_context);  
   from Pyxis.Commands import *