From e090ba909aeb28ce2f0504a77e5a800070907873 Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Thu, 11 Oct 2018 13:59:57 -0400 Subject: [PATCH 1/6] remove errors and warnings from py37 tests of sascalc --- src/sas/sascalc/calculator/instrument.py | 6 ++++- .../calculator/resolution_calculator.py | 8 +++--- src/sas/sascalc/corfunc/corfunc_calculator.py | 15 +++++------ src/sas/sascalc/data_util/calcthread.py | 27 ++++++++----------- src/sas/sascalc/data_util/nxsunit.py | 7 +++-- src/sas/sascalc/data_util/ordereddicttest.py | 8 +++--- src/sas/sascalc/data_util/registry.py | 2 +- src/sas/sascalc/data_util/uncertainty.py | 2 +- src/sas/sascalc/dataloader/data_info.py | 13 +++++---- src/sas/sascalc/dataloader/loader.py | 24 ++++++++--------- src/sas/sascalc/dataloader/manipulations.py | 15 +++++------ .../dataloader/readers/associations.py | 4 +-- .../dataloader/readers/cansas_reader.py | 5 ++-- .../dataloader/readers/cansas_reader_HDF5.py | 4 +-- src/sas/sascalc/fit/expression.py | 2 +- src/sas/sascalc/fit/models.py | 21 ++++++++------- src/sas/sascalc/fit/pagestate.py | 16 +++++------ src/sas/sascalc/invariant/invariant.py | 3 ++- src/sas/sascalc/pr/distance_explorer.py | 4 +-- src/sas/sascalc/pr/fit/expression.py | 2 +- src/sas/sascalc/pr/num_term.py | 4 +-- src/sas/sascalc/realspace/VolumeCanvas.py | 2 +- 22 files changed, 97 insertions(+), 97 deletions(-) diff --git a/src/sas/sascalc/calculator/instrument.py b/src/sas/sascalc/calculator/instrument.py index d8b1d57f4e..4455d6f766 100644 --- a/src/sas/sascalc/calculator/instrument.py +++ b/src/sas/sascalc/calculator/instrument.py @@ -127,7 +127,11 @@ def set_size(self, size=[]): if len(size) == 0: self.size = 0 else: - self.size = size + # TODO: Make sure detector size is number of pixels + # Could be detector dimensions in e.g., mm, but + # the resolution calculator assumes it is pixels. + # Being pixels, it has to be integers rather than float + self.size = [int(s) for s in size] validate(size[0]) def set_pix_size(self, size=[]): diff --git a/src/sas/sascalc/calculator/resolution_calculator.py b/src/sas/sascalc/calculator/resolution_calculator.py index 5acf6cd2cf..6b4619fc8a 100644 --- a/src/sas/sascalc/calculator/resolution_calculator.py +++ b/src/sas/sascalc/calculator/resolution_calculator.py @@ -1006,8 +1006,8 @@ def _get_detector_qxqy_pixels(self): detector_offset = 0 try: detector_offset = self.sample2detector_distance[1] - except: - logger.error(sys.exc_value) + except Exception as exc: + logger.error(exc) # detector size in [no of pix_x,no of pix_y] detector_pix_nums_x = self.detector_size[0] @@ -1093,8 +1093,8 @@ def _get_detector_qxqy_pixels(self): output.data = inten output.qx_data = qx_value output.qy_data = qy_value - except: - logger.error(sys.exc_value) + except Exception as exc: + logger.error(exc) return output diff --git a/src/sas/sascalc/corfunc/corfunc_calculator.py b/src/sas/sascalc/corfunc/corfunc_calculator.py index 6ceca0b99e..b481cf43f8 100644 --- a/src/sas/sascalc/corfunc/corfunc_calculator.py +++ b/src/sas/sascalc/corfunc/corfunc_calculator.py @@ -29,21 +29,20 @@ def __init__(self, f, g, start, stop): self.g = g self.start = start self.stop = stop - self._lastx = [] - self._lasty = [] + self._lastx = np.empty(0, dtype='d') + self._lasty = None def __call__(self, x): # If input is a single number, evaluate the function at that number # and return a single number - if type(x) == float or type(x) == int: + if isinstance(x, (float, int)): return self._smoothed_function(np.array([x]))[0] # If input is a list, and is different to the last input, evaluate # the function at each point. If the input is the same as last time # the function was called, return the result that was calculated # last time instead of explicity evaluating the function again. - elif self._lastx == [] or x.tolist() != self._lastx.tolist(): - self._lasty = self._smoothed_function(x) - self._lastx = x + if not np.array_equal(x, self._lastx): + self._lastx, self._lasty = x, self._smoothed_function(x) return self._lasty def _smoothed_function(self,x): @@ -87,7 +86,7 @@ def set_data(self, data, scale=1): return # Only process data of the class Data1D if not issubclass(data.__class__, Data1D): - raise ValueError("Data must be of the type DataLoader.Data1D") + raise ValueError("Correlation function cannot be computed with 2D data.") # Prepare the data new_data = Data1D(x=data.x, y=data.y) @@ -245,7 +244,7 @@ def _porod(self, q, K, sigma, bg): def _fit_guinier(self, q, iq): """Fit the Guinier region of the curve""" A = np.vstack([q**2, np.ones(q.shape)]).T - return lstsq(A, np.log(iq)) + return lstsq(A, np.log(iq), rcond=None) def _fit_porod(self, q, iq): """Fit the Porod region of the curve""" diff --git a/src/sas/sascalc/data_util/calcthread.py b/src/sas/sascalc/data_util/calcthread.py index bea2fe37dc..556cf2edfa 100644 --- a/src/sas/sascalc/data_util/calcthread.py +++ b/src/sas/sascalc/data_util/calcthread.py @@ -5,30 +5,25 @@ # from __future__ import print_function -import traceback import sys import logging +import traceback +from time import sleep + try: import _thread as thread except ImportError: # CRUFT: python 2 support import thread - -if sys.platform.count("darwin") > 0: - import time - stime = time.time() - - def clock(): - return time.time() - stime - - def sleep(t): - return time.sleep(t) -else: - from time import clock - from time import sleep +try: + from time import perf_counter as clock +except ImportError: # CRUFT: python < 3.3 + if sys.platform.count("darwin") > 0: + from time import time as clock + else: + from time import clock logger = logging.getLogger(__name__) - class CalcThread: """Threaded calculation class. Inherit from here and specialize the compute() method to perform the appropriate operations for the @@ -247,7 +242,7 @@ def exception(self): try: self.exception_handler(*sys.exc_info()) return - except Exception: + except Exception as exc: pass logger.error(traceback.format_exc()) #print 'CalcThread exception', diff --git a/src/sas/sascalc/data_util/nxsunit.py b/src/sas/sascalc/data_util/nxsunit.py index a948cc75bc..c5205d2a7f 100644 --- a/src/sas/sascalc/data_util/nxsunit.py +++ b/src/sas/sascalc/data_util/nxsunit.py @@ -98,11 +98,10 @@ def _caret_optional(s): """ Strip '^' from unit names. - * WARNING * this will incorrect transform 10^3 to 103. + * WARNING * this will incorrectly transform 10^3 to 103. """ - s.update((k.replace('^',''),v) - for k, v in list(s.items()) - if '^' in k) + stripped = [(k.replace('^',''),v) for k, v in s.items() if '^' in k] + s.update(stripped) def _build_all_units(): distance = _build_metric_units('meter','m') diff --git a/src/sas/sascalc/data_util/ordereddicttest.py b/src/sas/sascalc/data_util/ordereddicttest.py index f732f7906f..7e30fc354b 100644 --- a/src/sas/sascalc/data_util/ordereddicttest.py +++ b/src/sas/sascalc/data_util/ordereddicttest.py @@ -146,10 +146,10 @@ def test_copying(self): OrderedDict(od), ]): self.assert_(dup is not od) - self.assertEquals(dup, od) - self.assertEquals(list(dup.items()), list(od.items())) - self.assertEquals(len(dup), len(od)) - self.assertEquals(type(dup), type(od)) + self.assertEqual(dup, od) + self.assertEqual(list(dup.items()), list(od.items())) + self.assertEqual(len(dup), len(od)) + self.assertEqual(type(dup), type(od)) def test_repr(self): od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) diff --git a/src/sas/sascalc/data_util/registry.py b/src/sas/sascalc/data_util/registry.py index f20077f56f..7e3bd70a41 100644 --- a/src/sas/sascalc/data_util/registry.py +++ b/src/sas/sascalc/data_util/registry.py @@ -147,4 +147,4 @@ def load(self, path, format=None): if last_exc is not None and len(loaders) != 0: # If file has associated loader(s) and they;ve failed raise last_exc - raise NoKnownLoaderException(e.message) # raise generic exception + raise NoKnownLoaderException(str(message)) # raise generic exception diff --git a/src/sas/sascalc/data_util/uncertainty.py b/src/sas/sascalc/data_util/uncertainty.py index 13e059253d..3175e6b7dc 100644 --- a/src/sas/sascalc/data_util/uncertainty.py +++ b/src/sas/sascalc/data_util/uncertainty.py @@ -18,7 +18,7 @@ import numpy as np -from .import err1d +from . import err1d from .formatnum import format_uncertainty __all__ = ['Uncertainty'] diff --git a/src/sas/sascalc/dataloader/data_info.py b/src/sas/sascalc/dataloader/data_info.py index 052fb7f2de..28e6cd6746 100644 --- a/src/sas/sascalc/dataloader/data_info.py +++ b/src/sas/sascalc/dataloader/data_info.py @@ -25,6 +25,7 @@ from sas.sascalc.data_util.uncertainty import Uncertainty import numpy as np import math +from math import fabs class plottable_1D(object): """ @@ -655,7 +656,7 @@ def operation(a, b): return b * a return self._perform_operation(other, operation) - def __div__(self, other): + def __truediv__(self, other): """ Divided a data set by another @@ -666,8 +667,9 @@ def __div__(self, other): def operation(a, b): return a/b return self._perform_operation(other, operation) + __div__ = __truediv__ - def __rdiv__(self, other): + def __rtruediv__(self, other): """ Divided a data set by another @@ -678,6 +680,7 @@ def __rdiv__(self, other): def operation(a, b): return b/a return self._perform_operation(other, operation) + __rdiv__ = __rtruediv__ def __or__(self, other): """ @@ -799,7 +802,7 @@ def _validity_check(self, other): # Here we could also extrapolate between data points TOLERANCE = 0.01 for i in range(len(self.x)): - if math.fabs((self.x[i] - other.x[i])/self.x[i]) > TOLERANCE: + if fabs(self.x[i] - other.x[i]) > self.x[i]*TOLERANCE: msg = "Incompatible data sets: x-values do not match" raise ValueError(msg) @@ -1021,10 +1024,10 @@ def _validity_check(self, other): msg = "Unable to perform operation: data length are not equal" raise ValueError(msg) for ind in range(len(self.data)): - if math.fabs((self.qx_data[ind] - other.qx_data[ind])/self.qx_data[ind]) > TOLERANCE: + if fabs(self.qx_data[ind] - other.qx_data[ind]) > fabs(self.qx_data[ind])*TOLERANCE: msg = "Incompatible data sets: qx-values do not match: %s %s" % (self.qx_data[ind], other.qx_data[ind]) raise ValueError(msg) - if math.fabs((self.qy_data[ind] - other.qy_data[ind])/self.qy_data[ind]) > TOLERANCE: + if fabs(self.qy_data[ind] - other.qy_data[ind]) > fabs(self.qy_data[ind])*TOLERANCE: msg = "Incompatible data sets: qy-values do not match: %s %s" % (self.qy_data[ind], other.qy_data[ind]) raise ValueError(msg) diff --git a/src/sas/sascalc/dataloader/loader.py b/src/sas/sascalc/dataloader/loader.py index 691dfc629d..49d41ae2de 100644 --- a/src/sas/sascalc/dataloader/loader.py +++ b/src/sas/sascalc/dataloader/loader.py @@ -168,9 +168,9 @@ def find_plugins(self, dir): module = __import__(toks[0], globals(), locals()) if self._identify_plugin(module): readers_found += 1 - except: + except Exception as exc: msg = "Loader: Error importing " - msg += "%s\n %s" % (item, sys.exc_value) + msg += "%s\n %s" % (item, exc) logger.error(msg) # Process zip files @@ -190,14 +190,14 @@ def find_plugins(self, dir): locals(), [""]) if self._identify_plugin(module): readers_found += 1 - except: + except Exception as exc: msg = "Loader: Error importing" - msg += " %s\n %s" % (mfile, sys.exc_value) + msg += " %s\n %s" % (mfile, exc) logger.error(msg) - except: + except Exception as exc: msg = "Loader: Error importing " - msg += " %s\n %s" % (item, sys.exc_value) + msg += " %s\n %s" % (item, exc) logger.error(msg) return readers_found @@ -241,9 +241,9 @@ def associate_file_type(self, ext, module): # Append the new writer to the list self.writers[ext].append(loader.write) - except: + except Exception as exc: msg = "Loader: Error accessing" - msg += " Reader in %s\n %s" % (module.__name__, sys.exc_value) + msg += " Reader in %s\n %s" % (module.__name__, exc) logger.error(msg) return reader_found @@ -274,9 +274,9 @@ def associate_file_reader(self, ext, loader): if wcard not in self.wildcards: self.wildcards.append(wcard) - except: + except Exception as exc: msg = "Loader: Error accessing Reader " - msg += "in %s\n %s" % (loader.__name__, sys.exc_value) + msg += "in %s\n %s" % (loader.__name__, exc) logger.error(msg) return reader_found @@ -319,9 +319,9 @@ def _identify_plugin(self, module): self.writers[ext] = [] self.writers[ext].insert(0, loader.write) - except: + except Exception as exc: msg = "Loader: Error accessing Reader" - msg += " in %s\n %s" % (module.__name__, sys.exc_value) + msg += " in %s\n %s" % (module.__name__, exc) logger.error(msg) return reader_found diff --git a/src/sas/sascalc/dataloader/manipulations.py b/src/sas/sascalc/dataloader/manipulations.py index 7b5f32a8d4..47a8612e78 100644 --- a/src/sas/sascalc/dataloader/manipulations.py +++ b/src/sas/sascalc/dataloader/manipulations.py @@ -927,15 +927,14 @@ def _agv(self, data2D, run='phi'): y_counts[i_bin] += 1 # Organize the results - for i in range(self.nbins): - y[i] = y[i] / y_counts[i] - y_err[i] = math.sqrt(y_err[i]) / y_counts[i] - + with np.errstate(divide='ignore', invalid='ignore'): + y = y/y_counts + y_err = np.sqrt(y_err)/y_counts # The type of averaging: phi,q2, or q # Calculate x[i]should be at the center of the bin if run.lower() == 'phi': - x[i] = (self.phi_max - self.phi_min) / self.nbins * \ - (1.0 * i + 0.5) + self.phi_min + step = (self.phi_max - self.phi_min) / self.nbins + x = (np.arange(self.nbins) + 0.5) * step + self.phi_min else: # We take the center of ring area, not radius. # This is more accurate than taking the radial center of ring. @@ -943,8 +942,8 @@ def _agv(self, data2D, run='phi'): # r_inner = self.r_min + delta_r * i # r_outer = r_inner + delta_r # x[i] = math.sqrt((r_inner * r_inner + r_outer * r_outer) / 2) - x[i] = x[i] / y_counts[i] - y_err[y_err == 0] = np.average(y_err) + x = x/y_counts + idx = (np.isfinite(y) & np.isfinite(y_err)) if x_err is not None: d_x = x_err[idx] / y_counts[idx] diff --git a/src/sas/sascalc/dataloader/readers/associations.py b/src/sas/sascalc/dataloader/readers/associations.py index f504d5c652..08056b754b 100644 --- a/src/sas/sascalc/dataloader/readers/associations.py +++ b/src/sas/sascalc/dataloader/readers/associations.py @@ -52,7 +52,7 @@ def read_associations(loader, settings=FILE_ASSOCIATIONS): % (ext.lower(), reader)) exec("loader.associate_file_type('%s', %s)" % (ext.upper(), reader)) - except: + except Exception as exc: msg = "read_associations: skipping association" - msg += " for %s\n %s" % (ext.lower(), sys.exc_value) + msg += " for %s\n %s" % (ext.lower(), exc) logger.error(msg) diff --git a/src/sas/sascalc/dataloader/readers/cansas_reader.py b/src/sas/sascalc/dataloader/readers/cansas_reader.py index ddb085817a..37663bee3b 100644 --- a/src/sas/sascalc/dataloader/readers/cansas_reader.py +++ b/src/sas/sascalc/dataloader/readers/cansas_reader.py @@ -1238,11 +1238,10 @@ def _store_float(self, location, node, variable, storage, optional=True): try: conv = Converter(units) setattrchain(storage, variable, conv(value, units=local_unit)) - except Exception: - _, exc_value, _ = sys.exc_info() + except Exception as exc: err_mess = "CanSAS reader: could not convert" err_mess += " %s unit [%s]; expecting [%s]\n %s" \ - % (variable, units, local_unit, exc_value) + % (variable, units, local_unit, exc) self.errors.add(err_mess) if optional: logger.info(err_mess) diff --git a/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py b/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py index f234770168..944e256df0 100644 --- a/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py +++ b/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py @@ -474,8 +474,8 @@ def final_data_cleanup(self): try: for i in range(0, dataset.mask.size - 1): zeros[i] = dataset.mask[i] - except: - self.errors.add(sys.exc_value) + except Exception as exc: + self.errors.add(exc) dataset.mask = zeros # Calculate the actual Q matrix try: diff --git a/src/sas/sascalc/fit/expression.py b/src/sas/sascalc/fit/expression.py index f913319b3d..a855d74ee1 100644 --- a/src/sas/sascalc/fit/expression.py +++ b/src/sas/sascalc/fit/expression.py @@ -209,7 +209,7 @@ def eval_expressions(): """%("\n ".join(assignments),"\n ".join(code)) #print("Function: "+functiondef) - exec functiondef in globals,locals + exec(functiondef, globals, locals) retfn = locals['eval_expressions'] # Remove garbage added to globals by exec diff --git a/src/sas/sascalc/fit/models.py b/src/sas/sascalc/fit/models.py index 3f9b4aa99a..17889e293a 100644 --- a/src/sas/sascalc/fit/models.py +++ b/src/sas/sascalc/fit/models.py @@ -12,6 +12,8 @@ import py_compile import shutil +from six import reraise + from sasmodels.sasview_model import load_custom_model, load_standard_models from sas import get_user_dir @@ -61,19 +63,18 @@ def _check_plugin(model, name): return None try: new_instance = model() - except Exception: - msg = "Plugin %s error in __init__ \n\t: %s %s\n" % (str(name), - str(sys.exc_type), - sys.exc_info()[1]) + except Exception as exc: + msg = ("Plugin %s error in __init__ \n\t: %s %s\n" + % (name, type(exc), exc)) plugin_log(msg) return None if hasattr(new_instance, "function"): try: value = new_instance.function() - except Exception: + except Exception as exc: msg = "Plugin %s: error writing function \n\t :%s %s\n " % \ - (str(name), str(sys.exc_type), sys.exc_info()[1]) + (str(name), str(type(exc)), exc) plugin_log(msg) return None else: @@ -138,7 +139,7 @@ def __nonzero__(self): type, value, tb = sys.exc_info() if type is not None and issubclass(type, py_compile.PyCompileError): print("Problem with", repr(value)) - raise type, value, tb + reraise(type, value, tb) return 1 report_problem = ReportProblem() @@ -152,8 +153,8 @@ def compile_file(dir): import compileall compileall.compile_dir(dir=dir, ddir=dir, force=0, quiet=report_problem) - except Exception: - return sys.exc_info()[1] + except Exception as exc: + return exc return None @@ -184,7 +185,7 @@ def find_plugin_models(): if not model.name.startswith(PLUGIN_NAME_BASE): model.name = PLUGIN_NAME_BASE + model.name plugins[model.name] = model - except Exception: + except Exception as exc: msg = traceback.format_exc() msg += "\nwhile accessing model in %r" % path plugin_log(msg) diff --git a/src/sas/sascalc/fit/pagestate.py b/src/sas/sascalc/fit/pagestate.py index 58ba769831..f6077dcb36 100644 --- a/src/sas/sascalc/fit/pagestate.py +++ b/src/sas/sascalc/fit/pagestate.py @@ -649,9 +649,9 @@ def _get_report_string(self): file_value = "File name:" + name #Truncating string so print doesn't complain of being outside margins if sys.platform != "win32": - MAX_STRING_LENGHT = 50 - if len(file_value) > MAX_STRING_LENGHT: - file_value = "File name:.."+file_value[-MAX_STRING_LENGHT+10:] + MAX_STRING_LENGTH = 50 + if len(file_value) > MAX_STRING_LENGTH: + file_value = "File name:.."+file_value[-MAX_STRING_LENGTH+10:] file_name = CENTRE % file_value if len(title) == 0: title = name + " [" + repo_time + "]" @@ -904,7 +904,7 @@ def to_xml(self, file="fitting_state.fitv", doc=None, for model in batch_fit_state.model_list: doc_model = newdoc.createElement('model_list_item') doc_model.setAttribute('checked', str(model[0].GetValue())) - keys = model[1].keys() + keys = list(model[1].keys()) doc_model.setAttribute('name', str(keys[0])) values = model[1].get(keys[0]) doc_model.setAttribute('fit_number', str(model[2])) @@ -963,7 +963,7 @@ def from_xml(self, file=None, node=None): if node.get('version'): # Get the version for model conversion purposes - x = re.sub('[^\d.]', '', node.get('version')) + x = re.sub(r'[^\d.]', '', node.get('version')) self.version = tuple(int(e) for e in str.split(x, ".")) # The tuple must be at least 3 items long while len(self.version) < 3: @@ -983,9 +983,9 @@ def from_xml(self, file=None, node=None): if entry is not None and entry.get('epoch'): try: self.timestamp = float(entry.get('epoch')) - except Exception: + except Exception as exc: msg = "PageState.fromXML: Could not" - msg += " read timestamp\n %s" % sys.exc_value + msg += " read timestamp\n %s" % exc logger.error(msg) if entry is not None: @@ -1281,7 +1281,7 @@ def _read_cansas(self, path): if data.run_name is not None and len(data.run_name) != 0: if isinstance(data.run_name, dict): # Note: key order in dict is not guaranteed, so sort - name = data.run_name.keys()[0] + name = list(data.run_name.keys())[0] else: name = data.run_name else: diff --git a/src/sas/sascalc/invariant/invariant.py b/src/sas/sascalc/invariant/invariant.py index 0a9a99267d..2237db51ac 100644 --- a/src/sas/sascalc/invariant/invariant.py +++ b/src/sas/sascalc/invariant/invariant.py @@ -343,7 +343,8 @@ def fit(self, power=None, qmin=None, qmax=None): return [a, b], [0, math.sqrt(err)] else: A = np.vstack([linearized_data.x / linearized_data.dy, 1.0 / linearized_data.dy]).T - (p, residuals, _, _) = np.linalg.lstsq(A, linearized_data.y / linearized_data.dy) + p, residuals, _, _ = np.linalg.lstsq(A, linearized_data.y / linearized_data.dy, + rcond=None) # Get the covariance matrix, defined as inv_cov = a_transposed * a err = np.zeros(2) diff --git a/src/sas/sascalc/pr/distance_explorer.py b/src/sas/sascalc/pr/distance_explorer.py index e1cc73876c..994e9a5356 100644 --- a/src/sas/sascalc/pr/distance_explorer.py +++ b/src/sas/sascalc/pr/distance_explorer.py @@ -98,10 +98,10 @@ def __call__(self, dmin=None, dmax=None, npts=10): results.pos.append(pos) results.pos_err.append(pos_err) results.osc.append(osc) - except: + except Exception as exc: # This inversion failed, skip this D_max value msg = "ExploreDialog: inversion failed for " - msg += "D_max=%s\n %s" % (str(d), sys.exc_value) + msg += "D_max=%s\n %s" % (str(d), exc) results.errors.append(msg) return results diff --git a/src/sas/sascalc/pr/fit/expression.py b/src/sas/sascalc/pr/fit/expression.py index 3c36c1bf2b..2378d65b0e 100644 --- a/src/sas/sascalc/pr/fit/expression.py +++ b/src/sas/sascalc/pr/fit/expression.py @@ -209,7 +209,7 @@ def eval_expressions(): """%("\n ".join(assignments),"\n ".join(code)) #print("Function: "+functiondef) - exec functiondef in globals,locals + exec(functiondef, globals, locals) retfn = locals['eval_expressions'] # Remove garbage added to globals by exec diff --git a/src/sas/sascalc/pr/num_term.py b/src/sas/sascalc/pr/num_term.py index 04ffc2870e..bcdcca1a20 100644 --- a/src/sas/sascalc/pr/num_term.py +++ b/src/sas/sascalc/pr/num_term.py @@ -181,8 +181,8 @@ def load(path): data_x = np.append(data_x, test_x) data_y = np.append(data_y, test_y) data_err = np.append(data_err, err) - except: - logger.error(sys.exc_value) + except Exception as exc: + logger.error(exc) return data_x, data_y, data_err diff --git a/src/sas/sascalc/realspace/VolumeCanvas.py b/src/sas/sascalc/realspace/VolumeCanvas.py index 61552283f5..36765c6894 100644 --- a/src/sas/sascalc/realspace/VolumeCanvas.py +++ b/src/sas/sascalc/realspace/VolumeCanvas.py @@ -471,7 +471,7 @@ def getShapeList(self): """ Return a list of the shapes """ - return self.shapes.keys() + return list(self.shapes.keys()) def _addSingleShape(self, shapeDesc): """ From 88d2e70bb28d8f9c855c86d94ad8ddd63154277b Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Thu, 11 Oct 2018 14:03:40 -0400 Subject: [PATCH 2/6] support for py37 in unit tests --- test/calculatorview/test/utest_gui_sld.py | 64 ++++++++-------- test/pr_inversion/test/utest_explorer.py | 2 +- test/sascalculator/test/utest_sld.py | 54 +++++++------- test/sasdataloader/plugins/test_reader.py | 4 +- test/sasdataloader/test/utest_abs_reader.py | 10 +-- .../test/utest_generic_file_reader_class.py | 12 +-- test/sasinvariant/test/utest_data_handling.py | 26 +++---- test/sasinvariant/test/utest_use_cases.py | 74 +++++++++---------- test/sasrealspace/test/utest_oriented.py | 32 ++++---- test/sasrealspace/test/utest_realspace.py | 24 +++--- 10 files changed, 151 insertions(+), 151 deletions(-) diff --git a/test/calculatorview/test/utest_gui_sld.py b/test/calculatorview/test/utest_gui_sld.py index 7004a9046c..e7d4b69fc8 100644 --- a/test/calculatorview/test/utest_gui_sld.py +++ b/test/calculatorview/test/utest_gui_sld.py @@ -32,19 +32,19 @@ def testCompoundTextCtrl(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'pink') + self.assertTrue(bkg.GetAsString() == 'pink') #compute invariant without entering a value for compound self.sld_frame.panel.compound_ctl.SetValue("") self.sld_frame.panel.density_ctl.SetValue(str(H2O_DENSITY)) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'pink') + self.assertTrue(bkg.GetAsString() == 'pink') #compute invariant without entering a value for compound self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue(str(H2O_DENSITY)) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'white') + self.assertTrue(bkg.GetAsString() == 'white') def testDensityTextCtrl(self): """ @@ -58,19 +58,19 @@ def testDensityTextCtrl(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'pink') + self.assertTrue(bkg.GetAsString() == 'pink') #compute invariant without entering a value for density self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue("") self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'pink') + self.assertTrue(bkg.GetAsString() == 'pink') #compute invariant without entering a value for density self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue(str(H2O_DENSITY)) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'white') + self.assertTrue(bkg.GetAsString() == 'white') def testWavelengthTextCtrl(self): """ @@ -85,20 +85,20 @@ def testWavelengthTextCtrl(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() - self.assert_(bkg.GetAsString() == 'pink') + self.assertTrue(bkg.GetAsString() == 'pink') #compute invariant without entering a value for wavelegnth self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue(str(H2O_DENSITY)) self.sld_frame.panel.wavelength_ctl.SetValue("") self.sld_frame.panel.ProcessEvent(clickEvent) cp_bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(cp_bkg.GetAsString() == 'white') + self.assertTrue(cp_bkg.GetAsString() == 'white') ds_bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(ds_bkg.GetAsString() == 'white') + self.assertTrue(ds_bkg.GetAsString() == 'white') wv_bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() value = self.sld_frame.panel.wavelength_ctl.GetValue() - self.assert_(wv_bkg.GetAsString() == 'white') - self.assert_(float(value) == WAVELENGTH) + self.assertTrue(wv_bkg.GetAsString() == 'white') + self.assertTrue(float(value) == WAVELENGTH) sld_real = self.sld_frame.panel.neutron_sld_real_ctl.GetValue() sld_im = self.sld_frame.panel.neutron_sld_im_ctl.GetValue() mo_real = self.sld_frame.panel.mo_ka_sld_real_ctl.GetValue() @@ -109,19 +109,19 @@ def testWavelengthTextCtrl(self): incoh = self.sld_frame.panel.neutron_inc_ctl.GetValue() length = self.sld_frame.panel.neutron_length_ctl.GetValue() - self.assertAlmostEquals(float(sld_real), 1.04e-6, 1) - self.assertAlmostEquals(float(sld_im), -1.5e-7, 1) + self.assertAlmostEqual(float(sld_real), 1.04e-6, 1) + self.assertAlmostEqual(float(sld_im), -1.5e-7, 1) #test absorption value - self.assertAlmostEquals(float(abs) , 0.0741, 2) - self.assertAlmostEquals(float(incoh), 5.62, 2) + self.assertAlmostEqual(float(abs) , 0.0741, 2) + self.assertAlmostEqual(float(incoh), 5.62, 2) #Test length - self.assertAlmostEquals(float(length), 0.1755, 2) + self.assertAlmostEqual(float(length), 0.1755, 2) #test Cu sld - self.assertAlmostEquals(float(cu_real), 9.46e-6, 1) - self.assertAlmostEquals(float(cu_im), 3.01e-8) + self.assertAlmostEqual(float(cu_real), 9.46e-6, 1) + self.assertAlmostEqual(float(cu_im), 3.01e-8) # test Mo sld - self.assertAlmostEquals(float(mo_real), 9.43e-6) - self.assertAlmostEquals(float(mo_im), 5.65e-7, 1) + self.assertAlmostEqual(float(mo_real), 9.43e-6) + self.assertAlmostEqual(float(mo_im), 5.65e-7, 1) #compute invariant with all correct inputs value self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue(str(H2O_DENSITY)) @@ -129,8 +129,8 @@ def testWavelengthTextCtrl(self): self.sld_frame.panel.ProcessEvent(clickEvent) bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() value = self.sld_frame.panel.wavelength_ctl.GetValue() - self.assert_(bkg.GetAsString() == 'white') - self.assert_(float(value) == WAVELENGTH/2) + self.assertTrue(bkg.GetAsString() == 'white') + self.assertTrue(float(value) == WAVELENGTH/2) def testSomeCombination(self): """ @@ -144,11 +144,11 @@ def testSomeCombination(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) cp_bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(cp_bkg.GetAsString() == 'white') + self.assertTrue(cp_bkg.GetAsString() == 'white') ds_bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(ds_bkg.GetAsString() == 'white') + self.assertTrue(ds_bkg.GetAsString() == 'white') wv_bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() - self.assert_(wv_bkg.GetAsString() == 'pink') + self.assertTrue(wv_bkg.GetAsString() == 'pink') #density, wavelength is invalid self.sld_frame.panel.compound_ctl.SetValue("H2O") self.sld_frame.panel.density_ctl.SetValue("invalid density") @@ -157,11 +157,11 @@ def testSomeCombination(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) cp_bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(cp_bkg.GetAsString() == 'white') + self.assertTrue(cp_bkg.GetAsString() == 'white') ds_bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(ds_bkg.GetAsString() == 'pink') + self.assertTrue(ds_bkg.GetAsString() == 'pink') wv_bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() - self.assert_(wv_bkg.GetAsString() == 'pink') + self.assertTrue(wv_bkg.GetAsString() == 'pink') #density, wavelength is invalid self.sld_frame.panel.compound_ctl.SetValue("invalid compound") self.sld_frame.panel.density_ctl.SetValue("invalid density") @@ -170,13 +170,13 @@ def testSomeCombination(self): clickEvent = wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, id) self.sld_frame.panel.ProcessEvent(clickEvent) cp_bkg = self.sld_frame.panel.compound_ctl.GetBackgroundColour() - self.assert_(cp_bkg.GetAsString() == 'pink') + self.assertTrue(cp_bkg.GetAsString() == 'pink') ds_bkg = self.sld_frame.panel.density_ctl.GetBackgroundColour() - self.assert_(ds_bkg.GetAsString() == 'pink') + self.assertTrue(ds_bkg.GetAsString() == 'pink') wv_bkg = self.sld_frame.panel.wavelength_ctl.GetBackgroundColour() - self.assert_(wv_bkg.GetAsString() == 'white') + self.assertTrue(wv_bkg.GetAsString() == 'white') value = self.sld_frame.panel.wavelength_ctl.GetValue() - self.assert_(float(value) == WAVELENGTH) + self.assertTrue(float(value) == WAVELENGTH) diff --git a/test/pr_inversion/test/utest_explorer.py b/test/pr_inversion/test/utest_explorer.py index 827677e5b7..ee322f1eb7 100644 --- a/test/pr_inversion/test/utest_explorer.py +++ b/test/pr_inversion/test/utest_explorer.py @@ -4,9 +4,9 @@ import os.path import unittest, math, numpy -from utest_invertor import load from sas.sascalc.pr.invertor import Invertor from sas.sascalc.pr.distance_explorer import DistExplorer +from .utest_invertor import load def find(filename): diff --git a/test/sascalculator/test/utest_sld.py b/test/sascalculator/test/utest_sld.py index 1f12cab378..54bde72ff1 100644 --- a/test/sascalculator/test/utest_sld.py +++ b/test/sascalculator/test/utest_sld.py @@ -50,19 +50,19 @@ def test_neutron_sld(self): mo_real, mo_im = calculate_xray_sld(element="Mo", density=self.density, molecule_formula=self.sld_formula) #test sld - self.assertAlmostEquals(sld_real * _SCALE, -5.6e-7, 1) - self.assertAlmostEquals(sld_im * _SCALE, 0) + self.assertAlmostEqual(sld_real * _SCALE, -5.6e-7, 1) + self.assertAlmostEqual(sld_im * _SCALE, 0) #test absorption value - self.assertAlmostEquals(abs, 0.0741, 2) - self.assertAlmostEquals(incoh, 5.62, 2) + self.assertAlmostEqual(abs, 0.0741, 2) + self.assertAlmostEqual(incoh, 5.62, 2) #Test length - self.assertAlmostEquals(length, 0.1755, 3) + self.assertAlmostEqual(length, 0.1755, 3) #test Cu sld - self.assertAlmostEquals(cu_real * _SCALE, 9.46e-6, 1) - self.assertAlmostEquals(cu_im * _SCALE, 3.01e-8) + self.assertAlmostEqual(cu_real * _SCALE, 9.46e-6, 1) + self.assertAlmostEqual(cu_im * _SCALE, 3.01e-8) # test Mo sld - self.assertAlmostEquals(mo_real * _SCALE, 9.43e-6) - self.assertAlmostEquals(mo_im * _SCALE, 5.65e-7,1) + self.assertAlmostEqual(mo_real * _SCALE, 9.43e-6) + self.assertAlmostEqual(mo_im * _SCALE, 5.65e-7,1) class TestD2O(unittest.TestCase): @@ -90,19 +90,19 @@ def test_neutron_sld(self): mo_real, mo_im = calculate_xray_sld(element="Mo", density=self.density, molecule_formula=self.sld_formula) #test sld - self.assertAlmostEquals(sld_real * _SCALE, 6.33e-6, 1) - self.assertAlmostEquals(sld_im * _SCALE, 0) + self.assertAlmostEqual(sld_real * _SCALE, 6.33e-6, 1) + self.assertAlmostEqual(sld_im * _SCALE, 0) #test absorption value - self.assertAlmostEquals(abs, 1.35e-4, 2) - self.assertAlmostEquals(incoh, 0.138, 2) + self.assertAlmostEqual(abs, 1.35e-4, 2) + self.assertAlmostEqual(incoh, 0.138, 2) #Test length - self.assertAlmostEquals(length, 1.549, 3) + self.assertAlmostEqual(length, 1.549, 3) #test Cu sld - self.assertAlmostEquals(cu_real * _SCALE, 9.36e-6, 1) - self.assertAlmostEquals(cu_im * _SCALE, 2.98e-8) + self.assertAlmostEqual(cu_real * _SCALE, 9.36e-6, 1) + self.assertAlmostEqual(cu_im * _SCALE, 2.98e-8) # test Mo sld - self.assertAlmostEquals(mo_real * _SCALE, 9.33e-6) - self.assertAlmostEquals(mo_im * _SCALE, 5.59e-9,1) + self.assertAlmostEqual(mo_real * _SCALE, 9.33e-6) + self.assertAlmostEqual(mo_im * _SCALE, 5.59e-9,1) class TestCd(unittest.TestCase): @@ -130,19 +130,19 @@ def test_neutron_sld(self): mo_real, mo_im = calculate_xray_sld(element="Mo", density=self.density, molecule_formula=self.sld_formula) #test sld - self.assertAlmostEquals(sld_real * _SCALE, 1.04e-6, 1) - self.assertAlmostEquals(sld_im * _SCALE, -1.5e-7, 1) + self.assertAlmostEqual(sld_real * _SCALE, 1.04e-6, 1) + self.assertAlmostEqual(sld_im * _SCALE, -1.5e-7, 1) #test absorption value - self.assertAlmostEquals(abs, 180.0,0) - self.assertAlmostEquals(incoh, 0.0754, 2) + self.assertAlmostEqual(abs, 180.0,0) + self.assertAlmostEqual(incoh, 0.0754, 2) #Test length - self.assertAlmostEquals(length, 0.005551, 4) + self.assertAlmostEqual(length, 0.005551, 4) #test Cu sld - self.assertAlmostEquals(cu_real * _SCALE, 2.89e-5, 1) - self.assertAlmostEquals(cu_im * _SCALE, 2.81e-6) + self.assertAlmostEqual(cu_real * _SCALE, 2.89e-5, 1) + self.assertAlmostEqual(cu_im * _SCALE, 2.81e-6) # test Mo sld - self.assertAlmostEquals(mo_real * _SCALE, 2.84e-5, 1) - self.assertAlmostEquals(mo_im * _SCALE, 7.26e-7,1) + self.assertAlmostEqual(mo_real * _SCALE, 2.84e-5, 1) + self.assertAlmostEqual(mo_im * _SCALE, 7.26e-7,1) if __name__ == '__main__': unittest.main() diff --git a/test/sasdataloader/plugins/test_reader.py b/test/sasdataloader/plugins/test_reader.py index 6169bcd28d..29989927c5 100644 --- a/test/sasdataloader/plugins/test_reader.py +++ b/test/sasdataloader/plugins/test_reader.py @@ -39,7 +39,7 @@ def read(self, path): try: input_f = open(path,'r') except : - raise RuntimeError, "ascii_reader: cannot open %s" % path + raise RuntimeError("ascii_reader: cannot open %s" % path) buff = input_f.read() lines = buff.split('\n') x = np.zeros(0) @@ -54,7 +54,7 @@ def read(self, path): output.x = x return output else: - raise RuntimeError, "%s is not a file" % path + raise RuntimeError("%s is not a file" % path) return None if __name__ == "__main__": diff --git a/test/sasdataloader/test/utest_abs_reader.py b/test/sasdataloader/test/utest_abs_reader.py index ed1d4320e5..02ac3bcf7b 100644 --- a/test/sasdataloader/test/utest_abs_reader.py +++ b/test/sasdataloader/test/utest_abs_reader.py @@ -79,10 +79,10 @@ def test_usans_negative_dxl(self): data_abs = Loader().load(find("sam14_cor.ABS")) data_cor = Loader().load(find("sam14_cor.cor")) for i in range(0, len(data_abs) - 1): - self.assertEquals(data_abs.x[i], data_cor.x[i]) - self.assertEquals(data_abs.y[i], data_cor.y[i]) - self.assertEquals(data_abs.dxl[i], data_cor.dxl[i]) - self.assertEquals(data_abs.dxw[i], data_cor.dxw[i]) + self.assertEqual(data_abs.x[i], data_cor.x[i]) + self.assertEqual(data_abs.y[i], data_cor.y[i]) + self.assertEqual(data_abs.dxl[i], data_cor.dxl[i]) + self.assertEqual(data_abs.dxw[i], data_cor.dxw[i]) self.assertTrue(data_abs.dxl > 0) @@ -357,7 +357,7 @@ def test_slits(self): self.assertEqual(self.data.dxw[0], 0.001) self.assertEqual(self.data.dy[0], 3) self.assertEqual(self.data.x[1], 0.03) - self.assertAlmostEquals(self.data.y[1], 1001.0) + self.assertAlmostEqual(self.data.y[1], 1001.0) self.assertEqual(self.data.dxl[1], 0.005) self.assertEqual(self.data.dxw[1], 0.001) self.assertEqual(self.data.dy[1], 4) diff --git a/test/sasdataloader/test/utest_generic_file_reader_class.py b/test/sasdataloader/test/utest_generic_file_reader_class.py index fade103901..868b75dfd3 100644 --- a/test/sasdataloader/test/utest_generic_file_reader_class.py +++ b/test/sasdataloader/test/utest_generic_file_reader_class.py @@ -44,7 +44,7 @@ def test_old_file_types(self): f = self.generic_reader.load(self.deprecated_file_type) last_f = f[0] if hasattr(last_f, "errors"): - self.assertEquals(len(last_f.errors), 1) + self.assertEqual(len(last_f.errors), 1) else: self.fail("Errors did not propogate to the file properly.") @@ -69,14 +69,14 @@ def test_same_file_unknown_extensions(self): self.check_unknown_extension(not_xml_load[0]) self.check_unknown_extension(xml_load[0]) # Be sure the deprecation warning is passed with the file - self.assertEquals(len(asc_load[0].errors), 1) - self.assertEquals(len(nxs_load[0].errors), 1) + self.assertEqual(len(asc_load[0].errors), 1) + self.assertEqual(len(nxs_load[0].errors), 1) def check_unknown_extension(self, data): self.assertTrue(isinstance(data, Data1D)) - self.assertEquals(len(data.x), 138) - self.assertEquals(data.sample.ID, "TK49 c10_SANS") - self.assertEquals(data.meta_data["loader"], "CanSAS XML 1D") + self.assertEqual(len(data.x), 138) + self.assertEqual(data.sample.ID, "TK49 c10_SANS") + self.assertEqual(data.meta_data["loader"], "CanSAS XML 1D") def tearDown(self): if os.path.isfile(self.bad_file): diff --git a/test/sasinvariant/test/utest_data_handling.py b/test/sasinvariant/test/utest_data_handling.py index 09bd874dd4..e09c066c49 100644 --- a/test/sasinvariant/test/utest_data_handling.py +++ b/test/sasinvariant/test/utest_data_handling.py @@ -44,8 +44,8 @@ def test_fit_linear_data(self): p, dp = fit.fit() # Test results - self.assertAlmostEquals(p[0], 1.0, 5) - self.assertAlmostEquals(p[1], 0.0, 5) + self.assertAlmostEqual(p[0], 1.0, 5) + self.assertAlmostEqual(p[1], 0.0, 5) def test_fit_linear_data_with_noise(self): """ @@ -73,8 +73,8 @@ def test_fit_with_fixed_parameter(self): p, dp = fit.fit(power=-1.0) # Test results - self.assertAlmostEquals(p[0], 1.0, 5) - self.assertAlmostEquals(p[1], 0.0, 5) + self.assertAlmostEqual(p[0], 1.0, 5) + self.assertAlmostEqual(p[1], 0.0, 5) def test_fit_linear_data_with_noise_and_fixed_par(self): """ @@ -505,7 +505,7 @@ def test_low_q(self): test_y = inv._low_extrapolation_function.evaluate_model(x=self.data.x) for i in range(len(self.data.x)): value = math.fabs(test_y[i]-self.data.y[i])/self.data.y[i] - self.assert_(value < 0.001) + self.assertTrue(value < 0.001) class TestDataExtraLowSlitGuinier(unittest.TestCase): """ @@ -552,11 +552,11 @@ def test_low_q(self): qstar = inv.get_qstar(extrapolation='low') test_y = inv._low_extrapolation_function.evaluate_model(x=self.data.x[:inv._low_extrapolation_npts]) - self.assert_(len(test_y) == len(self.data.y[:inv._low_extrapolation_npts])) + self.assertTrue(len(test_y) == len(self.data.y[:inv._low_extrapolation_npts])) for i in range(inv._low_extrapolation_npts): value = math.fabs(test_y[i]-self.data.y[i])/self.data.y[i] - self.assert_(value < 0.001) + self.assertTrue(value < 0.001) def test_low_data(self): """ @@ -588,10 +588,10 @@ def test_low_data(self): data_in_range = inv.get_extra_data_low(q_start=self.data.x[0], npts = inv._low_extrapolation_npts) test_y = data_in_range.y - self.assert_(len(test_y) == len(self.data.y[:inv._low_extrapolation_npts])) + self.assertTrue(len(test_y) == len(self.data.y[:inv._low_extrapolation_npts])) for i in range(inv._low_extrapolation_npts): value = math.fabs(test_y[i]-self.data.y[i])/self.data.y[i] - self.assert_(value < 0.001) + self.assertTrue(value < 0.001) class TestDataExtraHighSlitPowerLaw(unittest.TestCase): @@ -641,11 +641,11 @@ def test_high_q(self): qstar = inv.get_qstar(extrapolation='high') test_y = inv._high_extrapolation_function.evaluate_model(x=self.data.x[start: ]) - self.assert_(len(test_y) == len(self.data.y[start:])) + self.assertTrue(len(test_y) == len(self.data.y[start:])) for i in range(len(self.data.x[start:])): value = math.fabs(test_y[i]-self.data.y[start+i])/self.data.y[start+i] - self.assert_(value < 0.001) + self.assertTrue(value < 0.001) def test_high_data(self): """ @@ -676,9 +676,9 @@ def test_high_data(self): data_in_range= inv.get_extra_data_high(q_end = max(self.data.x), npts = inv._high_extrapolation_npts) test_y = data_in_range.y - self.assert_(len(test_y) == len(self.data.y[start:])) + self.assertTrue(len(test_y) == len(self.data.y[start:])) temp = self.data.y[start:] for i in range(len(self.data.x[start:])): value = math.fabs(test_y[i]- temp[i])/temp[i] - self.assert_(value < 0.001) + self.assertTrue(value < 0.001) diff --git a/test/sasinvariant/test/utest_use_cases.py b/test/sasinvariant/test/utest_use_cases.py index 5adfc196b0..ebf47aad1d 100644 --- a/test/sasinvariant/test/utest_use_cases.py +++ b/test/sasinvariant/test/utest_use_cases.py @@ -38,8 +38,8 @@ def test_fit_line_data(self): p, dp = fit.fit(power=None) # Test results - self.assertAlmostEquals(p[0], 2.3983,3) - self.assertAlmostEquals(p[1], 0.87833,3) + self.assertAlmostEqual(p[0], 2.3983,3) + self.assertAlmostEqual(p[1], 0.87833,3) def test_fit_line_data_fixed(self): """ @@ -53,8 +53,8 @@ def test_fit_line_data_fixed(self): p, dp = fit.fit(power=-4) # Test results - self.assertAlmostEquals(p[0], 4) - self.assertAlmostEquals(p[1], -4.0676,3) + self.assertAlmostEqual(p[0], 4) + self.assertAlmostEqual(p[1], -4.0676,3) class TestLineFitNoweight(unittest.TestCase): @@ -77,8 +77,8 @@ def skip_test_fit_line_data_no_weight(self): p, dp = fit.fit(power=None) # Test results - self.assertAlmostEquals(p[0], 2.4727,3) - self.assertAlmostEquals(p[1], 0.6,3) + self.assertAlmostEqual(p[0], 2.4727,3) + self.assertAlmostEqual(p[1], 0.6,3) def test_fit_line_data_fixed_no_weight(self): """ @@ -92,8 +92,8 @@ def test_fit_line_data_fixed_no_weight(self): p, dp = fit.fit(power=-4) # Test results - self.assertAlmostEquals(p[0], 4) - self.assertAlmostEquals(p[1], -7.8,3) + self.assertAlmostEqual(p[0], 4) + self.assertAlmostEqual(p[1], -7.8,3) class TestInvPolySphere(unittest.TestCase): @@ -131,9 +131,9 @@ def test_use_case_1(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.48959e-5,2) - self.assertAlmostEquals(v, 0.005644689, 4) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.48959e-5,2) + self.assertAlmostEqual(v, 0.005644689, 4) + self.assertAlmostEqual(s , 941.7452, 3) def test_use_case_2(self): """ @@ -152,9 +152,9 @@ def test_use_case_2(self): v, dv = inv.get_volume_fraction_with_error(contrast=2.6e-6) s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.48959e-5,2) - self.assertAlmostEquals(v, 0.005644689, 1) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.48959e-5,2) + self.assertAlmostEqual(v, 0.005644689, 1) + self.assertAlmostEqual(s , 941.7452, 3) def test_use_case_3(self): """ @@ -189,9 +189,9 @@ def test_use_case_3(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.49e-5, 1) - self.assertAlmostEquals(v, 0.005648401, 4) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.49e-5, 1) + self.assertAlmostEqual(v, 0.005648401, 4) + self.assertAlmostEqual(s , 941.7452, 3) def test_use_case_4(self): """ @@ -217,9 +217,9 @@ def test_use_case_4(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.49e-5,2) - self.assertAlmostEquals(v, 0.005952674, 3) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.49e-5,2) + self.assertAlmostEqual(v, 0.005952674, 3) + self.assertAlmostEqual(s , 941.7452, 3) def test_use_case_5(self): """ @@ -246,9 +246,9 @@ def test_use_case_5(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.88981e-5,2) - self.assertAlmostEquals(v, 0.005952674, 3) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.88981e-5,2) + self.assertAlmostEqual(v, 0.005952674, 3) + self.assertAlmostEqual(s , 941.7452, 3) def test_use_case_6(self): """ @@ -272,9 +272,9 @@ def test_use_case_6(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 7.49e-5,2) - self.assertAlmostEquals(v, 0.005952674, 3) - self.assertAlmostEquals(s , 941.7452, 3) + self.assertAlmostEqual(qstar, 7.49e-5,2) + self.assertAlmostEqual(v, 0.005952674, 3) + self.assertAlmostEqual(s , 941.7452, 3) class TestInvPinholeSmear(unittest.TestCase): @@ -296,9 +296,9 @@ def test_use_case_1(self): v = inv.get_volume_fraction(contrast=2.6e-6) s = inv.get_surface(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 1.361677e-3, 4) - self.assertAlmostEquals(v, 0.115352622, 2) - self.assertAlmostEquals(s , 941.7452, 3 ) + self.assertAlmostEqual(qstar, 1.361677e-3, 4) + self.assertAlmostEqual(v, 0.115352622, 2) + self.assertAlmostEqual(s , 941.7452, 3 ) def test_use_case_2(self): """ @@ -314,9 +314,9 @@ def test_use_case_2(self): v, dv = inv.get_volume_fraction_with_error(contrast=2.6e-6) s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 1.361677e-3, 4) - self.assertAlmostEquals(v, 0.115352622, 2) - self.assertAlmostEquals(s , 941.7452, 3 ) + self.assertAlmostEqual(qstar, 1.361677e-3, 4) + self.assertAlmostEqual(v, 0.115352622, 2) + self.assertAlmostEqual(s , 941.7452, 3 ) def test_use_case_3(self): """ @@ -335,9 +335,9 @@ def test_use_case_3(self): s, ds = inv.get_surface_with_error(contrast=2.6e-6, porod_const=2) # Test results - self.assertAlmostEquals(qstar, 0.00138756,2) - self.assertAlmostEquals(v, 0.117226896,2) - self.assertAlmostEquals(s ,941.7452, 3) + self.assertAlmostEqual(qstar, 0.00138756,2) + self.assertAlmostEqual(v, 0.117226896,2) + self.assertAlmostEqual(s ,941.7452, 3) def test_use_case_4(self): """ @@ -353,7 +353,7 @@ def test_use_case_4(self): qstar, qstar_err = inv.get_qstar_with_error(extrapolation='high') # Test results - self.assertAlmostEquals(qstar, 0.0045773,2) + self.assertAlmostEqual(qstar, 0.0045773,2) def test_use_case_5(self): """ @@ -373,7 +373,7 @@ def test_use_case_5(self): qstar, qstar_err = inv.get_qstar_with_error(extrapolation='both') # Test results - self.assertAlmostEquals(qstar, 0.00460319,3) + self.assertAlmostEqual(qstar, 0.00460319,3) if __name__ == '__main__': diff --git a/test/sasrealspace/test/utest_oriented.py b/test/sasrealspace/test/utest_oriented.py index f18f534b8c..0b964d90f2 100644 --- a/test/sasrealspace/test/utest_oriented.py +++ b/test/sasrealspace/test/utest_oriented.py @@ -58,7 +58,7 @@ def testdefault(self): # Default orientation ana_val = self.ana.runXY([0.1, 0.1]) sim_val = self.model.getIq2D(0.1, 0.1) - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.1 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.1 ) class TestCylinderAddObject(unittest.TestCase): """ Tests for oriented (2D) systems """ @@ -100,7 +100,7 @@ def testalongY(self): sim_val = self.model.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) class TestCylinder(unittest.TestCase): @@ -142,7 +142,7 @@ def testalongY(self): sim_val = self.model.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) def testalongZ(self): """ Testing cylinder along Z axis """ @@ -155,7 +155,7 @@ def testalongZ(self): sim_val = self.model.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) def testalongX(self): """ Testing cylinder along X axis """ @@ -168,7 +168,7 @@ def testalongX(self): sim_val = self.model.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) class TestEllipsoid(unittest.TestCase): """ Tests for oriented (2D) systems """ @@ -212,7 +212,7 @@ def testalongX(self): sim_val = self.canvas.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) def testalongZ(self): """ Testing ellipsoid along Z """ @@ -225,7 +225,7 @@ def testalongZ(self): sim_val = self.canvas.getIq2D(0.1, 0.2) #print ana_val, sim_val, sim_val/ana_val - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) def testalongY(self): """ Testing ellipsoid along Y """ @@ -239,7 +239,7 @@ def testalongY(self): #print ana_val, sim_val, sim_val/ana_val try: - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) except Exception: print("Error", ana_val, sim_val, sim_val/ana_val) raise @@ -294,7 +294,7 @@ def testdefault(self): ana_val = self.ana.runXY([0.1, 0.2]) sim_val, err = self.canvas.getIq2DError(0.1, 0.2) - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) class TestCoreShellError(unittest.TestCase): """ Tests for oriented (2D) systems """ @@ -346,7 +346,7 @@ def testdefault(self): ana_val = self.ana.runXY([0.1, 0.2]) sim_val, err = self.canvas.getIq2DError(0.1, 0.2) - self.assert_( math.fabs(sim_val-ana_val) < 3.0 * err ) + self.assertTrue( math.fabs(sim_val-ana_val) < 3.0 * err ) class TestRunMethods(unittest.TestCase): """ Tests run methods for oriented (2D) systems """ @@ -391,7 +391,7 @@ def testRunXY_List(self): #print ana_val, sim_val, sim_val/ana_val try: - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) except Exception: print("Error", ana_val, sim_val, sim_val/ana_val) raise @@ -403,7 +403,7 @@ def testRunXY_float(self): #print ana_val, sim_val, sim_val/ana_val try: - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) except Exception: print("Error", ana_val, sim_val, sim_val/ana_val) raise @@ -415,7 +415,7 @@ def testRun_float(self): #print ana_val, sim_val, sim_val/ana_val try: - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) except Exception: print("Error", ana_val, sim_val, sim_val/ana_val) raise @@ -427,7 +427,7 @@ def testRun_list(self): #print ana_val, sim_val, sim_val/ana_val try: - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) except Exception: print("Error", ana_val, sim_val, sim_val/ana_val) raise @@ -470,7 +470,7 @@ def testalongY(self): ana_val = self.ana.runXY([0.1, 0.2]) sim_val = self.model.getIq2D(0.1, 0.2) - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) # Change the radius a re-evaluate self.ana.setParam('radius', 10) @@ -478,7 +478,7 @@ def testalongY(self): ana_val = self.ana.runXY([0.1, 0.2]) sim_val = self.model.getIq2D(0.1, 0.2) - self.assert_( math.fabs(sim_val/ana_val-1.0)<0.05 ) + self.assertTrue( math.fabs(sim_val/ana_val-1.0)<0.05 ) if __name__ == '__main__': diff --git a/test/sasrealspace/test/utest_realspace.py b/test/sasrealspace/test/utest_realspace.py index 78ec3cfd19..94b8187698 100644 --- a/test/sasrealspace/test/utest_realspace.py +++ b/test/sasrealspace/test/utest_realspace.py @@ -37,7 +37,7 @@ def testAdding(self): def testDeleting(self): self.model.add('ellipsoid','elli2') self.model.delete('elli2') - self.assert_('elli2' not in self.model.getShapeList()) + self.assertTrue('elli2' not in self.model.getShapeList()) def testsetParam(self): self.model.setParam('q_max', 0.2) @@ -80,7 +80,7 @@ def testSetDensity(self): npts_2 = vol/0.2 value_2 = self.canvas.getIq(0.001) - self.assert_( (value_1-value_2)/value_1 < 0.1) + self.assertTrue( (value_1-value_2)/value_1 < 0.1) def testSetDensityTiming(self): """Testing change in computation time with density""" @@ -98,17 +98,17 @@ def testSetDensityTiming(self): self.canvas.getIq(0.001) t_2 = time.time()-t_0 - self.assert_( t_2 < t_1 and (t_1-t_2)/t_2 > 2) + self.assertTrue( t_2 < t_1 and (t_1-t_2)/t_2 > 2) def testGetParamList(self): """ Test GetParamList on empty canvas""" - self.assert_('lores_density' in self.canvas.getParamList()) + self.assertTrue('lores_density' in self.canvas.getParamList()) handle = self.canvas.add('sphere') def testGetParamListWithShape(self): """ Test GetParamList on filled canvas""" self.canvas.add('sphere') - self.assert_('lores_density' in self.canvas.getParamList()) + self.assertTrue('lores_density' in self.canvas.getParamList()) def testAdd(self): handle = "s1" @@ -150,10 +150,10 @@ def testGetIq(self): # on the output and it should be compatible with zero # THIS WILL DEPEND ON THE NUMBER OF SPACE POINTS: # that why we need some error analysis. - self.assert_( (sim_2*ana_1/sim_1 - ana_2)/ana_2 < 0.1) + self.assertTrue( (sim_2*ana_1/sim_1 - ana_2)/ana_2 < 0.1) # test the absolute amplitude - self.assert_( math.fabs(sim_2-ana_2)/ana_2 < 0.1) + self.assertTrue( math.fabs(sim_2-ana_2)/ana_2 < 0.1) def testGetIq2(self): """ Test two different q values @@ -195,7 +195,7 @@ def testGetIq_Identical2(self): self.canvas.setParam('lores_density', 0.1) sim_2 = self.canvas.getIq(0.01) - self.assert_((sim_2-sim_1)/sim_1<0.05) + self.assertTrue((sim_2-sim_1)/sim_1<0.05) def testGetIq_time(self): """ Time profile @@ -215,7 +215,7 @@ def testGetIq_time(self): sim_2 = self.canvas.getIq(0.01) delta_2 = time.time()-t_0 - self.assert_((delta_2-delta_1)/delta_1<0.05) + self.assertTrue((delta_2-delta_1)/delta_1<0.05) def testGetPr(self): @@ -341,7 +341,7 @@ def testDefaultOrder(self): ana = self.sphere.run(0.05) val, err = self.canvas.getIqError(0.05) - self.assert_(math.fabs(ana-val)<2.0*err) + self.assertTrue(math.fabs(ana-val)<2.0*err) def testRightOrder(self): self.set_coreshell_on_canvas(3.0, 6.0) @@ -349,7 +349,7 @@ def testRightOrder(self): ana = self.sphere.run(0.05) val, err = self.canvas.getIqError(0.05) #print 'right', ana, val, err - self.assert_(math.fabs(ana-val)/ana < 1.1) + self.assertTrue(math.fabs(ana-val)/ana < 1.1) def testWrongOrder(self): self.set_coreshell_on_canvas(1, 0) @@ -364,7 +364,7 @@ def testWrongOrder(self): ana = sphere.run(0.05) val, err = self.canvas.getIqError(0.05) #print 'wrong', ana, val, err - self.assert_(math.fabs(ana-val)/ana < 1.1) + self.assertTrue(math.fabs(ana-val)/ana < 1.1) if __name__ == '__main__': From 7c4bb4dbc7861021d98b1ad347e2c9f3ed90b336 Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Thu, 11 Oct 2018 14:34:40 -0400 Subject: [PATCH 3/6] fix pylint relative import error on pr_inversion test --- test/pr_inversion/test/utest_explorer.py | 38 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/test/pr_inversion/test/utest_explorer.py b/test/pr_inversion/test/utest_explorer.py index ee322f1eb7..863c660ad8 100644 --- a/test/pr_inversion/test/utest_explorer.py +++ b/test/pr_inversion/test/utest_explorer.py @@ -2,12 +2,11 @@ Unit tests for DistExplorer class """ +import sys import os.path import unittest, math, numpy from sas.sascalc.pr.invertor import Invertor from sas.sascalc.pr.distance_explorer import DistExplorer -from .utest_invertor import load - def find(filename): return os.path.join(os.path.dirname(__file__), filename) @@ -35,6 +34,41 @@ def test_exploration(self): results = self.explo(120, 200, 25) self.assertEqual(len(results.errors), 0) self.assertEqual(len(results.chi2), 25) + + +# Note: duplicated from utest_invertor because the following failed: +#from .utest_invertor import load +def load(path = "sphere_60_q0_2.txt"): + import numpy as np + import math + import sys + # Read the data from the data file + data_x = np.zeros(0) + data_y = np.zeros(0) + data_err = np.zeros(0) + scale = None + if path is not None: + input_f = open(path,'r') + buff = input_f.read() + lines = buff.split('\n') + for line in lines: + try: + toks = line.split() + x = float(toks[0]) + y = float(toks[1]) + if len(toks)>2: + err = float(toks[2]) + else: + if scale==None: + scale = 0.15*math.sqrt(y) + err = scale*math.sqrt(y) + data_x = np.append(data_x, x) + data_y = np.append(data_y, y) + data_err = np.append(data_err, err) + except: + pass + + return data_x, data_y, data_err if __name__ == '__main__': unittest.main() From 1852b17578f93eff22ccdad62975a4cabdd21da7 Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Thu, 11 Oct 2018 19:08:17 -0400 Subject: [PATCH 4/6] fix pylint relative import error on pr_inversion test (reprise) --- test/pr_inversion/test/utest_explorer.py | 40 +++--------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/test/pr_inversion/test/utest_explorer.py b/test/pr_inversion/test/utest_explorer.py index 863c660ad8..7626ed4ad3 100644 --- a/test/pr_inversion/test/utest_explorer.py +++ b/test/pr_inversion/test/utest_explorer.py @@ -8,6 +8,11 @@ from sas.sascalc.pr.invertor import Invertor from sas.sascalc.pr.distance_explorer import DistExplorer +try: + from utest_invertor import load +except ImportError: + from .utest_invertor import load + def find(filename): return os.path.join(os.path.dirname(__file__), filename) @@ -35,40 +40,5 @@ def test_exploration(self): self.assertEqual(len(results.errors), 0) self.assertEqual(len(results.chi2), 25) - -# Note: duplicated from utest_invertor because the following failed: -#from .utest_invertor import load -def load(path = "sphere_60_q0_2.txt"): - import numpy as np - import math - import sys - # Read the data from the data file - data_x = np.zeros(0) - data_y = np.zeros(0) - data_err = np.zeros(0) - scale = None - if path is not None: - input_f = open(path,'r') - buff = input_f.read() - lines = buff.split('\n') - for line in lines: - try: - toks = line.split() - x = float(toks[0]) - y = float(toks[1]) - if len(toks)>2: - err = float(toks[2]) - else: - if scale==None: - scale = 0.15*math.sqrt(y) - err = scale*math.sqrt(y) - data_x = np.append(data_x, x) - data_y = np.append(data_y, y) - data_err = np.append(data_err, err) - except: - pass - - return data_x, data_y, data_err - if __name__ == '__main__': unittest.main() From e4e9162012e4f9bbd8ce8fe25bb99135a3a37e40 Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Wed, 24 Oct 2018 11:10:35 -0400 Subject: [PATCH 5/6] update code comments --- src/sas/sascalc/dataloader/manipulations.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/sas/sascalc/dataloader/manipulations.py b/src/sas/sascalc/dataloader/manipulations.py index 47a8612e78..fef1acd3ec 100644 --- a/src/sas/sascalc/dataloader/manipulations.py +++ b/src/sas/sascalc/dataloader/manipulations.py @@ -930,20 +930,22 @@ def _agv(self, data2D, run='phi'): with np.errstate(divide='ignore', invalid='ignore'): y = y/y_counts y_err = np.sqrt(y_err)/y_counts - # The type of averaging: phi,q2, or q - # Calculate x[i]should be at the center of the bin + # The type of averaging: phi, q2, or q + # Calculate x values at the center of the bin if run.lower() == 'phi': - step = (self.phi_max - self.phi_min) / self.nbins + step = (self.phi_max - self.phi_min) / self.nbins x = (np.arange(self.nbins) + 0.5) * step + self.phi_min else: - # We take the center of ring area, not radius. - # This is more accurate than taking the radial center of ring. - # delta_r = (self.r_max - self.r_min) / self.nbins - # r_inner = self.r_min + delta_r * i - # r_outer = r_inner + delta_r - # x[i] = math.sqrt((r_inner * r_inner + r_outer * r_outer) / 2) + # set q to the average of the q values within each bin x = x/y_counts + ### Alternate algorithm + ## We take the center of ring area, not radius. + ## This is more accurate than taking the radial center of ring. + #step = (self.r_max - self.r_min) / self.nbins + #r_inner = self.r_min + step * np.arange(self.nbins) + #x = math.sqrt((r_inner**2 + (r_inner + step)**2) / 2) + idx = (np.isfinite(y) & np.isfinite(y_err)) if x_err is not None: d_x = x_err[idx] / y_counts[idx] From 4cbb2f52e9c512eb469ba3ddf768c686364d3748 Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Thu, 21 Feb 2019 17:23:13 -0500 Subject: [PATCH 6/6] pull conflicting changes from master --- .../dataloader/readers/cansas_reader_HDF5.py | 741 +++++++++++------- test/sasdataloader/test/utest_abs_reader.py | 13 +- .../test/utest_generic_file_reader_class.py | 12 +- 3 files changed, 454 insertions(+), 312 deletions(-) diff --git a/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py b/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py index 944e256df0..ddf5ca05df 100644 --- a/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py +++ b/src/sas/sascalc/dataloader/readers/cansas_reader_HDF5.py @@ -1,5 +1,5 @@ """ - CanSAS 2D data reader for reading HDF5 formatted CanSAS files. + NXcanSAS data reader for reading HDF5 formatted CanSAS files. """ import h5py @@ -11,46 +11,42 @@ from ..data_info import plottable_1D, plottable_2D,\ Data1D, Data2D, DataInfo, Process, Aperture, Collimation, \ TransmissionSpectrum, Detector -from ..data_info import combine_data_info_with_plottable from ..loader_exceptions import FileContentsException, DefaultReaderException from ..file_reader_base_class import FileReader, decode +try: + basestring +except NameError: # CRUFT: python 2 support + basestring = str + + def h5attr(node, key, default=None): return decode(node.attrs.get(key, default)) + class Reader(FileReader): """ - A class for reading in CanSAS v2.0 data files. The existing iteration opens - Mantid generated HDF5 formatted files with file extension .h5/.H5. Any - number of data sets may be present within the file and any dimensionality - of data may be used. Currently 1D and 2D SAS data sets are supported, but - future implementations will include 1D and 2D SESANS data. - - Any number of SASdata sets may be present in a SASentry and the data within - can be either 1D I(Q) or 2D I(Qx, Qy). + A class for reading in NXcanSAS data files. The current implementation has + been tested to load data generated by multiple facilities, all of which are + known to produce NXcanSAS standards compliant data. Any number of data sets + may be present within the file and any dimensionality of data may be used. + Currently 1D and 2D SAS data sets are supported, but should be immediately + extensible to SESANS data. - Also supports reading NXcanSAS formatted HDF5 files + Any number of SASdata groups may be present in a SASentry and the data + within each SASdata group can be a single 1D I(Q), multi-framed 1D I(Q), + 2D I(Qx, Qy) or multi-framed 2D I(Qx, Qy). :Dependencies: - The CanSAS HDF5 reader requires h5py => v2.5.0 or later. + The NXcanSAS HDF5 reader requires h5py => v2.5.0 or later. """ # CanSAS version cansas_version = 2.0 - # Logged warnings or messages - logging = None - # List of errors for the current data set - errors = None - # Raw file contents to be processed - raw_data = None - # List of plottable1D objects that should be linked to the current_datainfo - data1d = None - # List of plottable2D objects that should be linked to the current_datainfo - data2d = None # Data type name - type_name = "CanSAS 2.0" + type_name = "NXcanSAS" # Wildcards - type = ["CanSAS 2.0 HDF5 Files (*.h5)|*.h5"] + type = ["NXcanSAS HDF5 Files (*.h5)|*.h5|"] # List of allowed extensions ext = ['.h5', '.H5'] # Flag to bypass extension check @@ -80,7 +76,8 @@ def get_file_contents(self): self.raw_data = h5py.File(filename, 'r') except Exception as e: if extension not in self.ext: - msg = "CanSAS2.0 HDF5 Reader could not load file {}".format(basename + extension) + msg = "NXcanSAS Reader could not load file {}".format( + basename + extension) raise DefaultReaderException(msg) raise FileContentsException(e.message) try: @@ -94,11 +91,12 @@ def get_file_contents(self): # Close the data file self.raw_data.close() - for dataset in self.output: - if isinstance(dataset, Data1D): - if dataset.x.size < 5: - self.output = [] - raise FileContentsException("Fewer than 5 data points found.") + for data_set in self.output: + if isinstance(data_set, Data1D): + if data_set.x.size < 5: + exception = FileContentsException( + "Fewer than 5 data points found.") + data_set.errors.append(exception) def reset_state(self): """ @@ -108,8 +106,18 @@ def reset_state(self): self.data1d = [] self.data2d = [] self.raw_data = None - self.errors = set() + self.multi_frame = False + self.data_frames = [] + self.data_uncertainty_frames = [] + self.errors = [] self.logging = [] + self.q_names = [] + self.mask_name = u'' + self.i_name = u'' + self.i_node = u'' + self.i_uncertainties_name = u'' + self.q_uncertainty_names = [] + self.q_resolution_names = [] self.parent_class = u'' self.detector = Detector() self.collimation = Collimation() @@ -130,6 +138,8 @@ def read_children(self, data, parent_list): # Get all information for the current key value = data.get(key) class_name = h5attr(value, u'canSAS_class') + if isinstance(class_name, (list, tuple, np.ndarray)): + class_name = class_name[0] if class_name is None: class_name = h5attr(value, u'NX_class') if class_name is not None: @@ -139,6 +149,7 @@ def read_children(self, data, parent_list): if isinstance(value, h5py.Group): # Set parent class before recursion + last_parent_class = self.parent_class self.parent_class = class_name parent_list.append(key) # If a new sasentry, store the current data sets and create @@ -146,82 +157,20 @@ def read_children(self, data, parent_list): if class_prog.match(u'SASentry'): self.add_data_set(key) elif class_prog.match(u'SASdata'): - self._initialize_new_data_set(parent_list) + self._find_data_attributes(value) + self._initialize_new_data_set(value) # Recursion step to access data within the group self.read_children(value, parent_list) - # Reset parent class when returning from recursive method - self.parent_class = class_name self.add_intermediate() + # Reset parent class when returning from recursive method + self.parent_class = last_parent_class parent_list.remove(key) elif isinstance(value, h5py.Dataset): # If this is a dataset, store the data appropriately - data_set = data[key][:] + data_set = value.value unit = self._get_unit(value) - # I and Q Data - if key == u'I': - if isinstance(self.current_dataset, plottable_2D): - self.current_dataset.data = data_set - self.current_dataset.zaxis("Intensity", unit) - else: - self.current_dataset.y = data_set.flatten() - self.current_dataset.yaxis("Intensity", unit) - continue - elif key == u'Idev': - if isinstance(self.current_dataset, plottable_2D): - self.current_dataset.err_data = data_set.flatten() - else: - self.current_dataset.dy = data_set.flatten() - continue - elif key == u'Q': - self.current_dataset.xaxis("Q", unit) - if isinstance(self.current_dataset, plottable_2D): - self.current_dataset.q = data_set.flatten() - else: - self.current_dataset.x = data_set.flatten() - continue - elif key == u'Qdev': - self.current_dataset.dx = data_set.flatten() - continue - elif key == u'dQw': - self.current_dataset.dxw = data_set.flatten() - continue - elif key == u'dQl': - self.current_dataset.dxl = data_set.flatten() - continue - elif key == u'Qy': - self.current_dataset.yaxis("Q_y", unit) - self.current_dataset.qy_data = data_set.flatten() - continue - elif key == u'Qydev': - self.current_dataset.dqy_data = data_set.flatten() - continue - elif key == u'Qx': - self.current_dataset.xaxis("Q_x", unit) - self.current_dataset.qx_data = data_set.flatten() - continue - elif key == u'Qxdev': - self.current_dataset.dqx_data = data_set.flatten() - continue - elif key == u'Mask': - self.current_dataset.mask = data_set.flatten() - continue - # Transmission Spectrum - elif (key == u'T' - and self.parent_class == u'SAStransmission_spectrum'): - self.trans_spectrum.transmission = data_set.flatten() - continue - elif (key == u'Tdev' - and self.parent_class == u'SAStransmission_spectrum'): - self.trans_spectrum.transmission_deviation = \ - data_set.flatten() - continue - elif (key == u'lambda' - and self.parent_class == u'SAStransmission_spectrum'): - self.trans_spectrum.wavelength = data_set.flatten() - continue - for data_point in data_set: if isinstance(data_point, np.ndarray): if data_point.dtype.char == 'S': @@ -230,178 +179,69 @@ def read_children(self, data, parent_list): data_point = decode(data_point) # Top Level Meta Data if key == u'definition': - self.current_datainfo.meta_data['reader'] = data_point + if isinstance(data_set, basestring): + self.current_datainfo.meta_data['reader'] = data_set + break + else: + self.current_datainfo.meta_data[ + 'reader'] = data_point + # Run elif key == u'run': - self.current_datainfo.run.append(data_point) try: run_name = h5attr(value, 'name') - run_dict = {data_point: run_name} + run_dict = {data_set: run_name} self.current_datainfo.run_name = run_dict except Exception: pass + if isinstance(data_set, basestring): + self.current_datainfo.run.append(data_set) + break + else: + self.current_datainfo.run.append(data_point) + # Title elif key == u'title': - self.current_datainfo.title = data_point + if isinstance(data_set, basestring): + self.current_datainfo.title = data_set + break + else: + self.current_datainfo.title = data_point + # Note elif key == u'SASnote': - self.current_datainfo.notes.append(data_point) - + self.current_datainfo.notes.append(data_set) + break # Sample Information - # CanSAS 2.0 format - elif key == u'Title' and self.parent_class == u'SASsample': - self.current_datainfo.sample.name = data_point - # NXcanSAS format - elif key == u'name' and self.parent_class == u'SASsample': - self.current_datainfo.sample.name = data_point - # NXcanSAS format - elif key == u'ID' and self.parent_class == u'SASsample': - self.current_datainfo.sample.name = data_point - elif (key == u'thickness' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.thickness = data_point - elif (key == u'temperature' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.temperature = data_point - elif (key == u'transmission' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.transmission = data_point - elif (key == u'x_position' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.position.x = data_point - elif (key == u'y_position' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.position.y = data_point - elif key == u'pitch' and self.parent_class == u'SASsample': - self.current_datainfo.sample.orientation.x = data_point - elif key == u'yaw' and self.parent_class == u'SASsample': - self.current_datainfo.sample.orientation.y = data_point - elif key == u'roll' and self.parent_class == u'SASsample': - self.current_datainfo.sample.orientation.z = data_point - elif (key == u'details' - and self.parent_class == u'SASsample'): - self.current_datainfo.sample.details.append(data_point) - + elif self.parent_class == u'SASsample': + self.process_sample(data_point, key) # Instrumental Information elif (key == u'name' and self.parent_class == u'SASinstrument'): self.current_datainfo.instrument = data_point - elif key == u'name' and self.parent_class == u'SASdetector': - self.detector.name = data_point - elif key == u'SDD' and self.parent_class == u'SASdetector': - self.detector.distance = float(data_point) - self.detector.distance_unit = unit - elif (key == u'slit_length' - and self.parent_class == u'SASdetector'): - self.detector.slit_length = float(data_point) - self.detector.slit_length_unit = unit - elif (key == u'x_position' - and self.parent_class == u'SASdetector'): - self.detector.offset.x = float(data_point) - self.detector.offset_unit = unit - elif (key == u'y_position' - and self.parent_class == u'SASdetector'): - self.detector.offset.y = float(data_point) - self.detector.offset_unit = unit - elif (key == u'pitch' - and self.parent_class == u'SASdetector'): - self.detector.orientation.x = float(data_point) - self.detector.orientation_unit = unit - elif key == u'roll' and self.parent_class == u'SASdetector': - self.detector.orientation.z = float(data_point) - self.detector.orientation_unit = unit - elif key == u'yaw' and self.parent_class == u'SASdetector': - self.detector.orientation.y = float(data_point) - self.detector.orientation_unit = unit - elif (key == u'beam_center_x' - and self.parent_class == u'SASdetector'): - self.detector.beam_center.x = float(data_point) - self.detector.beam_center_unit = unit - elif (key == u'beam_center_y' - and self.parent_class == u'SASdetector'): - self.detector.beam_center.y = float(data_point) - self.detector.beam_center_unit = unit - elif (key == u'x_pixel_size' - and self.parent_class == u'SASdetector'): - self.detector.pixel_size.x = float(data_point) - self.detector.pixel_size_unit = unit - elif (key == u'y_pixel_size' - and self.parent_class == u'SASdetector'): - self.detector.pixel_size.y = float(data_point) - self.detector.pixel_size_unit = unit - elif (key == u'distance' - and self.parent_class == u'SAScollimation'): - self.collimation.length = data_point - self.collimation.length_unit = unit - elif (key == u'name' - and self.parent_class == u'SAScollimation'): - self.collimation.name = data_point - elif (key == u'shape' - and self.parent_class == u'SASaperture'): - self.aperture.shape = data_point - elif (key == u'x_gap' - and self.parent_class == u'SASaperture'): - self.aperture.size.x = data_point - elif (key == u'y_gap' - and self.parent_class == u'SASaperture'): - self.aperture.size.y = data_point - + # Detector + elif self.parent_class == u'SASdetector': + self.process_detector(data_point, key, unit) + # Collimation + elif self.parent_class == u'SAScollimation': + self.process_collimation(data_point, key, unit) + # Aperture + elif self.parent_class == u'SASaperture': + self.process_aperture(data_point, key) # Process Information - elif (key == u'Title' - and self.parent_class == u'SASprocess'): # CanSAS 2.0 - self.process.name = data_point - elif (key == u'name' - and self.parent_class == u'SASprocess'): # NXcanSAS - self.process.name = data_point - elif (key == u'description' - and self.parent_class == u'SASprocess'): - self.process.description = data_point - elif key == u'date' and self.parent_class == u'SASprocess': - self.process.date = data_point - elif key == u'term' and self.parent_class == u'SASprocess': - self.process.term = data_point - elif self.parent_class == u'SASprocess': - self.process.notes.append(data_point) - + elif self.parent_class == u'SASprocess': # CanSAS 2.0 + self.process_process(data_point, key) # Source - elif (key == u'wavelength' - and self.parent_class == u'SASdata'): - self.current_datainfo.source.wavelength = data_point - self.current_datainfo.source.wavelength_unit = unit - elif (key == u'incident_wavelength' - and self.parent_class == 'SASsource'): - self.current_datainfo.source.wavelength = data_point - self.current_datainfo.source.wavelength_unit = unit - elif (key == u'wavelength_max' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.wavelength_max = data_point - self.current_datainfo.source.wavelength_max_unit = unit - elif (key == u'wavelength_min' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.wavelength_min = data_point - self.current_datainfo.source.wavelength_min_unit = unit - elif (key == u'incident_wavelength_spread' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.wavelength_spread = \ - data_point - self.current_datainfo.source.wavelength_spread_unit = \ - unit - elif (key == u'beam_size_x' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.beam_size.x = data_point - self.current_datainfo.source.beam_size_unit = unit - elif (key == u'beam_size_y' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.beam_size.y = data_point - self.current_datainfo.source.beam_size_unit = unit - elif (key == u'beam_shape' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.beam_shape = data_point - elif (key == u'radiation' - and self.parent_class == u'SASsource'): - self.current_datainfo.source.radiation = data_point - elif (key == u'transmission' - and self.parent_class == u'SASdata'): - self.current_datainfo.sample.transmission = data_point - + elif self.parent_class == u'SASsource': + self.process_source(data_point, key, unit) # Everything else goes in meta_data + elif self.parent_class == u'SASdata': + if isinstance(self.current_dataset, plottable_2D): + self.process_2d_data_object(data_set, key, unit) + else: + self.process_1d_data_object(data_set, key, unit) + + break + elif self.parent_class == u'SAStransmission_spectrum': + self.process_trans_spectrum(data_set, key) + break else: new_key = self._create_unique_key( self.current_datainfo.meta_data, key) @@ -409,7 +249,259 @@ def read_children(self, data, parent_list): else: # I don't know if this reachable code - self.errors.add("ShouldNeverHappenException") + self.errors.append("ShouldNeverHappenException") + + def process_1d_data_object(self, data_set, key, unit): + """ + SASdata processor method for 1d data items + :param data_set: data from HDF5 file + :param key: canSAS_class attribute + :param unit: unit attribute + """ + if key == self.i_name: + if self.multi_frame: + for x in range(0, data_set.shape[0]): + self.data_frames.append(data_set[x].flatten()) + else: + self.current_dataset.y = data_set.flatten() + self.current_dataset.yaxis("Intensity", unit) + elif key == self.i_uncertainties_name: + if self.multi_frame: + for x in range(0, data_set.shape[0]): + self.data_uncertainty_frames.append(data_set[x].flatten()) + self.current_dataset.dy = data_set.flatten() + elif key in self.q_names: + self.current_dataset.xaxis("Q", unit) + self.current_dataset.x = data_set.flatten() + elif key in self.q_resolution_names: + if (len(self.q_resolution_names) > 1 + and np.where(self.q_resolution_names == key)[0] == 0): + self.current_dataset.dxw = data_set.flatten() + elif (len(self.q_resolution_names) > 1 + and np.where(self.q_resolution_names == key)[0] == 1): + self.current_dataset.dxl = data_set.flatten() + else: + self.current_dataset.dx = data_set.flatten() + elif key in self.q_uncertainty_names: + if (len(self.q_uncertainty_names) > 1 + and np.where(self.q_uncertainty_names == key)[0] == 0): + self.current_dataset.dxw = data_set.flatten() + elif (len(self.q_uncertainty_names) > 1 + and np.where(self.q_uncertainty_names == key)[0] == 1): + self.current_dataset.dxl = data_set.flatten() + else: + self.current_dataset.dx = data_set.flatten() + elif key == self.mask_name: + self.current_dataset.mask = data_set.flatten() + elif key == u'wavelength': + self.current_datainfo.source.wavelength = data_set[0] + self.current_datainfo.source.wavelength_unit = unit + + def process_2d_data_object(self, data_set, key, unit): + if key == self.i_name: + self.current_dataset.data = data_set + self.current_dataset.zaxis("Intensity", unit) + elif key == self.i_uncertainties_name: + self.current_dataset.err_data = data_set.flatten() + elif key in self.q_names: + self.current_dataset.xaxis("Q_x", unit) + self.current_dataset.yaxis("Q_y", unit) + if self.q_names[0] == self.q_names[1]: + # All q data in a single array + self.current_dataset.qx_data = data_set[0] + self.current_dataset.qy_data = data_set[1] + elif self.q_names.index(key) == 0: + self.current_dataset.qx_data = data_set + elif self.q_names.index(key) == 1: + self.current_dataset.qy_data = data_set + elif key in self.q_uncertainty_names or key in self.q_resolution_names: + if ((self.q_uncertainty_names[0] == self.q_uncertainty_names[1]) or + (self.q_resolution_names[0] == self.q_resolution_names[1])): + # All q data in a single array + self.current_dataset.dqx_data = data_set[0].flatten() + self.current_dataset.dqy_data = data_set[1].flatten() + elif (self.q_uncertainty_names.index(key) == 0 or + self.q_resolution_names.index(key) == 0): + self.current_dataset.dqx_data = data_set.flatten() + elif (self.q_uncertainty_names.index(key) == 1 or + self.q_resolution_names.index(key) == 1): + self.current_dataset.dqy_data = data_set.flatten() + self.current_dataset.yaxis("Q_y", unit) + elif key == self.mask_name: + self.current_dataset.mask = data_set.flatten() + elif key == u'Qy': + self.current_dataset.yaxis("Q_y", unit) + self.current_dataset.qy_data = data_set.flatten() + elif key == u'Qydev': + self.current_dataset.dqy_data = data_set.flatten() + elif key == u'Qx': + self.current_dataset.xaxis("Q_x", unit) + self.current_dataset.qx_data = data_set.flatten() + elif key == u'Qxdev': + self.current_dataset.dqx_data = data_set.flatten() + + def process_trans_spectrum(self, data_set, key): + """ + SAStransmission_spectrum processor + :param data_set: data from HDF5 file + :param key: canSAS_class attribute + """ + if key == u'T': + self.trans_spectrum.transmission = data_set.flatten() + elif key == u'Tdev': + self.trans_spectrum.transmission_deviation = data_set.flatten() + elif key == u'lambda': + self.trans_spectrum.wavelength = data_set.flatten() + + def process_sample(self, data_point, key): + """ + SASsample processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + """ + if key == u'Title': + self.current_datainfo.sample.name = data_point + elif key == u'name': + self.current_datainfo.sample.name = data_point + elif key == u'ID': + self.current_datainfo.sample.name = data_point + elif key == u'thickness': + self.current_datainfo.sample.thickness = data_point + elif key == u'temperature': + self.current_datainfo.sample.temperature = data_point + elif key == u'transmission': + self.current_datainfo.sample.transmission = data_point + elif key == u'x_position': + self.current_datainfo.sample.position.x = data_point + elif key == u'y_position': + self.current_datainfo.sample.position.y = data_point + elif key == u'pitch': + self.current_datainfo.sample.orientation.x = data_point + elif key == u'yaw': + self.current_datainfo.sample.orientation.y = data_point + elif key == u'roll': + self.current_datainfo.sample.orientation.z = data_point + elif key == u'details': + self.current_datainfo.sample.details.append(data_point) + + def process_detector(self, data_point, key, unit): + """ + SASdetector processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + :param unit: unit attribute from data set + """ + if key == u'name': + self.detector.name = data_point + elif key == u'SDD': + self.detector.distance = float(data_point) + self.detector.distance_unit = unit + elif key == u'slit_length': + self.detector.slit_length = float(data_point) + self.detector.slit_length_unit = unit + elif key == u'x_position': + self.detector.offset.x = float(data_point) + self.detector.offset_unit = unit + elif key == u'y_position': + self.detector.offset.y = float(data_point) + self.detector.offset_unit = unit + elif key == u'pitch': + self.detector.orientation.x = float(data_point) + self.detector.orientation_unit = unit + elif key == u'roll': + self.detector.orientation.z = float(data_point) + self.detector.orientation_unit = unit + elif key == u'yaw': + self.detector.orientation.y = float(data_point) + self.detector.orientation_unit = unit + elif key == u'beam_center_x': + self.detector.beam_center.x = float(data_point) + self.detector.beam_center_unit = unit + elif key == u'beam_center_y': + self.detector.beam_center.y = float(data_point) + self.detector.beam_center_unit = unit + elif key == u'x_pixel_size': + self.detector.pixel_size.x = float(data_point) + self.detector.pixel_size_unit = unit + elif key == u'y_pixel_size': + self.detector.pixel_size.y = float(data_point) + self.detector.pixel_size_unit = unit + + def process_collimation(self, data_point, key, unit): + """ + SAScollimation processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + :param unit: unit attribute from data set + """ + if key == u'distance': + self.collimation.length = data_point + self.collimation.length_unit = unit + elif key == u'name': + self.collimation.name = data_point + + def process_aperture(self, data_point, key): + """ + SASaperture processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + """ + if key == u'shape': + self.aperture.shape = data_point + elif key == u'x_gap': + self.aperture.size.x = data_point + elif key == u'y_gap': + self.aperture.size.y = data_point + + def process_source(self, data_point, key, unit): + """ + SASsource processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + :param unit: unit attribute from data set + """ + if key == u'incident_wavelength': + self.current_datainfo.source.wavelength = data_point + self.current_datainfo.source.wavelength_unit = unit + elif key == u'wavelength_max': + self.current_datainfo.source.wavelength_max = data_point + self.current_datainfo.source.wavelength_max_unit = unit + elif key == u'wavelength_min': + self.current_datainfo.source.wavelength_min = data_point + self.current_datainfo.source.wavelength_min_unit = unit + elif key == u'incident_wavelength_spread': + self.current_datainfo.source.wavelength_spread = data_point + self.current_datainfo.source.wavelength_spread_unit = unit + elif key == u'beam_size_x': + self.current_datainfo.source.beam_size.x = data_point + self.current_datainfo.source.beam_size_unit = unit + elif key == u'beam_size_y': + self.current_datainfo.source.beam_size.y = data_point + self.current_datainfo.source.beam_size_unit = unit + elif key == u'beam_shape': + self.current_datainfo.source.beam_shape = data_point + elif key == u'radiation': + self.current_datainfo.source.radiation = data_point + + def process_process(self, data_point, key): + """ + SASprocess processor + :param data_point: Single point from an HDF5 data file + :param key: class name data_point was taken from + """ + term_match = re.compile(u'^term[0-9]+$') + if key == u'Title': # CanSAS 2.0 + self.process.name = data_point + elif key == u'name': # NXcanSAS + self.process.name = data_point + elif key == u'description': + self.process.description = data_point + elif key == u'date': + self.process.date = data_point + elif term_match.match(key): + self.process.term.append(data_point) + else: + self.process.notes.append(data_point) def add_intermediate(self): """ @@ -439,7 +531,15 @@ def add_intermediate(self): if isinstance(self.current_dataset, plottable_2D): self.data2d.append(self.current_dataset) elif isinstance(self.current_dataset, plottable_1D): - self.data1d.append(self.current_dataset) + if self.multi_frame: + for x in range(0, len(self.data_frames)): + self.current_dataset.y = self.data_frames[x] + if len(self.data_uncertainty_frames) > x: + self.current_dataset.dy = \ + self.data_uncertainty_frames[x] + self.data1d.append(self.current_dataset) + else: + self.data1d.append(self.current_dataset) def final_data_cleanup(self): """ @@ -451,13 +551,9 @@ def final_data_cleanup(self): if len(self.current_datainfo.trans_spectrum) > 0: spectrum_list = [] for spectrum in self.current_datainfo.trans_spectrum: - spectrum.transmission = np.delete(spectrum.transmission, [0]) spectrum.transmission = spectrum.transmission.astype(np.float64) - spectrum.transmission_deviation = np.delete( - spectrum.transmission_deviation, [0]) spectrum.transmission_deviation = \ spectrum.transmission_deviation.astype(np.float64) - spectrum.wavelength = np.delete(spectrum.wavelength, [0]) spectrum.wavelength = spectrum.wavelength.astype(np.float64) if len(spectrum.transmission) > 0: spectrum_list.append(spectrum) @@ -465,33 +561,33 @@ def final_data_cleanup(self): # Append errors to dataset and reset class errors self.current_datainfo.errors = self.errors - self.errors.clear() + self.errors = [] # Combine all plottables with datainfo and append each to output # Type cast data arrays to float64 and find min/max as appropriate for dataset in self.data2d: - zeros = np.ones(dataset.data.size, dtype=bool) - try: - for i in range(0, dataset.mask.size - 1): - zeros[i] = dataset.mask[i] - except Exception as exc: - self.errors.add(exc) - dataset.mask = zeros # Calculate the actual Q matrix try: if dataset.q_data.size <= 1: dataset.q_data = np.sqrt(dataset.qx_data * dataset.qx_data + dataset.qy_data - * dataset.qy_data) + * dataset.qy_data).flatten() except: dataset.q_data = None if dataset.data.ndim == 2: - (n_rows, n_cols) = dataset.data.shape - dataset.y_bins = dataset.qy_data[0::n_cols] - dataset.x_bins = dataset.qx_data[:n_cols] + dataset.y_bins = np.unique(dataset.qy_data.flatten()) + dataset.x_bins = np.unique(dataset.qx_data.flatten()) dataset.data = dataset.data.flatten() + dataset.qx_data = dataset.qx_data.flatten() + dataset.qy_data = dataset.qy_data.flatten() + + try: + iter(dataset.mask) + dataset.mask = np.invert(np.asarray(dataset.mask, dtype=bool)) + except TypeError: + dataset.mask = np.ones(dataset.data.shape, dtype=bool) self.current_dataset = dataset self.send_to_output() @@ -510,12 +606,13 @@ def add_data_set(self, key=""): if self.current_datainfo and self.current_dataset: self.final_data_cleanup() + self.data_frames = [] + self.data_uncertainty_frames = [] self.data1d = [] self.data2d = [] self.current_datainfo = DataInfo() - - def _initialize_new_data_set(self, parent_list=None): + def _initialize_new_data_set(self, value=None): """ A private class method to generate a new 1D or 2D data object based on the type of data within the set. Outside methods should call @@ -523,10 +620,7 @@ def _initialize_new_data_set(self, parent_list=None): :param parent_list: List of names of parent elements """ - - if parent_list is None: - parent_list = [] - if self._find_intermediate(parent_list, "Qx"): + if self._is_2d_not_multi_frame(value): self.current_dataset = plottable_2D() else: x = np.array(0) @@ -534,26 +628,80 @@ def _initialize_new_data_set(self, parent_list=None): self.current_dataset = plottable_1D(x, y) self.current_datainfo.filename = self.raw_data.filename - def _find_intermediate(self, parent_list, basename=""): + @staticmethod + def as_list_or_array(iterable): """ - A private class used to find an entry by either using a direct key or - knowing the approximate basename. - - :param parent_list: List of parents nodes in the HDF5 file - :param basename: Approximate name of an entry to search for + Return value as a list if not already a list or array. + :param iterable: :return: """ + if not (isinstance(iterable, np.ndarray) or isinstance(iterable, list)): + iterable = iterable.split(",") if isinstance(iterable, basestring)\ + else [iterable] + return iterable - entry = False - key_prog = re.compile(basename) - top = self.raw_data - for parent in parent_list: - top = top.get(parent) - for key in top.keys(): - if key_prog.match(key): - entry = True - break - return entry + def _find_data_attributes(self, value): + """ + A class to find the indices for Q, the name of the Qdev and Idev, and + the name of the mask. + :param value: SASdata/NXdata HDF5 Group + """ + # Initialize values to base types + self.mask_name = u'' + self.i_name = u'' + self.i_node = u'' + self.i_uncertainties_name = u'' + self.q_names = [] + self.q_uncertainty_names = [] + self.q_resolution_names = [] + # Get attributes + attrs = value.attrs + signal = attrs.get("signal", "I") + i_axes = attrs.get("I_axes", ["Q"]) + q_indices = attrs.get("Q_indices", [0]) + i_axes = self.as_list_or_array(i_axes) + keys = value.keys() + # Assign attributes to appropriate class variables + self.q_names = [i_axes[int(v)] for v in self.as_list_or_array(q_indices)] + self.mask_name = attrs.get("mask") + self.i_name = signal + self.i_node = value.get(self.i_name) + for item in self.q_names: + if item in keys: + q_vals = value.get(item) + if q_vals.attrs.get("uncertainties") is not None: + self.q_uncertainty_names = q_vals.attrs.get("uncertainties") + elif q_vals.attrs.get("uncertainty") is not None: + self.q_uncertainty_names = q_vals.attrs.get("uncertainty") + if isinstance(self.q_uncertainty_names, basestring): + self.q_uncertainty_names = self.q_uncertainty_names.split(",") + if q_vals.attrs.get("resolutions") is not None: + self.q_resolution_names = q_vals.attrs.get("resolutions") + if isinstance(self.q_resolution_names, basestring): + self.q_resolution_names = self.q_resolution_names.split(",") + if self.i_name in keys: + i_vals = value.get(self.i_name) + self.i_uncertainties_name = i_vals.attrs.get("uncertainties") + if self.i_uncertainties_name is None: + self.i_uncertainties_name = i_vals.attrs.get("uncertainty") + + def _is_2d_not_multi_frame(self, value, i_base="", q_base=""): + """ + A private class to determine if the data set is 1d or 2d. + + :param value: Nexus/NXcanSAS data group + :param basename: Approximate name of an entry to search for + :return: True if 2D, otherwise false + """ + i_basename = i_base if i_base != "" else self.i_name + i_vals = value.get(i_basename) + q_basename = q_base if q_base != "" else self.q_names + q_vals = value.get(q_basename[0]) + self.multi_frame = (i_vals is not None and q_vals is not None + and len(i_vals.shape) != 1 + and len(q_vals.shape) == 1) + return (i_vals is not None and len(i_vals.shape) != 1 + and not self.multi_frame) def _create_unique_key(self, dictionary, name, numb=0): """ @@ -582,9 +730,4 @@ def _get_unit(self, value): unit = h5attr(value, u'units') if unit is None: unit = h5attr(value, u'unit') - # Convert the unit formats - if unit == "1/A": - unit = "A^{-1}" - elif unit == "1/cm": - unit = "cm^{-1}" return unit diff --git a/test/sasdataloader/test/utest_abs_reader.py b/test/sasdataloader/test/utest_abs_reader.py index 02ac3bcf7b..7193f38abc 100644 --- a/test/sasdataloader/test/utest_abs_reader.py +++ b/test/sasdataloader/test/utest_abs_reader.py @@ -76,14 +76,13 @@ def test_generic_loader(self): self.assertEqual(data[0].meta_data['loader'], "IGOR 1D") def test_usans_negative_dxl(self): - data_abs = Loader().load(find("sam14_cor.ABS")) - data_cor = Loader().load(find("sam14_cor.cor")) - for i in range(0, len(data_abs) - 1): + data_abs = Loader().load(find("sam14_cor.ABS"))[0] + data_cor = Loader().load(find("sam14_cor.txt"))[0] + for i in range(0, len(data_abs.x) - 1): self.assertEqual(data_abs.x[i], data_cor.x[i]) self.assertEqual(data_abs.y[i], data_cor.y[i]) - self.assertEqual(data_abs.dxl[i], data_cor.dxl[i]) - self.assertEqual(data_abs.dxw[i], data_cor.dxw[i]) - self.assertTrue(data_abs.dxl > 0) + self.assertEqual(data_abs.dxl[i], -data_cor.dx[i]) + self.assertTrue(data_abs.dxl[i] > 0) class DanseReaderTests(unittest.TestCase): @@ -117,7 +116,7 @@ def test_checkdata(self): self.assertEqual(self.data.detector[0].beam_center.x, center_x) self.assertEqual(self.data.detector[0].beam_center.y, center_y) - self.assertEqual(self.data.I_unit, '1/cm') + self.assertEqual(self.data.I_unit, 'cm^{-1}') self.assertEqual(self.data.data[0], 1.57831) self.assertEqual(self.data.data[1], 2.70983) self.assertEqual(self.data.data[2], 3.83422) diff --git a/test/sasdataloader/test/utest_generic_file_reader_class.py b/test/sasdataloader/test/utest_generic_file_reader_class.py index 868b75dfd3..c2e031ecd6 100644 --- a/test/sasdataloader/test/utest_generic_file_reader_class.py +++ b/test/sasdataloader/test/utest_generic_file_reader_class.py @@ -50,13 +50,13 @@ def test_old_file_types(self): def test_same_file_unknown_extensions(self): # Five files, all with the same content, but different file extensions - no_ext = find("test_data//TestExtensions") - not_xml = find("test_data//TestExtensions.notxml") + no_ext = find("test_data" + os.sep + "TestExtensions") + not_xml = find("test_data" + os.sep + "TestExtensions.notxml") # Deprecated extensions - asc_dep = find("test_data//TestExtensions.asc") - nxs_dep = find("test_data//TestExtensions.nxs") + asc_dep = find("test_data" + os.sep + "TestExtensions.asc") + nxs_dep = find("test_data" + os.sep + "TestExtensions.nxs") # Native extension as a baseline - xml_native = find("test_data//TestExtensions.xml") + xml_native = find("test_data" + os.sep + "TestExtensions.xml") # Load the files and check contents no_ext_load = self.generic_reader.load(no_ext) asc_load = self.generic_reader.load(asc_dep) @@ -70,7 +70,7 @@ def test_same_file_unknown_extensions(self): self.check_unknown_extension(xml_load[0]) # Be sure the deprecation warning is passed with the file self.assertEqual(len(asc_load[0].errors), 1) - self.assertEqual(len(nxs_load[0].errors), 1) + self.assertEqual(len(nxs_load[0].errors), 0) def check_unknown_extension(self, data): self.assertTrue(isinstance(data, Data1D))