diff --git a/pysd/py_backend/external.py b/pysd/py_backend/external.py index 25c0e885..5401d5f6 100644 --- a/pysd/py_backend/external.py +++ b/pysd/py_backend/external.py @@ -19,6 +19,9 @@ from .lookups import Lookups +_SPREADSHEET_EXTS = {'.xls', '.xlsx', '.xlsm', '.xlsb', '.odf', '.ods', '.odt'} + + class Excels(): """ Class to save the read Excel files and thus avoid double reading @@ -36,20 +39,16 @@ def read(cls, file_name, tab): # get the function to read the data based on its extension read_kwargs = {} ext = file_name.suffix.lower() - if ext in ['.xls', '.xlsx', '.xlsm', 'xlsb', 'odf', 'ods', 'odt']: + if ext in _SPREADSHEET_EXTS: read_func = pd.read_excel read_kwargs['sheet_name'] = tab elif ext == '.csv': - # TODO test read_func = pd.read_csv if tab and not tab[0].isalnum(): - # TODO test read_kwargs['sep'] = tab else: - # TODO test read_func = pd.read_table if tab and not tab[0].isalnum(): - # TODO test read_kwargs['sep'] = tab # read the data excel = np.array([ @@ -177,7 +176,6 @@ def _get_data_from_file_opyxl(self, cellname): try: excel = Excels.read_opyxl(self.file) except InvalidFileException: - # TODO test raise ValueError( self.py_name + "\n" f"Cannot read the file '{self.file}'...\n" @@ -1084,20 +1082,16 @@ def get_subscripts_cell(self, row_first, col_first, lastcell): # get the function to read the data based on its extension ext = self.file.suffix.lower() - if ext in ['.xls', '.xlsx', '.xlsm', 'xlsb', 'odf', 'ods', 'odt']: + if ext in _SPREADSHEET_EXTS: read_func = pd.read_excel read_kwargs['sheet_name'] = self.tab elif ext == '.csv': - # TODO test read_func = pd.read_csv if self.tab and not self.tab[0].isalnum(): - # TODO test read_kwargs['sep'] = self.tab else: - # TODO test read_func = pd.read_table if self.tab and not self.tab[0].isalnum(): - # TODO test read_kwargs['sep'] = self.tab # read the data @@ -1120,7 +1114,6 @@ def get_subscripts_name(self, cellname): try: excel = load_workbook(self.file, read_only=True, data_only=True) except InvalidFileException: - # TODO test raise ValueError( self.py_name + "\n" f"Cannot read the file '{self.file}'...\n" diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 5d97eb1c..dcce0090 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -544,7 +544,6 @@ def initialize_external_data(self, externals=None): except ValueError: # pragma: no cover raise ModuleNotFoundError("No module named 'netCDF4'") - for ext in self._external_elements: if ext.py_name in ds.data_vars.keys(): # Initialize external from nc file diff --git a/tests/data/input.csv b/tests/data/input.csv index b47e142b..878f2727 100644 --- a/tests/data/input.csv +++ b/tests/data/input.csv @@ -1 +1,5 @@ -1,2,3,4,5,6,7,8,9,10, \ No newline at end of file +,1,2,3,4,5,6,7,8,9,10,, +A,0,0,1,1,-1,-1,0,0,,, +B,0,1,1,-1,-1,0,0,0,,, +C,1,1,-1,-1,0,0,0,0,,, +,,,,,,,,,,,, diff --git a/tests/data/input.tab b/tests/data/input.tab new file mode 100644 index 00000000..4b0e48c8 --- /dev/null +++ b/tests/data/input.tab @@ -0,0 +1,5 @@ + 1 2 3 4 5 6 7 8 9 10 +A 0 0 1 1 -1 -1 0 0 +B 0 1 1 -1 -1 0 0 0 +C 1 1 -1 -1 0 0 0 0 + diff --git a/tests/data/input.txt b/tests/data/input.txt new file mode 100644 index 00000000..317d2cc7 --- /dev/null +++ b/tests/data/input.txt @@ -0,0 +1,5 @@ +=1=2=3=4=5=6=7=8=9=10== +A=0=0=1=1=-1=-1=0=0=== +B=0=1=1=-1=-1=0=0=0=== +C=1=1=-1=-1=0=0=0=0=== +============ diff --git a/tests/data/input2.csv b/tests/data/input2.csv new file mode 100644 index 00000000..43448bec --- /dev/null +++ b/tests/data/input2.csv @@ -0,0 +1,5 @@ +;1;2;3;4;5;6;7;8;9;10;; +A;0;0;1;1;-1;-1;0;0;;; +B;0;1;1;-1;-1;0;0;0;;; +C;1;1;-1;-1;0;0;0;0;;; +;;;;;;;;;;;; diff --git a/tests/pytest_builders/pytest_python.py b/tests/pytest_builders/pytest_python.py index 67890d5a..f23cb86e 100644 --- a/tests/pytest_builders/pytest_python.py +++ b/tests/pytest_builders/pytest_python.py @@ -190,19 +190,32 @@ def test_referencebuilder_subscripts_nowarning(self, component, class TestSubscriptManager: @pytest.mark.parametrize( - "arguments,raise_type,error_message", + "asrs,raise_type,error_message", [ ( # invalid definition - [[AbstractSubscriptRange("my subs", 5, [])], Path("here")], + [AbstractSubscriptRange("my subs", 5, [])], ValueError, "Invalid definition of subscript 'my subs':\n\t5" ), + ( # empty range exts + [AbstractSubscriptRange("my subs", dict( + file="data/input.csv", + tab="", + firstcell="A5", + lastcell="5", + prefix="sr" + ), [])], + ValueError, + "Subscript range 'my subs' empty:\n\t" + "{'file': 'data/input.csv', 'tab': '', " + "'firstcell': 'A5', 'lastcell': '5', 'prefix': 'sr'}" + ), ], - ids=["invalid definition"] + ids=["invalid definition", "empty range exts"] ) - def test_invalid_subscripts(self, arguments, raise_type, error_message): + def test_invalid_subscripts(self, asrs, raise_type, error_message, _root): with pytest.raises(raise_type, match=error_message): - SubscriptManager(*arguments) + SubscriptManager(asrs, _root) class TestNamespaceManager: diff --git a/tests/pytest_types/external/pytest_external.py b/tests/pytest_types/external/pytest_external.py index 60ee5772..c94962f6 100644 --- a/tests/pytest_types/external/pytest_external.py +++ b/tests/pytest_types/external/pytest_external.py @@ -1754,6 +1754,189 @@ def test_constant_vn3d(self, _root, _exp): assert data().equals(_exp.constant_3d) + def test_constant_h1d_csv(self, _root, _exp): + """ + ExtConstant test for horizontal 1d data + """ + import pysd + + file_name = "data/input.csv" + sheet = "" + cell = "B2" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8]} + py_name = "test_constant_h1d_csv" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_1d) + + def test_constant_h2d_csv(self, _root, _exp): + """ + ExtConstant test for horizontal 2d data + """ + import pysd + + file_name = "data/input.csv" + sheet = "," + cell = "B2" + coords = {'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + py_name = "test_constant_h2d_csv" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d) + + def test_constant_v2d_csv(self, _root, _exp): + """ + ExtConstant test for vertical 2d data + """ + import pysd + + file_name = "data/input.csv" + sheet = "vertical" + cell = "B2*" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8], 'ABC': ['A', 'B', 'C']} + py_name = "test_constant_v2d_csv" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d.transpose()) + + def test_constant_v2d_csv2(self, _root, _exp): + """ + ExtConstant test for vertical 2d data + """ + import pysd + + file_name = "data/input2.csv" + sheet = ";" + cell = "B2*" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8], 'ABC': ['A', 'B', 'C']} + py_name = "test_constant_v2d_csv2" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d.transpose()) + + def test_constant_h1d_tab(self, _root, _exp): + """ + ExtConstant test for horizontal 1d data + """ + import pysd + + file_name = "data/input.tab" + sheet = "" + cell = "B2" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8]} + py_name = "test_constant_h1d_tab" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_1d) + + def test_constant_h2d_tab(self, _root, _exp): + """ + ExtConstant test for horizontal 2d data + """ + import pysd + + file_name = "data/input.tab" + sheet = "\t" + cell = "B2" + coords = {'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + py_name = "test_constant_h2d_tab" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d) + + def test_constant_v2d_tab(self, _root, _exp): + """ + ExtConstant test for vertical 2d data + """ + import pysd + + file_name = "data/input.tab" + sheet = "vertical" + cell = "B2*" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8], 'ABC': ['A', 'B', 'C']} + py_name = "test_constant_v2d_tab" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d.transpose()) + + def test_constant_v2d_txt(self, _root, _exp): + """ + ExtConstant test for vertical 2d data + """ + import pysd + + file_name = "data/input.txt" + sheet = "=" + cell = "B2*" + coords = {'val': [0, 1, 2, 3, 5, 6, 7, 8], 'ABC': ['A', 'B', 'C']} + py_name = "test_constant_v2d_txt" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + data.initialize() + + assert data().equals(_exp.constant_2d.transpose()) class TestSubscript(): """ @@ -1830,6 +2013,52 @@ def test_subscript_h_csv(self, _root): assert data.subscript == expected + def test_subscript_h_tab(self, _root): + """ + ExtSubscript test for horizontal subscripts + """ + import pysd + + file_name = "data/input.tab" + sheet = "" + firstcell = "B1" + lastcell = "1" + prefix = 'h' + expected = ['h1', 'h2', 'h3', 'h4', 'h5', + 'h6', 'h7', 'h8', 'h9', 'h10'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + + def test_subscript_h_txt(self, _root): + """ + ExtSubscript test for horizontal subscripts + """ + import pysd + + file_name = "data/input.txt" + sheet = "=" + firstcell = "B1" + lastcell = "1" + prefix = 'h' + expected = ['h1', 'h2', 'h3', 'h4', 'h5', + 'h6', 'h7', 'h8', 'h9', 'h10'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + def test_subscript_v(self, _root): """ ExtSubscript test for vertical subscripts @@ -1874,6 +2103,72 @@ def test_subscript_v2(self, _root): assert data.subscript == expected + def test_subscript_v_csv(self, _root): + """ + ExtSubscript test for vertical subscripts + """ + import pysd + + file_name = "data/input.csv" + sheet = "," + firstcell = "A2" + lastcell = "A" + prefix = 'v' + expected = ['vA', 'vB', 'vC'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + + def test_subscript_v_tab(self, _root): + """ + ExtSubscript test for vertical subscripts + """ + import pysd + + file_name = "data/input.tab" + sheet = "\t" + firstcell = "A1" + lastcell = "A" + prefix = 'v' + expected = ['vA', 'vB', 'vC'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + + def test_subscript_v_ncsv(self, _root): + """ + ExtSubscript test for vertical subscripts + """ + import pysd + + file_name = "data/input2.csv" + sheet = ";" + firstcell = "A1" + lastcell = "A" + prefix = 'v' + expected = ['vA', 'vB', 'vC'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + def test_subscript_d(self, _root): """ ExtSubscript test for diagonal subscripts @@ -1918,6 +2213,50 @@ def test_subscript_d2(self, _root): assert data.subscript == expected + def test_subscript_d_csv(self, _root): + """ + ExtSubscript test for diagonal subscripts + """ + import pysd + + file_name = "data/input.csv" + sheet = "123 name" + firstcell = "B1" + lastcell = "C2" + prefix = 'd' + expected = ['d1', 'd2', 'd0', 'd0'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + + def test_subscript_d_tab(self, _root): + """ + ExtSubscript test for diagonal subscripts + """ + import pysd + + file_name = "data/input.tab" + sheet = "sheet name!" + firstcell = "B1" + lastcell = "C2" + prefix = 'd' + expected = ['d1', 'd2', 'd0', 'd0'] + + data = pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + assert data.subscript == expected + def test_subscript_name_h(self, _root): """ ExtSubscript test for horizontal subscripts @@ -3335,6 +3674,70 @@ def test_subscript_name_non_existent_cellrange_name(self, _root): lastcell=lastcell, prefix=prefix) + def test_name_in_tab(self, _root): + """ + Test for cellrange input in cell for non spreadsheet file + """ + import pysd + + file_name = "data/input.csv" + sheet = "" + cell = "data" + coords = {} + py_name = "test_name_in_tab" + + data = pysd.external.ExtConstant(file_name=file_name, + tab=sheet, + root=_root, + cell=cell, + coords=coords, + final_coords=coords, + py_name=py_name) + + error_message = r"Cannot read the file '.*'...\n"\ + "It could happen that cell='data' was read as a "\ + "cell range name due to a wrong definition of cell value" + with pytest.raises(ValueError, match=error_message): + data.initialize() + + def test_subscript_name_in_csv(self, _root): + """ + Test for cellrange input in cell for non spreadsheet file + """ + import pysd + + file_name = "data/input.csv" + sheet = "" + firstcell = "fake-cell" + lastcell = "" + prefix = "" + + error_message = r"Cannot read the file '.*'...\n"\ + "It could happen that firstcell='fake-cell' was read as a "\ + "cell range name due to a wrong definition of cell value" + with pytest.raises(ValueError, match=error_message): + pysd.external.ExtSubscript(file_name=file_name, + tab=sheet, + root=_root, + firstcell=firstcell, + lastcell=lastcell, + prefix=prefix) + + + # TODO + def test_empty_subscript_range(self, _root): + """ + Test for an empty subscript range + """ + import pysd + + file_name = "data/input.csv" + sheet = "" + firstcell = "A5" + lastcell = "5" + prefix = "sr" + + class DownwardCompatibility(): """