From 49b2ae4c81ede7a384a7415928f5bcc279ac9dd5 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 18 Mar 2020 16:30:22 -0400 Subject: [PATCH 1/6] Ground work for using the yaml file when running the check_packages development tool. --- check_packages.py | 114 +++++++++++++--------------------------------- 1 file changed, 31 insertions(+), 83 deletions(-) diff --git a/check_packages.py b/check_packages.py index fff8bb9251..3d30bfa86e 100644 --- a/check_packages.py +++ b/check_packages.py @@ -4,99 +4,47 @@ from __future__ import print_function import sys +import yaml + +from colorama import Fore, Back, Style -# Fix for error: hash-collision-3-both-1-and-1/ -# See http://jaredforsyth.com/blog/2010/apr/28/accessinit-hash-collision-3-both-1-and-1/ -try: - import PIL.Image -except ImportError: - pass -else: - sys.modules['Image'] = PIL.Image -if sys.version_info[0] > 2: - print("To use the sasview GUI you must use Python 2\n") +if sys.platform == 'win32': + file_location = 'build_tools/conda/ymls/sasview-env-build_win.yml' +elif sys.platform == 'darwin': + file_location = 'build_tools/conda/ymls/sasview-env-build_osx.yml' +else: + file_location = 'build_tools/conda/ymls/sasview-env-build_linux.yml' -common_required_package_list = { - 'setuptools': {'version': '0.6c11', 'import_name': 'setuptools', 'test': '__version__'}, - 'pyparsing': {'version': '1.5.5', 'import_name': 'pyparsing', 'test': '__version__'}, - 'html5lib': {'version': '0.95', 'import_name': 'html5lib', 'test': '__version__'}, - 'reportlab': {'version': '2.5', 'import_name': 'reportlab', 'test': 'Version'}, - 'h5py': {'version': '2.5', 'import_name': 'h5py', 'test': '__version__'}, - 'lxml': {'version': '2.3', 'import_name': 'lxml.etree', 'test': 'LXML_VERSION'}, - 'PIL': {'version': '1.1.7', 'import_name': 'Image', 'test': 'VERSION'}, - 'pylint': {'version': None, 'import_name': 'pylint', 'test': None}, - 'periodictable': {'version': '1.5.0', 'import_name': 'periodictable', 'test': '__version__'}, - 'bumps': {'version': '0.7.5.9', 'import_name': 'bumps', 'test': '__version__'}, - 'numpy': {'version': '1.7.1', 'import_name': 'numpy', 'test': '__version__'}, - 'scipy': {'version': '0.18.0', 'import_name': 'scipy', 'test': '__version__'}, - 'wx': {'version': '2.8.12.1', 'import_name': 'wx', 'test': '__version__'}, - 'matplotlib': {'version': '1.1.0', 'import_name': 'matplotlib', 'test': '__version__'}, - 'xhtml2pdf': {'version': '3.0.33', 'import_name': 'xhtml2pdf', 'test': '__version__'}, - 'sphinx': {'version': '1.2.1', 'import_name': 'sphinx', 'test': '__version__'}, - 'unittest-xml-reporting': {'version': '1.10.0', 'import_name': 'xmlrunner', 'test': '__version__'}, - 'pyopencl': {'version': '2015.1', 'import_name': 'pyopencl', 'test': 'VERSION_TEXT'}, -} -win_required_package_list = { - 'comtypes': {'version': '0.6.2', 'import_name': 'comtypes', 'test': '__version__'}, - 'pywin': {'version': '217', 'import_name': 'pywin', 'test': '__version__'}, - 'py2exe': {'version': '0.6.9', 'import_name': 'py2exe', 'test': '__version__'}, -} -mac_required_package_list = { - 'py2app': {'version': None, 'import_name': 'py2app', 'test': '__version__'}, -} +with open(file_location, 'r') as stream: + try: + yaml_dict = yaml.load(stream, Loader=yaml.SafeLoader) + print(yaml_dict) + common_required_package_list = yaml_dict['dependencies'] + except Exception as e: + print(e) -deprecated_package_list = { - 'pyPdf': {'version': '1.13', 'import_name': 'pyPdf', 'test': '__version__'}, -} +print(Style.RESET_ALL) +print("CHECKING REQUIRED PACKAGES....") -print("Checking Required Package Versions....\n") -print("Common Packages") +for yaml_name in common_required_package_list: + text_split = yaml_name.split('=') + package_name = text_split[0] + package_version = text_split[1] -for package_name, test_vals in common_required_package_list.items(): + print(Style.RESET_ALL) try: - i = __import__(test_vals['import_name'], fromlist=['']) - if test_vals['test'] is None: + i = __import__(package_name, fromlist=['']) + if package_version is None: print("%s Installed (Unknown version)" % package_name) elif package_name == 'lxml': verstring = str(getattr(i, 'LXML_VERSION')) - print("%s Version Installed: %s"% (package_name, verstring.replace(', ', '.').lstrip('(').rstrip(')'))) + print("%s Version Installed: %s" % (package_name, verstring.replace(', ', '.').lstrip('(').rstrip(')'))) else: - print("%s Version Installed: %s"% (package_name, getattr(i, test_vals['test']))) + print("%s Version Installed: %s" % (package_name, getattr(i, '__version__'))) + except AttributeError: + print("%s Installed (Unknown version)" % package_name) except ImportError: - print('%s NOT INSTALLED'% package_name) - -if sys.platform == 'win32': - print("") - print("Windows Specific Packages:") - for package_name, test_vals in win_required_package_list.items(): - try: - i = __import__(test_vals['import_name'], fromlist=['']) - print("%s Version Installed: %s"% (package_name, getattr(i, test_vals['test'], "unknown"))) - except ImportError: - print('%s NOT INSTALLED'% package_name) - -if sys.platform == 'darwin': - print("") - print("MacOS Specific Packages:") - for package_name, test_vals in mac_required_package_list.items(): - try: - i = __import__(test_vals['import_name'], fromlist=['']) - print("%s Version Installed: %s"% (package_name, getattr(i, test_vals['test']))) - except ImportError: - print('%s NOT INSTALLED'% package_name) - + print(Fore.BLACK + Back.WHITE + '%s NOT INSTALLED' % package_name) -print("") -print("Deprecated Packages") -print("You can remove these unless you need them for other reasons!") -for package_name, test_vals in deprecated_package_list.items(): - try: - i = __import__(test_vals['import_name'], fromlist=['']) - if package_name == 'pyPdf': - # pyPdf doesn't have the version number internally - print('pyPDF Installed (Version unknown)') - else: - print("%s Version Installed: %s"% (package_name, getattr(i, test_vals['test']))) - except ImportError: - print('%s NOT INSTALLED'% package_name) +print(Style.RESET_ALL) From 7010b58de60e4a4d0d83b07ee8cb9b5337d3b673 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Wed, 18 Mar 2020 17:05:54 -0400 Subject: [PATCH 2/6] Code cleanup and the console output is also much clearer. Verified to work in Win64. --- check_packages.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/check_packages.py b/check_packages.py index 3d30bfa86e..e59fff8b8e 100644 --- a/check_packages.py +++ b/check_packages.py @@ -10,41 +10,48 @@ if sys.platform == 'win32': - file_location = 'build_tools/conda/ymls/sasview-env-build_win.yml' + file_location = './build_tools/conda/ymls/sasview-env-build_win.yml' elif sys.platform == 'darwin': - file_location = 'build_tools/conda/ymls/sasview-env-build_osx.yml' + file_location = './build_tools/conda/ymls/sasview-env-build_osx.yml' else: - file_location = 'build_tools/conda/ymls/sasview-env-build_linux.yml' + file_location = './build_tools/conda/ymls/sasview-env-build_linux.yml' with open(file_location, 'r') as stream: try: yaml_dict = yaml.load(stream, Loader=yaml.SafeLoader) - print(yaml_dict) common_required_package_list = yaml_dict['dependencies'] except Exception as e: print(e) -print(Style.RESET_ALL) -print("CHECKING REQUIRED PACKAGES....") +print("") +print(Style.RESET_ALL + + "....COMPARING PACKAGES LISTED IN {0} to INSTALLED PACKAGES....".format( + file_location)) +print("") for yaml_name in common_required_package_list: - text_split = yaml_name.split('=') + try: + text_split = yaml_name.split('=') + except AttributeError as e: + # Continue for tiered files including pip installs + continue package_name = text_split[0] package_version = text_split[1] - print(Style.RESET_ALL) try: - i = __import__(package_name, fromlist=['']) - if package_version is None: - print("%s Installed (Unknown version)" % package_name) - elif package_name == 'lxml': - verstring = str(getattr(i, 'LXML_VERSION')) - print("%s Version Installed: %s" % (package_name, verstring.replace(', ', '.').lstrip('(').rstrip(')'))) + if package_name == 'python': + full_version = sys.version.split() + installed_version = full_version[0] + else: + i = __import__(package_name, fromlist=['']) + installed_version = getattr(i, '__version__') + if package_version == installed_version: + print(Style.RESET_ALL + "{0} - Expected Version Installed: {1}".format(package_name, installed_version)) else: - print("%s Version Installed: %s" % (package_name, getattr(i, '__version__'))) + print(Fore.LIGHTYELLOW_EX + "{0} - Version Mismatch - Installed: {1}, Expected: {2}".format(package_name, installed_version, package_version)) except AttributeError: - print("%s Installed (Unknown version)" % package_name) + print(Fore.YELLOW + "{0} - Version Cannot Be Determined - Expected: {1}".format(package_name, package_version)) except ImportError: - print(Fore.BLACK + Back.WHITE + '%s NOT INSTALLED' % package_name) + print(Fore.LIGHTRED_EX + '{0} NOT INSTALLED'.format(package_name)) print(Style.RESET_ALL) From 3263dc13a5637880c59a89f6dc04ad222a1c98ee Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Thu, 19 Mar 2020 09:24:26 -0400 Subject: [PATCH 3/6] Use conda or pip list instead of import to limit name and version conflicts post-install, plus lint. --- check_packages.py | 57 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/check_packages.py b/check_packages.py index e59fff8b8e..9e2fe4f650 100644 --- a/check_packages.py +++ b/check_packages.py @@ -1,14 +1,24 @@ """ -Checking and reinstalling the external packages +Check the existing environment against expected values """ from __future__ import print_function import sys import yaml +import subprocess -from colorama import Fore, Back, Style +from colorama import Fore, Style +# Output strings +CORRECT = Style.RESET_ALL + "{0} - Expected Version Installed: {1}" +VERSION_MISMATCH = Fore.LIGHTYELLOW_EX +\ + "{0} - Version Mismatch - Installed: {1}, Expected: {2}" +VERSION_UNKNOWN = Fore.YELLOW +\ + "{0} - Version Cannot Be Determined - Expected: {1}" +NOT_INSTALLED = Fore.LIGHTRED_EX + '{0} NOT INSTALLED' + +# Location of yaml files if sys.platform == 'win32': file_location = './build_tools/conda/ymls/sasview-env-build_win.yml' elif sys.platform == 'darwin': @@ -16,6 +26,7 @@ else: file_location = './build_tools/conda/ymls/sasview-env-build_linux.yml' +# Open yaml and get packages with open(file_location, 'r') as stream: try: yaml_dict = yaml.load(stream, Loader=yaml.SafeLoader) @@ -23,19 +34,44 @@ except Exception as e: print(e) +# Get list of system packages (conda or pip) +isConda = True +try: + reqs = subprocess.check_output(['conda', 'list']) +except Exception: + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + isConda = False + +packages_installed = [] +versions_installed = [] + +if isConda: + for r in reqs.splitlines(): + if r.decode().split()[0] == "#" or len(r.decode().split()) < 2: + continue + package = r.decode().split()[0].lower() + version = r.decode().split()[1] + packages_installed.append(package) + versions_installed.append(version) +else: + packages_installed = [r.decode().split('==')[0].lower() for r in + reqs.split()] + versions_installed = [r.decode().split('==')[1] for r in reqs.split()] + print("") print(Style.RESET_ALL + "....COMPARING PACKAGES LISTED IN {0} to INSTALLED PACKAGES....".format( file_location)) print("") +# Step through each yaml package and check its version against system for yaml_name in common_required_package_list: try: text_split = yaml_name.split('=') except AttributeError as e: # Continue for tiered files including pip installs continue - package_name = text_split[0] + package_name = text_split[0].lower() package_version = text_split[1] try: @@ -43,15 +79,16 @@ full_version = sys.version.split() installed_version = full_version[0] else: - i = __import__(package_name, fromlist=['']) - installed_version = getattr(i, '__version__') + i = packages_installed.index(package_name) + installed_version = versions_installed[i] if package_version == installed_version: - print(Style.RESET_ALL + "{0} - Expected Version Installed: {1}".format(package_name, installed_version)) + print(CORRECT.format(package_name, installed_version)) else: - print(Fore.LIGHTYELLOW_EX + "{0} - Version Mismatch - Installed: {1}, Expected: {2}".format(package_name, installed_version, package_version)) + print(VERSION_MISMATCH.format(package_name, installed_version, + package_version)) except AttributeError: - print(Fore.YELLOW + "{0} - Version Cannot Be Determined - Expected: {1}".format(package_name, package_version)) - except ImportError: - print(Fore.LIGHTRED_EX + '{0} NOT INSTALLED'.format(package_name)) + print(VERSION_UNKNOWN.format(package_name, package_version)) + except ValueError: + print(NOT_INSTALLED.format(package_name)) print(Style.RESET_ALL) From 71297854951241bef51ea80fa6ef7d16a496fe32 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Sun, 22 Mar 2020 10:26:21 -0400 Subject: [PATCH 4/6] Give instructions on installing necessary packages if not found on system. --- check_packages.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/check_packages.py b/check_packages.py index 9e2fe4f650..1ccb9d7610 100644 --- a/check_packages.py +++ b/check_packages.py @@ -4,10 +4,16 @@ from __future__ import print_function import sys -import yaml import subprocess - -from colorama import Fore, Style +try: + import yaml + from colorama import Fore, Style +except ImportError: + print("The PyYaml and colorama packages are required for this module. To " + + "install them, please try one of the following:") + print("\tAnaconda: conda install pyyaml colorama") + print("\tBase python: pip install pyyaml colorama") + quit() # Output strings From 15507a87b9e1689a73ba6056626179da094704c3 Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 23 Mar 2020 09:58:23 -0400 Subject: [PATCH 5/6] Give explicit results at end of check_packages and better handling of items with unspecified, minimum, or maximum version numbers. --- check_packages.py | 52 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/check_packages.py b/check_packages.py index 1ccb9d7610..be6ff5f12e 100644 --- a/check_packages.py +++ b/check_packages.py @@ -17,12 +17,12 @@ # Output strings -CORRECT = Style.RESET_ALL + "{0} - Expected Version Installed: {1}" +CORRECT = Style.RESET_ALL + " - {0} - Expected Version Installed: {1}" VERSION_MISMATCH = Fore.LIGHTYELLOW_EX +\ - "{0} - Version Mismatch - Installed: {1}, Expected: {2}" + " - {0} - Version Mismatch - Installed: {1}, Expected: {2}" VERSION_UNKNOWN = Fore.YELLOW +\ - "{0} - Version Cannot Be Determined - Expected: {1}" -NOT_INSTALLED = Fore.LIGHTRED_EX + '{0} NOT INSTALLED' + " - {0} - Version Cannot Be Determined - Expected: {1}" +NOT_INSTALLED = Fore.LIGHTRED_EX + " - {0} NOT INSTALLED" # Location of yaml files if sys.platform == 'win32': @@ -36,7 +36,20 @@ with open(file_location, 'r') as stream: try: yaml_dict = yaml.load(stream, Loader=yaml.SafeLoader) - common_required_package_list = yaml_dict['dependencies'] + yaml_packages = yaml_dict['dependencies'] + common_required_package_list = [] + pip_packages = yaml_packages.pop() + inter_package_list = yaml_packages + if isinstance(pip_packages, dict): + pip_packages = pip_packages['pip'] + for pip_package in pip_packages: + inter_package_list.append(pip_package) + else: + inter_package_list = yaml_packages.append(pip_packages) + for package in inter_package_list: + package = package.replace(">=", + "=").replace("<=", "=").replace("==", "=") + common_required_package_list.append(package) except Exception as e: print(e) @@ -50,6 +63,10 @@ packages_installed = [] versions_installed = [] +packages_specified = 0 +packages_correct = 0 +packages_mismatch = 0 +packages_not_installed = 0 if isConda: for r in reqs.splitlines(): @@ -65,8 +82,7 @@ versions_installed = [r.decode().split('==')[1] for r in reqs.split()] print("") -print(Style.RESET_ALL + - "....COMPARING PACKAGES LISTED IN {0} to INSTALLED PACKAGES....".format( +print("....COMPARING PACKAGES LISTED IN {0} to INSTALLED PACKAGES....".format( file_location)) print("") @@ -74,13 +90,15 @@ for yaml_name in common_required_package_list: try: text_split = yaml_name.split('=') - except AttributeError as e: - # Continue for tiered files including pip installs - continue - package_name = text_split[0].lower() - package_version = text_split[1] + package_name = text_split[0].lower() + package_version = text_split[1] + except (AttributeError, IndexError) as e: + # No version specified + package_name = yaml_name + package_version = '0.0' try: + packages_specified += 1 if package_name == 'python': full_version = sys.version.split() installed_version = full_version[0] @@ -89,12 +107,18 @@ installed_version = versions_installed[i] if package_version == installed_version: print(CORRECT.format(package_name, installed_version)) + packages_correct += 1 else: print(VERSION_MISMATCH.format(package_name, installed_version, package_version)) - except AttributeError: - print(VERSION_UNKNOWN.format(package_name, package_version)) + packages_mismatch += 1 except ValueError: print(NOT_INSTALLED.format(package_name)) + packages_not_installed += 1 print(Style.RESET_ALL) +print("check_packages.py has finished:") +print("\tPackages required: {0}".format(packages_specified)) +print("\tInstalled correctly: {0}".format(packages_correct)) +print("\tMismatched versions: {0}".format(packages_mismatch)) +print("\tNot installed: {0}".format(packages_not_installed)) From 0efd594735b8a8c0ba2b44a92378558111845a5d Mon Sep 17 00:00:00 2001 From: Jeff Krzywon Date: Mon, 23 Mar 2020 11:26:16 -0400 Subject: [PATCH 6/6] Implement suggested shorthand for separating out pip packages --- check_packages.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/check_packages.py b/check_packages.py index be6ff5f12e..1c06a6880b 100644 --- a/check_packages.py +++ b/check_packages.py @@ -3,6 +3,7 @@ """ from __future__ import print_function +import re import sys import subprocess try: @@ -37,19 +38,10 @@ try: yaml_dict = yaml.load(stream, Loader=yaml.SafeLoader) yaml_packages = yaml_dict['dependencies'] - common_required_package_list = [] - pip_packages = yaml_packages.pop() - inter_package_list = yaml_packages - if isinstance(pip_packages, dict): - pip_packages = pip_packages['pip'] - for pip_package in pip_packages: - inter_package_list.append(pip_package) - else: - inter_package_list = yaml_packages.append(pip_packages) - for package in inter_package_list: - package = package.replace(">=", - "=").replace("<=", "=").replace("==", "=") - common_required_package_list.append(package) + inter_package_list = [p for entry in yaml_packages for p in ( + entry['pip'] if isinstance(entry, dict) else [entry])] + common_required_package_list = [re.sub("[<>=]=", "=", package) for + package in inter_package_list] except Exception as e: print(e)