From 52f063ebf8eb8b78cf18032dd27141d14247e029 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 6 Jun 2024 16:19:18 +0200 Subject: [PATCH 01/20] Move setup to pyproject --- IM/config.py | 2 +- im_service.py => IM/im_service.py | 9 +-- doc/source/manual.rst | 15 ----- docker-devel/Dockerfile | 12 +++- docker-devel/im.cfg | 1 + docker-devel/logging.conf | 1 + docker-py3/Dockerfile | 14 +++-- docker-py3/im.cfg | 1 + docker-py3/logging.conf | 1 + etc/im.cfg | 3 - pyproject.toml | 96 +++++++++++++++++++++++++++++++ setup.py | 74 ------------------------ 12 files changed, 125 insertions(+), 104 deletions(-) rename im_service.py => IM/im_service.py (99%) create mode 120000 docker-devel/im.cfg create mode 120000 docker-devel/logging.conf create mode 120000 docker-py3/im.cfg create mode 120000 docker-py3/logging.conf create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/IM/config.py b/IM/config.py index 0429ae94..3f5c9602 100644 --- a/IM/config.py +++ b/IM/config.py @@ -67,7 +67,7 @@ class Config: LOG_FILE = '/var/log/im/inf.log' LOG_FILE_MAX_SIZE = 10485760 LOG_LEVEL = "INFO" - CONTEXTUALIZATION_DIR = '/usr/share/im/contextualization' + CONTEXTUALIZATION_DIR = IM_PATH + '/../contextualization' RECIPES_DIR = CONTEXTUALIZATION_DIR + '/AnsibleRecipes' RECIPES_DB_FILE = CONTEXTUALIZATION_DIR + '/recipes_ansible.db' MAX_CONTEXTUALIZATION_TIME = 7200 diff --git a/im_service.py b/IM/im_service.py similarity index 99% rename from im_service.py rename to IM/im_service.py index 98c3ee38..0edaf452 100755 --- a/im_service.py +++ b/IM/im_service.py @@ -33,9 +33,6 @@ from IM.ServiceRequests import IMBaseRequest from IM import __version__ as version -if sys.version_info <= (2, 6): - print("Must use python 2.6 or greater") - sys.exit(1) logger = logging.getLogger('InfrastructureManager') @@ -420,7 +417,7 @@ def signal_int_handler(signal, frame): im_stop() -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser(description='IM service') parser.add_argument('--version', help='Show IM service version.', dest="version", action="store_true", default=False) @@ -436,3 +433,7 @@ def signal_int_handler(signal, frame): config_logging() launch_daemon() + + +if __name__ == "__main__": + main() diff --git a/doc/source/manual.rst b/doc/source/manual.rst index bf9a6d76..76d53d4b 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -338,21 +338,6 @@ Default Virtual Machine Options Contextualization ^^^^^^^^^^^^^^^^^ -.. confval:: CONTEXTUALIZATION_DIR - - Full path to the IM contextualization files. - The default value is :file:`/usr/share/im/contextualization`. - -.. confval:: RECIPES_DIR - - Full path to the Ansible recipes directory. - The default value is :file:`CONTEXTUALIZATION_DIR/AnsibleRecipes`. - -.. confval:: RECIPES_DB_FILE - - Full path to the Ansible recipes database file. - The default value is :file:`CONTEXTUALIZATION_DIR/recipes_ansible.db`. - .. confval:: MAX_CONTEXTUALIZATION_TIME Maximum time in seconds spent on contextualize a virtual machine before diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 69818681..76271dde 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile to create a container with the IM service -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG BRANCH=devel LABEL maintainer="Miguel Caballer " LABEL version="1.17.0" @@ -7,17 +7,23 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-distutils openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ pip3 install urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Copy im configuration files +RUN mkdir /etc/im +COPY im.cfg /etc/im/im.cfg +COPY logging.conf /etc/im/logging.conf + # Set the VM_NUM_USE_CTXT_DIST to 3 for the tests RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/im.cfg @@ -28,4 +34,4 @@ COPY ansible.cfg /etc/ansible/ansible.cfg COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json # Start IM service -CMD im_service.py \ No newline at end of file +CMD /usr/local/bin/im_service \ No newline at end of file diff --git a/docker-devel/im.cfg b/docker-devel/im.cfg new file mode 120000 index 00000000..57a77ff1 --- /dev/null +++ b/docker-devel/im.cfg @@ -0,0 +1 @@ +../etc/im.cfg \ No newline at end of file diff --git a/docker-devel/logging.conf b/docker-devel/logging.conf new file mode 120000 index 00000000..13fbecbb --- /dev/null +++ b/docker-devel/logging.conf @@ -0,0 +1 @@ +../etc/logging.conf \ No newline at end of file diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 8448cccb..53c42a2e 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,27 +1,33 @@ # Dockerfile to create a container with the IM service -FROM ubuntu:22.04 +FROM ubuntu:24.04 LABEL maintainer="Miguel Caballer " LABEL version="1.17.0" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-distutils openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ pip3 install urllib3==1.26.18 IM==1.17.0 && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Copy im configuration files +RUN mkdir /etc/im +COPY im.cfg /etc/im/im.cfg +COPY logging.conf /etc/im/logging.conf + # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg # Fix boto issue https://github.com/boto/boto/issues/3783 -COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json +COPY endpoints.json /usr/local/lib/python3.12/dist-packages/boto/endpoints.json # Start IM service -CMD im_service.py +CMD /usr/local/bin/im_service diff --git a/docker-py3/im.cfg b/docker-py3/im.cfg new file mode 120000 index 00000000..57a77ff1 --- /dev/null +++ b/docker-py3/im.cfg @@ -0,0 +1 @@ +../etc/im.cfg \ No newline at end of file diff --git a/docker-py3/logging.conf b/docker-py3/logging.conf new file mode 120000 index 00000000..13fbecbb --- /dev/null +++ b/docker-py3/logging.conf @@ -0,0 +1 @@ +../etc/logging.conf \ No newline at end of file diff --git a/etc/im.cfg b/etc/im.cfg index be1b5b90..6e1d5ff5 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -85,9 +85,6 @@ REST_PORT = 8800 REST_ADDRESS = 0.0.0.0 # Contextualization data -CONTEXTUALIZATION_DIR = /usr/share/im/contextualization -RECIPES_DIR = %(CONTEXTUALIZATION_DIR)s/AnsibleRecipes -RECIPES_DB_FILE = %(CONTEXTUALIZATION_DIR)s/recipes_ansible.db MAX_CONTEXTUALIZATION_TIME = 7200 REMOTE_CONF_DIR = /var/tmp/.im # Interval to update the state of the contextualization process in the VMs (in secs) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..d9121af5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,96 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +[build-system] +requires = [ + "setuptools~=66.1", + "wheel~=0.37.1" +] +build-backend = "setuptools.build_meta" + +[project] +name = "IM" +description = "IM is a tool to manage virtual infrastructures on Cloud deployments" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +requires-python = ">=3.6" +dependencies = [ + "ansible >=2.4", + "paramiko >= 1.14", + "PyYAML", + "suds-py3", + "cheroot", + "boto >= 2.29", + "apache-libcloud >= 3.2.0", + "RADL >= 1.3.3", + "flask", + "netaddr", + "requests >= 2.19", + "scp", + "tosca-parser", + "defusedxml", + "urllib3>=1.23", + "hvac", + "psutil", + "scar", + "requests-cache >= 1.0.0", + "packaging", + "werkzeug", + "xmltodict" +] +license = {text = "GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt"} +dynamic = ["version", "readme"] + +[project.urls] +Homepage = "https://www.grycap.upv.es/im" +Documentation = "https://imdocs.readthedocs.io" +Repository = "https://github.com/grycap/im" + +[project.optional-dependencies] +build = [ + "build==1.0.3" +] +publish = [ + "twine==4.0.2" +] +test = [ + "mock", + "coverage", +] + +[project.scripts] +im_service = "IM.im_service:main" + +[tool.setuptools] +packages = ["IM", "IM.ansible_utils", "IM.connectors", "IM.tosca", "IM.openid", "IM.tts", "contextualization"] + +[tool.setuptools.dynamic] +version = {attr = "IM.__version__"} +readme = {file = ["README.md"], content-type = "text/markdown"} + +[tool.distutils.bdist_wheel] +universal = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 2423968b..00000000 --- a/setup.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from IM import __version__ as version -from setuptools import setup -import os -import sys - -if not hasattr(sys, 'version_info') or sys.version_info < (2, 6): - raise SystemExit("IM requires Python version 2.6 or above.") - -suds_pkg = "suds" -sqlite_pkg = "pysqlite" -if sys.version_info > (3, 0): - suds_pkg = "suds-py3" - sqlite_pkg = "" - -# Avoid using wheel as it does not copy data_files to / dir -if 'bdist_wheel' in sys.argv: - raise RuntimeError("This setup.py does not support wheels") - -# Add contextualization dir files -install_path = '/usr/share/im' -datafiles = [(os.path.join(install_path, root), [os.path.join(root, f) for f in files]) - for root, dirs, files in os.walk("contextualization")] -# Add other special files -datafiles.append(('/etc/init.d', ['scripts/im'])) -datafiles.append(('/etc/systemd/system', ['scripts/im.service'])) -datafiles.append(('/etc/im', ['etc/im.cfg'])) -datafiles.append(('/etc/im', ['etc/logging.conf'])) -# force the im_service.py file to be allways in this path -datafiles.append(('/usr/bin', ['im_service.py'])) - -try: - long_desc = open('README.md').read() - long_desc_type = 'text/markdown' -except Exception as ex: - print("Error reading README: %s" % ex) - long_desc = "IM is a tool to manage virtual infrastructures on Cloud deployments" - long_desc_type = 'text/plain' - -setup(name="IM", version=version, - author='GRyCAP - Universitat Politecnica de Valencia', - author_email='micafer1@upv.es', - url='http://www.grycap.upv.es/im', - include_package_data=True, - packages=['IM', 'IM.ansible_utils', 'IM.connectors', 'IM.tosca', 'IM.openid', 'IM.tts'], - scripts=["im_service.py"], - data_files=datafiles, - license="GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt", - long_description=long_desc, - long_description_content_type=long_desc_type, - description="IM is a tool to manage virtual infrastructures on Cloud deployments", - platforms=["any"], - install_requires=["ansible >=2.4", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", - "boto >= 2.29", "apache-libcloud >= 3.2.0", "RADL >= 1.3.3", "flask", "netaddr", - "requests >= 2.19", "scp", "tosca-parser", 'defusedxml', 'urllib3>=1.23', 'hvac', - 'psutil', 'scar', 'requests-cache >= 1.0.0', 'packaging', 'werkzeug'] - ) From 3651dd94bbf92899c8490a75f408f10d285a9f2a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 6 Jun 2024 16:41:06 +0200 Subject: [PATCH 02/20] Move setup to pyproject --- docker-devel/Dockerfile | 5 +++-- docker-devel/im.cfg | 1 - docker-devel/logging.conf | 1 - docker-py3/Dockerfile | 12 ++++++++---- docker-py3/im.cfg | 1 - docker-py3/logging.conf | 1 - 6 files changed, 11 insertions(+), 10 deletions(-) delete mode 120000 docker-devel/im.cfg delete mode 120000 docker-devel/logging.conf delete mode 120000 docker-py3/im.cfg delete mode 120000 docker-py3/logging.conf diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 76271dde..0beb2e9e 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -21,8 +21,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto # Copy im configuration files RUN mkdir /etc/im -COPY im.cfg /etc/im/im.cfg -COPY logging.conf /etc/im/logging.conf +RUN wget https://raw.githubusercontent.com/grycap/im/${BRANCH}/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/${BRANCH}/etc/logging.conf -O /etc/im/logging.conf + # Set the VM_NUM_USE_CTXT_DIST to 3 for the tests RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/im.cfg diff --git a/docker-devel/im.cfg b/docker-devel/im.cfg deleted file mode 120000 index 57a77ff1..00000000 --- a/docker-devel/im.cfg +++ /dev/null @@ -1 +0,0 @@ -../etc/im.cfg \ No newline at end of file diff --git a/docker-devel/logging.conf b/docker-devel/logging.conf deleted file mode 120000 index 13fbecbb..00000000 --- a/docker-devel/logging.conf +++ /dev/null @@ -1 +0,0 @@ -../etc/logging.conf \ No newline at end of file diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 53c42a2e..006c5ddf 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,7 +1,10 @@ # Dockerfile to create a container with the IM service FROM ubuntu:24.04 + +ENV VERSION=1.17.0 + LABEL maintainer="Miguel Caballer " -LABEL version="1.17.0" +LABEL version="${VERSION}" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -14,14 +17,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install urllib3==1.26.18 IM==1.17.0 && \ + pip3 install urllib3==1.26.18 IM==${VERSION} && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Copy im configuration files RUN mkdir /etc/im -COPY im.cfg /etc/im/im.cfg -COPY logging.conf /etc/im/logging.conf +RUN mkdir /var/log/im +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.conf -O /etc/im/logging.conf # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg diff --git a/docker-py3/im.cfg b/docker-py3/im.cfg deleted file mode 120000 index 57a77ff1..00000000 --- a/docker-py3/im.cfg +++ /dev/null @@ -1 +0,0 @@ -../etc/im.cfg \ No newline at end of file diff --git a/docker-py3/logging.conf b/docker-py3/logging.conf deleted file mode 120000 index 13fbecbb..00000000 --- a/docker-py3/logging.conf +++ /dev/null @@ -1 +0,0 @@ -../etc/logging.conf \ No newline at end of file From 2493e8795e1dc820c166f62244d838a3bbf6b511 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 6 Jun 2024 16:53:52 +0200 Subject: [PATCH 03/20] Move setup to pyproject --- docker-devel/Dockerfile | 10 +++++----- docker-py3/Dockerfile | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 0beb2e9e..0ea5bed8 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -7,15 +7,15 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ - pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ - pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ + pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ + pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ + pip3 install --break-system-packages urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 006c5ddf..08a6ddf8 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -9,15 +9,15 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ - pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ - pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install urllib3==1.26.18 IM==${VERSION} && \ + pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ + pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ + pip3 install --break-system-packages urllib3==1.26.18 IM==${VERSION} && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ From 55a0c6f0c985705fc248e3ccc91709459f3fdc1f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 08:41:53 +0200 Subject: [PATCH 04/20] Move setup to pyproject --- README.md | 192 +--------------------------------------- doc/source/manual.rst | 145 +----------------------------- docker-devel/Dockerfile | 2 +- pyproject.toml | 7 +- 4 files changed, 11 insertions(+), 335 deletions(-) diff --git a/README.md b/README.md index c424ccfd..697a695d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ This software has received a gold badge according to the [Software Quality Baseline criteria](https://github.com/indigo-dc/sqa-baseline) defined by the [EOSC-Synergy](https://www.eosc-synergy.eu) project. -## 1 DOCKER IMAGE (Recommended Option) +## 1 DOCKER IMAGE The recommended option to use the Infrastructure Manager service is using the available docker image. A Docker image named `ghcr.io/grycap/im` has been @@ -106,193 +106,7 @@ helm install --namespace=im --create-namespace im grycap/IM All the information about this chart is available at the [IM chart README](https://github.com/grycap/helm-charts/blob/master/IM/README.md). -## 3 INSTALLATION - -### 3.1 REQUISITES - -IM is based on Python, so Python 2.7 or higher (Python 3.6 or higher -recommended) runtime and standard library must be installed in the system. - -If you use pip to install the IM, all the requisites will be installed. -However, if you install IM from sources you should install: - -* The RADL parser (), available in pip - as the ``RADL`` package. -* The paramiko ssh2 protocol library for python version 1.14 or later - (), typically available as the - ``python-paramiko`` package. -* The YAML library for Python, typically available as the ``python-yaml`` or - ``PyYAML`` package. -* The suds library for Python, typically available as the ``python-suds`` - package. -* The Netaddr library for Python, typically available as the ``python-netaddr`` - package. -* The Requests library for Python, typically available as the - ``python-requests`` package. -* TOSCA parser library for Python, available as the ``tosca-parser`` package in - pip. -* Ansible () to configure nodes in the - infrastructures. - In particular, Ansible 2.4+ must be installed. - To ensure the functionality the following values must be set in the - ansible.cfg file (usually found in /etc/ansible/): - -```yml -[defaults] -transport = smart -host_key_checking = False -nocolor = 1 - -become_user = root -become_method = sudo - -[paramiko_connection] - -record_host_keys=False - -[ssh_connection] - -# Only in systems with OpenSSH support to ControlPersist -ssh_args = -o ControlMaster=auto -o ControlPersist=900s -o UserKnownHostsFile=/dev/null -# In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) -#ssh_args = -o UserKnownHostsFile=/dev/null -pipelining = True -``` - -### 3.2 OPTIONAL PACKAGES - -The Flask framework () is used for the REST API. -It is typically available as the ``python3-flask`` and ``python3-werkzeug`` system -packages or ``flask`` and ``werkzeug`` pip packages. - -The CherryPy Web framework (), is needed for the REST -API. It is typically available as the ``python3-cherrypy`` system package or -``CherryPy`` pip package. In newer versions (9.0 and later) the functionality -has been moved to the ``cheroot`` library (). -It is typically available ``python3-cheroot`` system package or ``cheroot`` pip package. - -Apache-libcloud () 3.0 or later is used in the -LibCloud, OpenStack and GCE connectors. It is typically available as the -``python3-libcloud`` system package or ``apache-libcloud`` pip package. - -Boto () 2.29.0 or later is used as interface to -Amazon EC2. It is available as package named ``python3-boto`` in Debian based -distributions or ``boto`` pip package. It can also be downloaded from boto -GitHub repository (). -Download the file and copy the boto subdirectory into the IM install path. - -In case of using the a MySQL DB as the backend to store IM data. The Python -interface to MySQL must be installed, typically available as the package -``python-mysqldb`` or ``MySQL-python`` package. In case of using Python 3 use -the PyMySQL package, available as the package ``python3-pymysql`` on -debian systems or ``PyMySQL`` package in pip. - -In case of using the a MongoDB as the backend to store IM data. The Python -interface to MongoDB must be installed, typically available as the package -``python-pymongo``package in most distributions or ``pymongo`` pip package. - -In case of using the SSL secured version of the REST API pyOpenSSL -() must be installed. available as ``pyOpenSSL`` -package in pip. - -Azure python SDK () is used -to connect with the Microsoft Azure platform. The easiest way is to install all -the required packages with pip: - -```sh -pip install msrest msrestazure azure-common azure-mgmt-storage \ - azure-mgmt-compute azure-mgmt-network azure-mgmt-resource \ - azure-mgmt-dns azure-identity -``` - -The VMware vSphere API Python Bindings () -are needed by the vSphere connector. It is available as the package ``pyvmomi`` -at the pip repository. - -### 3.3 INSTALLING - -#### 3.3.1 From PIP - -First you need to install pip tool and some packages needed to compile some of -the IM requirements. To install them in Debian and Ubuntu based distributions, -do:: - -```sh -apt update -apt install -y gcc python3-dev libffi-dev libssl-dev python3-pip sshpass \ - default-libmysqlclient-dev -``` - -In Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, -Fedora, etc.), do: - -```sh -yum install -y epel-release -yum install -y which gcc python3-devel libffi-devel openssl-devel \ - python3-pip sshpass -``` - -Then you only have to call the install command of the pip tool with the IM -package: - -```sh -pip3 install IM -``` - -You can also install an specific branch of the Github repository: - -```sh -pip install git+https://github.com/grycap/im.git@master -``` - -Pip will also install the, non installed, pre-requisites needed. So Ansible 2.4 -or later will be installed in the system. Some of the optional packages are -also installed please check if some of IM features that you need requires to -install some of the packages of section OPTIONAL PACKAGES. - -You must also remember to modify the ansible.cfg file setting as specified in -the REQUISITES section. - -### 3.4 START IM ON BOOT - -In case that you want the IM service to be started at boot time, you must -execute the next set of commands: - -On Debian Systems: - -```sh -chkconfig im on -``` - -Or for newer systems like ubuntu 14.04: - -```sh -sysv-rc-conf im on -``` - -On RedHat Systems: - -```sh -update-rc.d im start 99 2 3 4 5 . stop 05 0 1 6 . -``` - -Or you can do it manually: - -```sh -ln -s /etc/init.d/im /etc/rc2.d/S99im -ln -s /etc/init.d/im /etc/rc3.d/S99im -ln -s /etc/init.d/im /etc/rc5.d/S99im -ln -s /etc/init.d/im /etc/rc1.d/K05im -ln -s /etc/init.d/im /etc/rc6.d/K05im -``` - -Adjust the installation path by setting the IMDAEMON variable at /etc/init.d/im -to the path where the IM im_service.py file is installed (e.g. -/usr/local/im/im_service.py), or set the name of the script file -(im_service.py) if the file is in the PATH (pip puts the im_service.py file in -the PATH as default). - -### 4 CONFIGURATION +### 3 CONFIGURATION Check the parameters in $IM_PATH/etc/im.cfg or /etc/im/im.cfg. See [IM Manual](https://imdocs.readthedocs.io/en/latest/manual.html#configuration) @@ -308,7 +122,7 @@ DATA_DB - must be set to the URL to access the database to store the IM data. SQLite: `sqlite:///etc/im/inf.dat` or MongoDB: `mongodb://username:password@server/db_name`, -#### 4.1 SECURITY +#### 3.1 SECURITY Security is disabled by default. Please notice that someone with local network access can "sniff" the traffic and get the messages with the IM with the diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 76d53d4b..b3caa58a 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -43,152 +43,9 @@ Then install the IM chart (with Helm v3):: All the information about this chart is available at the `IM chart README `_. -IM Service Installation -======================= - -Prerequisites -------------- - -IM needs at least Python 2.7 (Python 3.6 or higher recommended) to run, as well as the next libraries: - -* `The RADL parser `_. - (Since IM version 1.5.3, it requires RADL version 1.1.0 or later). -* `The TOSCA parser `_. - A TOSCA YAML Spec 1.0 Parser. -* `paramiko `_, ssh2 protocol library for python - (version 1.14 or later). -* `PyYAML `_, a YAML parser. -* `suds `_, a full-featured SOAP library. -* `Netaddr `_, A Python library for representing - and manipulating network addresses. -* `Requests `_, A Python library for access REST APIs. - -Also, IM uses `Ansible `_ (2.4 or later) to configure the -infrastructure nodes. - -These components are usually available from the distribution repositories. - -Finally, check the next values in the Ansible configuration file -:file:`ansible.cfg`, (usually found in :file:`/etc/ansible`):: - - [defaults] - transport = smart - host_key_checking = False - nocolor = 1 - become_user = root - become_method = sudo - - [paramiko_connection] - - record_host_keys=False - - [ssh_connection] - - # Only in systems with OpenSSH support to ControlPersist - ssh_args = -o ControlMaster=auto -o ControlPersist=900s - # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) - #ssh_args = - pipelining = True - -Optional Packages ------------------ - -* `The Flask framework `_ is used for the REST API. - It is typically available as the 'python3-flask' and 'python3-werkzeug' packages. -* `The CherryPy Web framework `_, is needed for the REST API. - It is typically available as the 'python3-cherrypy' package. In newer versions (9.0 - and later) the functionality has been moved `the cheroot - library `_. It is typically available as the - 'python3-cherrot' package. -* `apache-libcloud `_ 3.0 or later is used in the - LibCloud, OpenStack, EGI and GCE connectors. -* `boto `_ 2.29.0 or later is used as interface to - Amazon EC2. It is available as package named 'python3-boto' in Debian based - distributions. It can also be downloaded from `boto GitHub repository `_. - Download the file and copy the boto subdirectory into the IM install path. -* `pyOpenSSL `_ is needed to secure the REST API - with SSL certificates (see :confval:`REST_SSL`). - pyOpenSSL can be installed using pip. -* `The Python interface to MySQL `_, is needed to access MySQL server as IM data - backend. It is typically available as the package 'python-mysqldb' or 'MySQL-python' package. In case of - using Python 3 use the PyMySQL package, available as the package 'python3-pymysql' on debian systems or PyMySQL - package in pip. -* `The Python interface to MongoDB `_, is needed to access MongoDB server as IM data - backend. It is typically available as the package 'python-pymongo' package in most distributions or pymongo - package in pip. -* `The Azure Python SDK `_, is needed by the Azure - connector. It is available as the package 'azure' at the pip repository. -* `The VMware vSphere API Python Bindings `_ are needed by the vSphere - connector. It is available as the package 'pyvmomi' at the pip repository. - - -Installation ------------- - -From Pip -^^^^^^^^ - -First you need to install pip tool and some packages needed to compile some of the IM requirements. -To install them in Debian and Ubuntu based distributions, do:: - - $ apt update - $ apt install gcc python3-dev libffi-dev libssl-dev python3-pip sshpass python3-requests - -In Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, -Fedora, etc.), do:: - - $ yum install epel-release - $ yum install which gcc python3-devel libffi-devel openssl-devel python3-pip sshpass default-libmysqlclient-dev - -Then you only have to call the install command of the pip tool with the IM package:: - - $ pip install IM - -You can also install an specific branch of the Github repository:: - - $ pip install git+https://github.com/grycap/im.git@master - -Pip will also install the, non installed, pre-requisites needed. So Ansible 2.4 or later will -be installed in the system. Some of the optional packages are also installed please check if some -of IM features that you need requires to install some of the packages of section "Optional Packages". - -You must also remember to modify the ansible.cfg file setting as specified in the -"Prerequisites" section. - Configuration -------------- - -If you want the IM Service to be started at boot time, do - -1. Update the value of the variable ``IMDAEMON`` in :file:`/etc/init.d/im` file - to the path where the IM im_service.py file is installed (e.g. /usr/local/im/im_service.py), - or set the name of the script file (im_service.py) if the file is in the PATH - (pip puts the im_service.py file in the PATH as default):: - - $ sudo sed -i 's/`IMDAEMON=.*/`IMDAEMON=/usr/local/IM-0.1/im_service.py'/etc/init.d/im - -2. Register the service. - -To do the last step on a Debian based distributions, execute:: - - $ sudo sysv-rc-conf im on - -if the package 'sysv-rc-conf' is not available in your distribution, execute:: - - $ sudo update-rc.d im start 99 2 3 4 5 . stop 05 0 1 6 . - -For Red Hat based distributions:: - - $ sudo chkconfig im on - -Alternatively, it can be done manually:: - - $ ln -s /etc/init.d/im /etc/rc2.d/S99im - $ ln -s /etc/init.d/im /etc/rc3.d/S99im - $ ln -s /etc/init.d/im /etc/rc5.d/S99im - $ ln -s /etc/init.d/im /etc/rc1.d/K05im - $ ln -s /etc/init.d/im /etc/rc6.d/K05im +============= IM reads the configuration from :file:`$IM_PATH/etc/im.cfg`, and if it is not available, does from ``/etc/im/im.cfg``. There is a template of :file:`im.cfg` diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 0ea5bed8..1dc8cce9 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -32,7 +32,7 @@ RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/i COPY ansible.cfg /etc/ansible/ansible.cfg # Fix boto issue https://github.com/boto/boto/issues/3783 -COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json +COPY endpoints.json /usr/local/lib/python3.12/dist-packages/boto/endpoints.json # Start IM service CMD /usr/local/bin/im_service \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d9121af5..2f0753b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,13 @@ classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Operating System :: OS Independent", + "Framework :: CherryPy", + "Framework :: Flask", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -42,7 +47,7 @@ dependencies = [ "ansible >=2.4", "paramiko >= 1.14", "PyYAML", - "suds-py3", + "suds-community", "cheroot", "boto >= 2.29", "apache-libcloud >= 3.2.0", From 293d4b768d8de6359394eca2192825e982fded89 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 08:50:20 +0200 Subject: [PATCH 05/20] Increase timeout --- test/integration/TestIM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index da9ef55a..3217f25e 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -272,7 +272,7 @@ def test_19_addresource(self): str(len(vm_ids)) + "). It must be 4")) all_configured = self.wait_inf_state( - self.inf_id, VirtualMachine.CONFIGURED, 2100) + self.inf_id, VirtualMachine.CONFIGURED, 2700) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") From 1cb97a16f41f17f3ebb58c278da652ad45ee4d36 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 09:03:08 +0200 Subject: [PATCH 06/20] Changes to use py 3.12 --- .github/workflows/main.yaml | 2 +- docker-devel/Dockerfile | 2 +- docker-py3/Dockerfile | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8f58a84c..b0b202d4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3. uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: python -m pip install tox diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 1dc8cce9..40b87161 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y wget python3 op # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ pip3 install --break-system-packages urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 08a6ddf8..e530a747 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -15,17 +15,17 @@ RUN apt-get update && apt-get install --no-install-recommends -y wget python3 op # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ + pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install --break-system-packages urllib3==1.26.18 IM==${VERSION} && \ + pip3 install --break-system-packages urllib3==1.26.18 git+https://github.com/grycap/im@wheel && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Copy im configuration files RUN mkdir /etc/im RUN mkdir /var/log/im -RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/im.cfg -O /etc/im/im.cfg -RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.conf -O /etc/im/logging.conf +RUN wget https://raw.githubusercontent.com/grycap/im/wheel/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/wheel/etc/logging.conf -O /etc/im/logging.conf # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg From f51706e92d2ebd90fe402b966f8a9eb41b55d737 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 09:07:47 +0200 Subject: [PATCH 07/20] Remove urllib3 pin version --- docker-devel/Dockerfile | 2 +- docker-py3/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 40b87161..6a274d29 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install --break-system-packages urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ + pip3 install --break-system-packages git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index e530a747..83224e98 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install --break-system-packages urllib3==1.26.18 git+https://github.com/grycap/im@wheel && \ + pip3 install --break-system-packages git+https://github.com/grycap/im@wheel && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ From 819c89ade9887514221dd9224896b13ab02c9946 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 10:05:13 +0200 Subject: [PATCH 08/20] Apply patch to add ost net extra info --- docker-devel/Dockerfile | 7 +++++++ docker-devel/ost.patch | 14 ++++++++++++++ docker-py3/Dockerfile | 10 ++++++++-- docker-py3/ost.patch | 14 ++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 docker-devel/ost.patch create mode 100644 docker-py3/ost.patch diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 6a274d29..d3a3c399 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -12,6 +12,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y wget python3 op # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ + pip3 install --break-system-packages -U pip && \ pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ @@ -19,6 +20,12 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Patch libcloud to add network extra +# untill this PR is merged and released +# https://github.com/apache/libcloud/pull/2016 +COPY ost.patch /tmp/ost.patch +RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch + # Copy im configuration files RUN mkdir /etc/im RUN wget https://raw.githubusercontent.com/grycap/im/${BRANCH}/etc/im.cfg -O /etc/im/im.cfg diff --git a/docker-devel/ost.patch b/docker-devel/ost.patch new file mode 100644 index 00000000..ba39ef96 --- /dev/null +++ b/docker-devel/ost.patch @@ -0,0 +1,14 @@ +@@ -3158,6 +3158,12 @@ + extra["router:external"] = obj.get("router:external") + if obj.get("subnets", None): + extra["subnets"] = obj.get("subnets") ++ if obj.get("tags", None): ++ extra["tags"] = obj.get("tags") ++ if obj.get("is_default", None) is not None: ++ extra["is_default"] = obj.get("is_default") ++ if obj.get("description", None): ++ extra["description"] = obj.get("description") + return OpenStackNetwork(id=obj["id"], name=obj["name"], cidr=None, driver=self, extra=extra) + + def ex_list_networks(self): + diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 83224e98..ee0cd581 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -9,18 +9,24 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y patch wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ + pip3 install --break-system-packages -U pip && \ pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ - pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ pip3 install --break-system-packages git+https://github.com/grycap/im@wheel && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Patch libcloud to add network extra +# untill this PR is merged and released +# https://github.com/apache/libcloud/pull/2016 +COPY ost.patch /tmp/ost.patch +RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch + # Copy im configuration files RUN mkdir /etc/im RUN mkdir /var/log/im diff --git a/docker-py3/ost.patch b/docker-py3/ost.patch new file mode 100644 index 00000000..ba39ef96 --- /dev/null +++ b/docker-py3/ost.patch @@ -0,0 +1,14 @@ +@@ -3158,6 +3158,12 @@ + extra["router:external"] = obj.get("router:external") + if obj.get("subnets", None): + extra["subnets"] = obj.get("subnets") ++ if obj.get("tags", None): ++ extra["tags"] = obj.get("tags") ++ if obj.get("is_default", None) is not None: ++ extra["is_default"] = obj.get("is_default") ++ if obj.get("description", None): ++ extra["description"] = obj.get("description") + return OpenStackNetwork(id=obj["id"], name=obj["name"], cidr=None, driver=self, extra=extra) + + def ex_list_networks(self): + From 7da93367c4a48443d34f08183515ef544f3e292f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 10:21:04 +0200 Subject: [PATCH 09/20] Go back to ubuntu 22 --- .github/workflows/main.yaml | 2 +- docker-devel/Dockerfile | 12 ++++++------ docker-py3/Dockerfile | 10 +++++----- pyproject.toml | 3 +-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b0b202d4..dfb7eade 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3. uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.11' - name: Install dependencies run: python -m pip install tox diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index d3a3c399..cb648ce3 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile to create a container with the IM service -FROM ubuntu:24.04 +FROM ubuntu:22.04 ARG BRANCH=devel LABEL maintainer="Miguel Caballer " LABEL version="1.17.0" @@ -12,11 +12,11 @@ RUN apt-get update && apt-get install --no-install-recommends -y wget python3 op # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ - pip3 install --break-system-packages -U pip && \ - pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ - pip3 install --break-system-packages git+https://github.com/micafer/libcloud@ost_nets_extra && \ - pip3 install --break-system-packages git+https://github.com/grycap/im@$BRANCH && \ + pip3 install -U pip && \ + pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ + pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ + pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ + pip3 install apache-libcloud==3.8.0 git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index ee0cd581..5b1a6721 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile to create a container with the IM service -FROM ubuntu:24.04 +FROM ubuntu:22.04 ENV VERSION=1.17.0 @@ -14,10 +14,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y patch wget pyth # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ - pip3 install --break-system-packages -U pip && \ - pip3 install --break-system-packages msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install --break-system-packages pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ - pip3 install --break-system-packages git+https://github.com/grycap/im@wheel && \ + pip3 install -U pip && \ + pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ + pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ + pip3 install apache-libcloud==3.8.0 git+https://github.com/grycap/im@wheel && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/pyproject.toml b/pyproject.toml index 2f0753b4..b83fbcfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.11" ] requires-python = ">=3.6" dependencies = [ From 8e31408d38692a105c77f565a6a565a11f5d8848 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 10:29:41 +0200 Subject: [PATCH 10/20] Fix test changes --- docker-py3/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 5b1a6721..45d2be54 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto pip3 install -U pip && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ - pip3 install apache-libcloud==3.8.0 git+https://github.com/grycap/im@wheel && \ + pip3 install apache-libcloud==3.8.0 IM==${VERSION} && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ @@ -25,13 +25,13 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto # untill this PR is merged and released # https://github.com/apache/libcloud/pull/2016 COPY ost.patch /tmp/ost.patch -RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch +RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch # Copy im configuration files RUN mkdir /etc/im RUN mkdir /var/log/im -RUN wget https://raw.githubusercontent.com/grycap/im/wheel/etc/im.cfg -O /etc/im/im.cfg -RUN wget https://raw.githubusercontent.com/grycap/im/wheel/etc/logging.conf -O /etc/im/logging.conf +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.conf -O /etc/im/logging.conf # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg From ef235d80c3b4b6714036095c2b375d2bd412728a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 10:31:55 +0200 Subject: [PATCH 11/20] Fix test changes --- docker-devel/Dockerfile | 4 ++-- docker-py3/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index cb648ce3..c1825fee 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3-setupto # untill this PR is merged and released # https://github.com/apache/libcloud/pull/2016 COPY ost.patch /tmp/ost.patch -RUN patch /usr/local/lib/python3.12/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch +RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch # Copy im configuration files RUN mkdir /etc/im @@ -39,7 +39,7 @@ RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/i COPY ansible.cfg /etc/ansible/ansible.cfg # Fix boto issue https://github.com/boto/boto/issues/3783 -COPY endpoints.json /usr/local/lib/python3.12/dist-packages/boto/endpoints.json +COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json # Start IM service CMD /usr/local/bin/im_service \ No newline at end of file diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 45d2be54..e3d7f400 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -37,7 +37,7 @@ RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.con COPY ansible.cfg /etc/ansible/ansible.cfg # Fix boto issue https://github.com/boto/boto/issues/3783 -COPY endpoints.json /usr/local/lib/python3.12/dist-packages/boto/endpoints.json +COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json # Start IM service CMD /usr/local/bin/im_service From 22e2bbd4937dd6dbc65b8d622e0df03dc09d0cde Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 10:58:06 +0200 Subject: [PATCH 12/20] Add patch command --- docker-devel/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index c1825fee..101605f8 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -7,7 +7,7 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y patch wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM From bc7d5712fe311ee34a8a54cf6035c8929568bd2e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 12:04:24 +0200 Subject: [PATCH 13/20] Add REST index page --- IM/REST.py | 10 + IM/swagger_api.yaml | 1605 ++++++++++++++++++++++++++++++++++ test/integration/TestREST.py | 7 + test/unit/REST.py | 4 + 4 files changed, 1626 insertions(+) create mode 100644 IM/swagger_api.yaml diff --git a/IM/REST.py b/IM/REST.py index f04f8853..6b10191d 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -19,6 +19,8 @@ import json import base64 import flask +import os +import yaml from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher from cheroot.ssl.builtin import BuiltinSSLAdapter @@ -939,6 +941,14 @@ def RESTGetVersion(): return return_error(400, "Error getting IM version: %s" % get_ex_error(ex)) +@app.route('/') +def RESTIndex(): + rest_path = os.path.dirname(os.path.abspath(__file__)) + abs_file_path = os.path.join(rest_path, 'swagger_api.yaml') + api_docs = yaml.safe_load(open(abs_file_path, 'r')) + return flask.make_response(json.dumps(api_docs), 200, {'Content-Type': 'application/json'}) + + @app.route('/infrastructures//vms//disks//snapshot', methods=['PUT']) def RESTCreateDiskSnapshot(infid=None, vmid=None, disknum=None): try: diff --git a/IM/swagger_api.yaml b/IM/swagger_api.yaml new file mode 100644 index 00000000..114b04d6 --- /dev/null +++ b/IM/swagger_api.yaml @@ -0,0 +1,1605 @@ +openapi: 3.0.0 + +info: + description: Infrastructure Manager (IM) REST API. + version: 1.17.0 + title: Infrastructure Manager (IM) REST API + contact: + email: products@grycap.upv.es + license: + name: GPL 3.0 + url: 'https://www.gnu.org/licenses/gpl-3.0.en.html' + +tags: + - name: infrastructures + description: Manages Virtual Infrastructures. + - name: version + description: Get IM server version. + - name: clouds + description: Get cloud information. + +paths: + + /version: + get: + tags: + - version + summary: Get IM server version. + description: Get IM server version. + operationId: GetVersion + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: 1.10.0 + '400': + description: Invalid status value + + /infrastructures: + get: + tags: + - infrastructures + summary: List user infrastructures. + security: + - IMAuth: [] + description: >- + Return a list of URIs referencing the infrastructures associated to the + IM user. + operationId: GetInfrastructures + parameters: + - name: filter + in: query + description: >- + The filter parameter is optional and it is a regular expression + (python format) to search in the RADL or TOSCA used to create the + infrastructure. If not specified all the user infrastructures will + be returned. + required: false + schema: + type: string + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id1 + http://server.com:8800/infrastructures/inf_id2 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id1"}, + {"uri": "http://server.com:8800/infrastructures/inf_id2"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + post: + tags: + - infrastructures + summary: Creates an infrastructures. + security: + - IMAuth: [] + description: >- + Create and configure an infrastructure with the requirements specified + in RADL or TOSCA document. + operationId: CreateInfrastructure + parameters: + - name: async + in: query + description: >- + The async parameter is optional and is a flag to specify if the call + will not wait for the VMs to be created. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + - name: dry_run + in: query + description: >- + parameter is optional and is a flag to specify if the call will not create the VMs + and will only return the ammount of resources needed to deploy the infrastructure. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + requestBody: + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 2G and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + deploy node 1 + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 2147483648 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + - class: deploy + system: wn + vm_number: 1 + text/yaml: + schema: + type: string + example: + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + web_server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: linux + distribution: CentOS + version: 8 + description: RADL (in plain RADL or in JSON formats) or TOSCA document (YAML). + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: 'http://server.com:8800/infrastructures/inf_id1' + application/json: + examples: + response: + value: | + { + "uri": "http://server.com:8800/infrastructures/inf_id1" + } + response_dry_run: + value: | + { + "compute": [ + {"cpu": 2, "memory": 4096, "disk": 20}, + {"cpu": 1, "memory": 2048, "disk": 10} + ], + "storage": [ + {"size": 100} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + put: + tags: + - infrastructures + summary: Import an infrastructure. + security: + - IMAuth: [] + description: >- + Take control of the infrastructure serialized in in the body and return + the ID associated in the server. (See GET + /infrastructures/{infId}/data). + operationId: Importfrastructure + requestBody: + content: + application/json: + schema: + type: string + description: >- + JSON data of the infrastructure obtained with GET + /infrastructures/{infId}/data. + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: 'http://server.com:8800/infrastructures/inf_id1' + application/json: + examples: + response: + value: | + { + "uri": "http://server.com:8800/infrastructures/inf_id1" + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '415': + description: Unsupported Media type + + /infrastructures/{InfId}: + get: + tags: + - infrastructures + summary: List of VMs in a infrastructure. + security: + - IMAuth: [] + description: >- + Return a list of URIs referencing the virtual machines associated to the + infrastructure with ID InfId + operationId: GetVMList + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id/vms/0 + http://server.com:8800/infrastructures/inf_id/vms/1 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/0"}, + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/1"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + post: + tags: + - infrastructures + summary: Add resources to an infrastructures. + security: + - IMAuth: [] + description: > + Add the resources specified in the body contents (in TOSCA, RADL plain + or in JSON formats) to the infrastructure with ID infId. + + In case of using RADL as input the deploy instructions in the radl must + refer to systems already defined. If all the systems defined in radl are + new, they will be added. Otherwise the new systems defined will be + ignored. All the systems specified in the deploy must be specified in + the radl. If they has been already defined only a reference is needed. + This is a simple example to deploy one new VM from an alreay defined + system: + + Using TOSCA as input this method can be used to add or remove resources + depending on the number of resources specified in the new TOSCA document + sent. If new nodes are added in the body compared with the last TOSCA + sent to the IM, these new nodes will be added. For example an + infrastructure has been created with this TOSCA document: + [tosca_create.yml](https://github.com/grycap/im/blob/master/test/files/tosca_create.yml) + it launches one DB server and one Web server. If this TOSCA document is + sent as body of this POST function: + [tosca_add.yml](https://github.com/grycap/im/blob/master/test/files/tosca_add.yml), + a new web server will be added as the number of web servers has been + increased to two (count parameter of scalable capability). However if + this document is sent after the node addition (the number of web servers + will be two): + [tosca_remove.yml](https://github.com/grycap/im/blob/master/test/files/tosca_remove.yml) + , a web server (the VM with the ID 2 as specified in the removal_list + parameter) will be removed. + operationId: AddResources + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: context + in: query + description: >- + The context parameter is optional and is a flag to specify if the + contextualization step will be launched just after the VM addition. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'true' + requestBody: + content: + text/plain: + schema: + type: string + example: | + network net + system node + deploy node 1 + application/json: + schema: + type: string + example: + - class: network + id: net + reference: true + - class: system + id: node + reference: true + - class: deploy + system: wn + vm_number: 1 + text/yaml: + schema: + type: string + example: + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + web_server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: linux + distribution: CentOS + version: 8 + description: RADL (in plain RADL or in JSON formats) or TOSCA document (YAML). + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id/vms/0 + http://server.com:8800/infrastructures/inf_id/vms/1 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/0"}, + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/1"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + delete: + tags: + - infrastructures + summary: Delete an infrastructure. + security: + - IMAuth: [] + description: Undeploy the virtual machines associated to the infrastructure with ID. + operationId: DestroyInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: force + in: query + description: >- + The force parameter is optional and is a flag to specify that the + infra will be from the IM although not all resources are deleted. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + - name: async + in: query + description: >- + The async parameter is optional and is a flag to specify if the call + will not wait the infrastructure to be deleted. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/state: + get: + tags: + - infrastructures + summary: Get infrastructure state. + security: + - IMAuth: [] + description: Return a the state of the infrastructure with ID InfId. + operationId: GetInfrastructureState + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/InfrastructureState' + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/outputs: + get: + tags: + - infrastructures + summary: Get the infrastructure outputs. + security: + - IMAuth: [] + description: >- + In case of TOSCA documents it will return a JSON object with the outputs + of the TOSCA document. + operationId: GetInfrastructureOutpus + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: | + { + "output_name1": "output_value1", + "output_name2": "output_value2", + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/contmsg: + get: + tags: + - infrastructures + summary: Get a infrastructure contextualization message. + security: + - IMAuth: [] + description: >- + Return a string with the contextualization message. In case of + headeronly flag is set to ‘yes’, ‘true’ or ‘1’ only the initial part of + the infrastructure contextualization log will be returned (without any + VM contextualization log). + operationId: GetInfrastructureContmsg + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: headeronly + in: query + description: >- + In case of headeronly flag is set to `yes`, `true` or `1` only the + initial part of the infrastructure contextualization log will be + returned (without any VM contextualization log). + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: > + 2019-12-11 11:08:14.574891: Select master VM + + 2019-12-11 11:08:14.576685: Wait master VM to boot + + 2019-12-11 11:08:14.577905: Wait master VM to have the SSH + active. + application/json: + examples: + response: + value: + contmsg: | + 2019-12-11 11:08:14.574891: Select master VM + 2019-12-11 11:08:14.576685: Wait master VM to boot + 2019-12-11 11:08:14.577905: Wait master VM to have the SSH active. + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/data: + get: + tags: + - infrastructures + summary: Export an infrastructure. + security: + - IMAuth: [] + description: >- + Return the serialization of the infrastructure with ID InfId. This + function is useful to transfer the control of an infrastructure to other + IM server (See PUT /infrastructures). In case of delete flag is set to + ‘yes’, ‘true’ or ‘1’ the data not only will be exported but also the + infrastructure will be set deleted (the virtual infrastructure will not + be modified). + operationId: GetInfrastructureData + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: delete + in: query + description: >- + In case of delete flag is set to ‘yes’, ‘true’ or ‘1’ the data not + only will be exported but also the infrastructure will be set + deleted (the virtual infrastructure will not be modified). + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + vm_list: [] + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/radl: + get: + tags: + - infrastructures + summary: Get the infrastructure RADL used to create it. + security: + - IMAuth: [] + description: >- + Return a string with the original specified RADL of the infrastructure + with ID InfId. + operationId: GetInfrastructureRADL + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + network net (outbound = 'yes') + system node ( + ... + ) + application/json: + examples: + response: + value: + radl: "network net (outbound = 'yes')\nsystem node ( ... )" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/tosca: + get: + tags: + - infrastructures + summary: Get the TOSCA representation of the infrastructure. + security: + - IMAuth: [] + description: >- + Return a string with the TOSCA representation of the infrastructure with + ID InfId. + operationId: GetInfrastructureTOSCA + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + tosca: | + tosca_definitions_version: tosca_simple_yaml_1_0 + description: Some TOSCA template + ... + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/authorization: + get: + tags: + - infrastructures + summary: Get the list of infrastructure owners. + security: + - IMAuth: [] + description: >- + Return a list of strings with the set of users of the infrastructure + with ID InfId. + operationId: GetInfrastructureOwners + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + user1 + user2 + application/json: + examples: + response: + value: + owners: + - user1 + - user2 + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + post: + tags: + - infrastructures + summary: Change the authorization data of the infrastructure. + security: + - IMAuth: [] + description: >- + Change the authorization data of the infrastructure with ID InfId. + The body of the request contains the new authorization data in json + format. It can be a pair username-password or a token. In case of using + a token it must be a valid access token. + operationId: ChangeInfrastructureAuth + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: overwrite + in: query + description: >- + The overwrite parameter is optional and is a flag to specify if the + authorization data will be overwrited or will be appended. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + requestBody: + content: + application/json: + schema: + oneOf: + - type: object + properties: + username: + type: string + example: new_user + password: + type: string + example: new_password + - type: object + properties: + token: + type: string + example: acces_token_new_user + + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/stop: + put: + tags: + - infrastructures + summary: Stop an infrastructure. + security: + - IMAuth: [] + description: >- + Perform the stop operation in all the virtual machines in the + infrastructure. + operationId: StopInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/start: + put: + tags: + - infrastructures + summary: Start an infrastructure. + security: + - IMAuth: [] + description: >- + Perform the start operation in all the (previuosly stopped) virtual + machines in the infrastructure. + operationId: StartInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/reconfigure: + put: + tags: + - infrastructures + summary: Reconfigure an infrastructure. + security: + - IMAuth: [] + description: Re-start the contextualization process of the infrastructure. + operationId: ReconfigureInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: vm_list + in: query + description: >- + The vm_list parameter is optional and is a coma separated list of + IDs of the VMs to reconfigure. If not specified all the VMs will be + reconfigured + required: false + schema: + type: string + requestBody: + content: + text/plain: + schema: + type: string + example: | + configure node ( + @begin + --- + - tasks: + - debug: msg="Some message" + @end + ) + application/json: + schema: + type: string + example: + - class: configure + id: node + recipes: | + - tasks: + - debug: msg="Some message" + description: >- + Optional RADL (in plain RADL or in JSON formats) with new + configuration recipes. If not specified the current configuration + recipes will be executed again. + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + + /infrastructures/{InfId}/vms/{VMId}: + get: + tags: + - infrastructures + summary: Get VM info. + security: + - IMAuth: [] + description: >- + Return information about the virtual machine with ID VMId associated to + the infrastructure with ID InfId + operationId: GetVMInfo + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 2G and + net_interface.0.ip = '8.8.8.8' and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 2147483648 + net_interface.0.ip: 8.8.8.8 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + put: + tags: + - infrastructures + summary: Alter VM. + security: + - IMAuth: [] + description: >- + Change the features of the virtual machine with ID VMId in the + infrastructure with with ID infId. + operationId: AlterVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + requestBody: + content: + text/plain: + schema: + type: string + example: | + system node ( + memory.size >= 4G + ) + application/json: + schema: + type: string + example: + - class: system + id: node + memory.size_min: 4294967296 + description: RADL (in plain RADL or in JSON formats). + required: true + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 4G and + net_interface.0.ip = '8.8.8.8' and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 4294967296 + net_interface.0.ip: 8.8.8.8 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + delete: + tags: + - infrastructures + summary: Delete a VM. + security: + - IMAuth: [] + description: >- + Undeploy the virtual machine VMId associated to the infrastructure with + ID infId + operationId: DestroyVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/contmsg: + get: + tags: + - infrastructures + summary: Get a VM contextualization message. + security: + - IMAuth: [] + description: >- + Return a string with the contextualization message associated to the + virtual machine with ID VMId in the infrastructure with with ID infId + operationId: GetVMContMsg + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + Launch task: basic + ... + application/json: + examples: + response: + value: + contmsg: | + Launch task: basic + ... + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/{Property}: + get: + tags: + - infrastructures + summary: Get a VM property. + security: + - IMAuth: [] + description: >- + Return a property associated to the virtual machine with ID VMId in the + infrastructure with with ID infId + operationId: GetVMProperty + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + - name: Property + in: path + description: The specific VM RADL property to get. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: property_value + application/json: + examples: + response: + value: + property_name: property_value + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/stop: + put: + tags: + - infrastructures + summary: Stop a VM. + security: + - IMAuth: [] + description: >- + Perform the stop operation in the virtual machine VMId of the + infrastructure InfId. The default behavior is suspending the VM if this + operation is not available the VM will be stopped. + operationId: StopVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/start: + put: + tags: + - infrastructures + summary: Start a VM. + security: + - IMAuth: [] + description: >- + Perform the start operation in the (previuosly stopped) virtual machine + VMId of the infrastructure InfId. + operationId: StartVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/reboot: + put: + tags: + - infrastructures + summary: Reboot a VM. + security: + - IMAuth: [] + description: >- + Perform the reboot operation in the virtual machine VMId of the + infrastructure InfId. + operationId: RebootVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/disks/{diskNum}/snapshot: + put: + tags: + - infrastructures + summary: Create a disk snapshot. + security: + - IMAuth: [] + description: >- + Create a snapshot of the specified diskNum in the VM VMId of the + infrastructure with ID InfId and returns the image url of the new + created image in IM format. + operationId: CreateSnapShot + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + - name: diskNum + in: path + description: The number of the specific disk. + required: true + schema: + type: string + - name: image_name + in: query + description: The name to assing to the created snapshot. + required: true + schema: + type: string + - name: auto_delete + in: query + description: >- + The auto_delete flag specifies that the snapshot will be deleted + when the infrastructure is destroyed + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: 'one://server.com/image_id' + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /clouds/{CloudId}/images: + get: + tags: + - clouds + summary: Get the list of available images in the specified cloud provider. + security: + - IMAuth: [] + description: > + Return a list of URIs referencing the available images in the specified + cloud provider. The id cloudId is relative to the id field in the + AUTHORIZATION header. + operationId: GetCloudImages + parameters: + - name: CloudId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: filters + in: query + description: >- + The optional filters parameter enables filterin the list of images. + It is a comma separated list of keypair values + ('key1=val1,key2=value2'). + + This field is cloud provider specific (e.g. 'region=region_name' for + Amazon EC2, GCE or Azure). + required: false + schema: + type: string + default: '' + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + images: + - uri: ost://hostname/image-id1 + name: Image Name1 + - uri: ost://hostname/image-id2 + name: Image Name2 + '400': + description: Invalid status value + '401': + description: Unauthorized + + /clouds/{CloudId}/quotas: + get: + tags: + - clouds + summary: >- + Get available and used resources for the current user in the specified + cloud provider. + security: + - IMAuth: [] + description: > + Return a dictionary with available and used resources for the current + user in the specified cloud provider. The id cloudId is relative to the + id field in the AUTHORIZATION header. + operationId: GetCloudQuotas + parameters: + - name: CloudId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + quotas: + cores: + used: 1 + limit: 10 + ram: + used: 1 + limit: 10 + instances: + used: 1 + limit: 10 + floating_ips: + used: 1 + limit: 10 + security_groups: + used: 1 + limit: 10 + '400': + description: Invalid status value + '401': + description: Unauthorized + +externalDocs: + description: Find out more about IM + url: 'http://www.grycap.upv.es/im' +servers: + - url: 'https://appsgrycap.i3m.upv.es:31443/im' +components: + schemas: + State: + type: string + enum: + - pending + - running + - configured + - unconfigured + - stopped + - 'off' + - failed + - unknown + - deleting + title: State + example: running + InfrastructureState: + type: object + properties: + state: + $ref: '#/components/schemas/State' + vm_states: + type: array + items: + $ref: '#/components/schemas/State' + example: + - running + - running + title: InfrastructureState + + securitySchemes: + IMAuth: + type: apiKey + in: header + name: Authorization + description: >- + The Authentication header must provide the content of the + [Authorization + File](https://imdocs.readthedocs.io/en/latest/client.html#auth-file), + but putting all the elements in one line using “\n” as separator. diff --git a/test/integration/TestREST.py b/test/integration/TestREST.py index c7dd8c42..65bb4fec 100755 --- a/test/integration/TestREST.py +++ b/test/integration/TestREST.py @@ -139,6 +139,13 @@ def test_05_version(self): self.assertEqual( resp.text, version, msg="Incorrect version. Expected %s, obtained: %s" % (version, resp.text)) + def test_07_index(self): + resp = self.create_request("GET", "/") + self.assertEqual(resp.status_code, 200, + msg="ERROR getting IM index:" + resp.text) + res = json.loads(resp.text) + self.assertEqual(res['openapi'], '3.0.0') + def test_10_list(self): resp = self.create_request("GET", "/infrastructures") self.assertEqual(resp.status_code, 200, diff --git a/test/unit/REST.py b/test/unit/REST.py index 48f7a168..2c80b548 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -568,6 +568,10 @@ def test_GeVersion(self): res = self.client.get('/version') self.assertEqual(res.text, version) + def test_Index(self): + res = self.client.get('/') + self.assertEqual(res.json['openapi'], '3.0.0') + @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot): """Test REST StopVM.""" From 8f9df5f3aa832f8b19c193a6ac1a4738f38d60f0 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 12:36:45 +0200 Subject: [PATCH 14/20] Add REST index page --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b83fbcfe..fd6487a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,5 +96,8 @@ packages = ["IM", "IM.ansible_utils", "IM.connectors", "IM.tosca", "IM.openid", version = {attr = "IM.__version__"} readme = {file = ["README.md"], content-type = "text/markdown"} +[tool.setuptools.package-data] +IM = ["*.yaml"] + [tool.distutils.bdist_wheel] universal = true From 42dac021abb60a8f39090df90485827054ffe6d1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 12:38:29 +0200 Subject: [PATCH 15/20] Add REST index page --- test/integration/TestREST_JSON.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/integration/TestREST_JSON.py b/test/integration/TestREST_JSON.py index 30b26124..032503e6 100755 --- a/test/integration/TestREST_JSON.py +++ b/test/integration/TestREST_JSON.py @@ -131,6 +131,13 @@ def wait_inf_state(self, state, timeout, incorrect_states=None, vm_ids=None): return all_ok + def test_10_index(self): + resp = self.create_request("GET", "/") + self.assertEqual(resp.status_code, 200, + msg="ERROR getting IM index:" + resp.text) + res = json.loads(resp.text) + self.assertEqual(res['openapi'], '3.0.0') + def test_20_create(self): radl = read_file_as_string('../files/test_simple.json') resp = self.create_request("POST", "/infrastructures", From bdeaf8a812853a07110fe35fe4b3ca3bd0af95db Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 12:50:59 +0200 Subject: [PATCH 16/20] Add REST index page --- IM/REST.py | 1 + test/unit/REST.py | 1 + 2 files changed, 2 insertions(+) diff --git a/IM/REST.py b/IM/REST.py index 6b10191d..572f3c55 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -946,6 +946,7 @@ def RESTIndex(): rest_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(rest_path, 'swagger_api.yaml') api_docs = yaml.safe_load(open(abs_file_path, 'r')) + api_docs['servers'][0]['url'] = flask.request.url_root return flask.make_response(json.dumps(api_docs), 200, {'Content-Type': 'application/json'}) diff --git a/test/unit/REST.py b/test/unit/REST.py index 2c80b548..56b24ba4 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -571,6 +571,7 @@ def test_GeVersion(self): def test_Index(self): res = self.client.get('/') self.assertEqual(res.json['openapi'], '3.0.0') + self.assertEqual(res.json['servers'][0]['url'], 'http://localhost/') @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot): From adee1465122865a44a89c9024fe47e118f692efa Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 13:10:45 +0200 Subject: [PATCH 17/20] Add swaagger UI --- IM/REST.py | 13 +++++++++++++ pyproject.toml | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/IM/REST.py b/IM/REST.py index 572f3c55..ff57d8ca 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -22,6 +22,7 @@ import os import yaml +from flask_swagger_ui import get_swaggerui_blueprint from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher from cheroot.ssl.builtin import BuiltinSSLAdapter from werkzeug.middleware.proxy_fix import ProxyFix @@ -62,6 +63,13 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1) flask_server = None +# Add the swagger UI +swaggerui_blueprint = get_swaggerui_blueprint( + '/api/docs', # Swagger UI static files will be mapped to '{SWAGGER_URL}/dist/' + '/swagger.json' +) +app.register_blueprint(swaggerui_blueprint) + def run_in_thread(host, port): flask_thr = threading.Thread(target=run, args=(host, port)) @@ -943,6 +951,11 @@ def RESTGetVersion(): @app.route('/') def RESTIndex(): + return flask.redirect('/api/docs') + + +@app.route('/swagger.json') +def RESTSwagger(): rest_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(rest_path, 'swagger_api.yaml') api_docs = yaml.safe_load(open(abs_file_path, 'r')) diff --git a/pyproject.toml b/pyproject.toml index fd6487a1..064e3e0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,8 @@ dependencies = [ "requests-cache >= 1.0.0", "packaging", "werkzeug", - "xmltodict" + "xmltodict", + "flask-swagger-ui" ] license = {text = "GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt"} dynamic = ["version", "readme"] From c8a462f963da37832aea2f232d3668e8ece0b8bd Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 13:27:02 +0200 Subject: [PATCH 18/20] Fix test --- test/integration/TestREST.py | 7 ------- test/integration/TestREST_JSON.py | 7 ------- test/unit/REST.py | 12 ++++++++++-- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/test/integration/TestREST.py b/test/integration/TestREST.py index 65bb4fec..c7dd8c42 100755 --- a/test/integration/TestREST.py +++ b/test/integration/TestREST.py @@ -139,13 +139,6 @@ def test_05_version(self): self.assertEqual( resp.text, version, msg="Incorrect version. Expected %s, obtained: %s" % (version, resp.text)) - def test_07_index(self): - resp = self.create_request("GET", "/") - self.assertEqual(resp.status_code, 200, - msg="ERROR getting IM index:" + resp.text) - res = json.loads(resp.text) - self.assertEqual(res['openapi'], '3.0.0') - def test_10_list(self): resp = self.create_request("GET", "/infrastructures") self.assertEqual(resp.status_code, 200, diff --git a/test/integration/TestREST_JSON.py b/test/integration/TestREST_JSON.py index 032503e6..30b26124 100755 --- a/test/integration/TestREST_JSON.py +++ b/test/integration/TestREST_JSON.py @@ -131,13 +131,6 @@ def wait_inf_state(self, state, timeout, incorrect_states=None, vm_ids=None): return all_ok - def test_10_index(self): - resp = self.create_request("GET", "/") - self.assertEqual(resp.status_code, 200, - msg="ERROR getting IM index:" + resp.text) - res = json.loads(resp.text) - self.assertEqual(res['openapi'], '3.0.0') - def test_20_create(self): radl = read_file_as_string('../files/test_simple.json') resp = self.create_request("POST", "/infrastructures", diff --git a/test/unit/REST.py b/test/unit/REST.py index 56b24ba4..5f39f371 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -570,8 +570,16 @@ def test_GeVersion(self): def test_Index(self): res = self.client.get('/') - self.assertEqual(res.json['openapi'], '3.0.0') - self.assertEqual(res.json['servers'][0]['url'], 'http://localhost/') + self.assertEqual(res.status_code, 302) + self.assertEqual(res.headers['Location'], '/api/docs') + res = self.client.get('http://localhost/api/docs/') + self.assertEqual(res.status_code, 200) + self.assertIn("Swagger UI", res.text) + self.assertIn('"url": "/swagger.json"', res.text) + res = self.client.get('/swagger.json') + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json["openapi"], "3.0.0") + @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot): From fe5a0a98a819e7768be0100411973cf719c71971 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 13:29:18 +0200 Subject: [PATCH 19/20] Fix test --- test/unit/REST.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/REST.py b/test/unit/REST.py index 5f39f371..dde24b1c 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -580,7 +580,6 @@ def test_Index(self): self.assertEqual(res.status_code, 200) self.assertEqual(res.json["openapi"], "3.0.0") - @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot): """Test REST StopVM.""" From 3ec1a9f8c5b0acedbe9de76c618a4823cb339820 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 7 Jun 2024 13:52:08 +0200 Subject: [PATCH 20/20] Remove swagger ui --- IM/REST.py | 13 ------------- pyproject.toml | 3 +-- test/integration/TestREST_JSON.py | 7 +++++++ test/unit/REST.py | 11 ++--------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/IM/REST.py b/IM/REST.py index ff57d8ca..572f3c55 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -22,7 +22,6 @@ import os import yaml -from flask_swagger_ui import get_swaggerui_blueprint from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher from cheroot.ssl.builtin import BuiltinSSLAdapter from werkzeug.middleware.proxy_fix import ProxyFix @@ -63,13 +62,6 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1) flask_server = None -# Add the swagger UI -swaggerui_blueprint = get_swaggerui_blueprint( - '/api/docs', # Swagger UI static files will be mapped to '{SWAGGER_URL}/dist/' - '/swagger.json' -) -app.register_blueprint(swaggerui_blueprint) - def run_in_thread(host, port): flask_thr = threading.Thread(target=run, args=(host, port)) @@ -951,11 +943,6 @@ def RESTGetVersion(): @app.route('/') def RESTIndex(): - return flask.redirect('/api/docs') - - -@app.route('/swagger.json') -def RESTSwagger(): rest_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(rest_path, 'swagger_api.yaml') api_docs = yaml.safe_load(open(abs_file_path, 'r')) diff --git a/pyproject.toml b/pyproject.toml index 064e3e0d..fd6487a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,7 @@ dependencies = [ "requests-cache >= 1.0.0", "packaging", "werkzeug", - "xmltodict", - "flask-swagger-ui" + "xmltodict" ] license = {text = "GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt"} dynamic = ["version", "readme"] diff --git a/test/integration/TestREST_JSON.py b/test/integration/TestREST_JSON.py index 30b26124..fbbeddb4 100755 --- a/test/integration/TestREST_JSON.py +++ b/test/integration/TestREST_JSON.py @@ -131,6 +131,13 @@ def wait_inf_state(self, state, timeout, incorrect_states=None, vm_ids=None): return all_ok + def test_07_index(self): + resp = self.create_request("GET", "/") + self.assertEqual(resp.status_code, 200, + msg="ERROR getting IM index:" + resp.text) + res = json.loads(resp.text) + self.assertEqual(res['openapi'], '3.0.0') + def test_20_create(self): radl = read_file_as_string('../files/test_simple.json') resp = self.create_request("POST", "/infrastructures", diff --git a/test/unit/REST.py b/test/unit/REST.py index dde24b1c..56b24ba4 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -570,15 +570,8 @@ def test_GeVersion(self): def test_Index(self): res = self.client.get('/') - self.assertEqual(res.status_code, 302) - self.assertEqual(res.headers['Location'], '/api/docs') - res = self.client.get('http://localhost/api/docs/') - self.assertEqual(res.status_code, 200) - self.assertIn("Swagger UI", res.text) - self.assertIn('"url": "/swagger.json"', res.text) - res = self.client.get('/swagger.json') - self.assertEqual(res.status_code, 200) - self.assertEqual(res.json["openapi"], "3.0.0") + self.assertEqual(res.json['openapi'], '3.0.0') + self.assertEqual(res.json['servers'][0]['url'], 'http://localhost/') @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot):