diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8f58a84c..dfb7eade 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3. uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: python -m pip install tox diff --git a/IM/REST.py b/IM/REST.py index f04f8853..572f3c55 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -19,6 +19,8 @@ import json import base64 import flask +import os +import yaml from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher from cheroot.ssl.builtin import BuiltinSSLAdapter @@ -939,6 +941,15 @@ def RESTGetVersion(): return return_error(400, "Error getting IM version: %s" % get_ex_error(ex)) +@app.route('/') +def RESTIndex(): + rest_path = os.path.dirname(os.path.abspath(__file__)) + abs_file_path = os.path.join(rest_path, 'swagger_api.yaml') + api_docs = yaml.safe_load(open(abs_file_path, 'r')) + api_docs['servers'][0]['url'] = flask.request.url_root + return flask.make_response(json.dumps(api_docs), 200, {'Content-Type': 'application/json'}) + + @app.route('/infrastructures//vms//disks//snapshot', methods=['PUT']) def RESTCreateDiskSnapshot(infid=None, vmid=None, disknum=None): try: diff --git a/IM/config.py b/IM/config.py index 0429ae94..3f5c9602 100644 --- a/IM/config.py +++ b/IM/config.py @@ -67,7 +67,7 @@ class Config: LOG_FILE = '/var/log/im/inf.log' LOG_FILE_MAX_SIZE = 10485760 LOG_LEVEL = "INFO" - CONTEXTUALIZATION_DIR = '/usr/share/im/contextualization' + CONTEXTUALIZATION_DIR = IM_PATH + '/../contextualization' RECIPES_DIR = CONTEXTUALIZATION_DIR + '/AnsibleRecipes' RECIPES_DB_FILE = CONTEXTUALIZATION_DIR + '/recipes_ansible.db' MAX_CONTEXTUALIZATION_TIME = 7200 diff --git a/im_service.py b/IM/im_service.py similarity index 99% rename from im_service.py rename to IM/im_service.py index 98c3ee38..0edaf452 100755 --- a/im_service.py +++ b/IM/im_service.py @@ -33,9 +33,6 @@ from IM.ServiceRequests import IMBaseRequest from IM import __version__ as version -if sys.version_info <= (2, 6): - print("Must use python 2.6 or greater") - sys.exit(1) logger = logging.getLogger('InfrastructureManager') @@ -420,7 +417,7 @@ def signal_int_handler(signal, frame): im_stop() -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser(description='IM service') parser.add_argument('--version', help='Show IM service version.', dest="version", action="store_true", default=False) @@ -436,3 +433,7 @@ def signal_int_handler(signal, frame): config_logging() launch_daemon() + + +if __name__ == "__main__": + main() diff --git a/IM/swagger_api.yaml b/IM/swagger_api.yaml new file mode 100644 index 00000000..114b04d6 --- /dev/null +++ b/IM/swagger_api.yaml @@ -0,0 +1,1605 @@ +openapi: 3.0.0 + +info: + description: Infrastructure Manager (IM) REST API. + version: 1.17.0 + title: Infrastructure Manager (IM) REST API + contact: + email: products@grycap.upv.es + license: + name: GPL 3.0 + url: 'https://www.gnu.org/licenses/gpl-3.0.en.html' + +tags: + - name: infrastructures + description: Manages Virtual Infrastructures. + - name: version + description: Get IM server version. + - name: clouds + description: Get cloud information. + +paths: + + /version: + get: + tags: + - version + summary: Get IM server version. + description: Get IM server version. + operationId: GetVersion + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: 1.10.0 + '400': + description: Invalid status value + + /infrastructures: + get: + tags: + - infrastructures + summary: List user infrastructures. + security: + - IMAuth: [] + description: >- + Return a list of URIs referencing the infrastructures associated to the + IM user. + operationId: GetInfrastructures + parameters: + - name: filter + in: query + description: >- + The filter parameter is optional and it is a regular expression + (python format) to search in the RADL or TOSCA used to create the + infrastructure. If not specified all the user infrastructures will + be returned. + required: false + schema: + type: string + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id1 + http://server.com:8800/infrastructures/inf_id2 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id1"}, + {"uri": "http://server.com:8800/infrastructures/inf_id2"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + post: + tags: + - infrastructures + summary: Creates an infrastructures. + security: + - IMAuth: [] + description: >- + Create and configure an infrastructure with the requirements specified + in RADL or TOSCA document. + operationId: CreateInfrastructure + parameters: + - name: async + in: query + description: >- + The async parameter is optional and is a flag to specify if the call + will not wait for the VMs to be created. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + - name: dry_run + in: query + description: >- + parameter is optional and is a flag to specify if the call will not create the VMs + and will only return the ammount of resources needed to deploy the infrastructure. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + requestBody: + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 2G and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + deploy node 1 + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 2147483648 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + - class: deploy + system: wn + vm_number: 1 + text/yaml: + schema: + type: string + example: + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + web_server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: linux + distribution: CentOS + version: 8 + description: RADL (in plain RADL or in JSON formats) or TOSCA document (YAML). + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: 'http://server.com:8800/infrastructures/inf_id1' + application/json: + examples: + response: + value: | + { + "uri": "http://server.com:8800/infrastructures/inf_id1" + } + response_dry_run: + value: | + { + "compute": [ + {"cpu": 2, "memory": 4096, "disk": 20}, + {"cpu": 1, "memory": 2048, "disk": 10} + ], + "storage": [ + {"size": 100} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + put: + tags: + - infrastructures + summary: Import an infrastructure. + security: + - IMAuth: [] + description: >- + Take control of the infrastructure serialized in in the body and return + the ID associated in the server. (See GET + /infrastructures/{infId}/data). + operationId: Importfrastructure + requestBody: + content: + application/json: + schema: + type: string + description: >- + JSON data of the infrastructure obtained with GET + /infrastructures/{infId}/data. + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: 'http://server.com:8800/infrastructures/inf_id1' + application/json: + examples: + response: + value: | + { + "uri": "http://server.com:8800/infrastructures/inf_id1" + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '415': + description: Unsupported Media type + + /infrastructures/{InfId}: + get: + tags: + - infrastructures + summary: List of VMs in a infrastructure. + security: + - IMAuth: [] + description: >- + Return a list of URIs referencing the virtual machines associated to the + infrastructure with ID InfId + operationId: GetVMList + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id/vms/0 + http://server.com:8800/infrastructures/inf_id/vms/1 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/0"}, + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/1"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + post: + tags: + - infrastructures + summary: Add resources to an infrastructures. + security: + - IMAuth: [] + description: > + Add the resources specified in the body contents (in TOSCA, RADL plain + or in JSON formats) to the infrastructure with ID infId. + + In case of using RADL as input the deploy instructions in the radl must + refer to systems already defined. If all the systems defined in radl are + new, they will be added. Otherwise the new systems defined will be + ignored. All the systems specified in the deploy must be specified in + the radl. If they has been already defined only a reference is needed. + This is a simple example to deploy one new VM from an alreay defined + system: + + Using TOSCA as input this method can be used to add or remove resources + depending on the number of resources specified in the new TOSCA document + sent. If new nodes are added in the body compared with the last TOSCA + sent to the IM, these new nodes will be added. For example an + infrastructure has been created with this TOSCA document: + [tosca_create.yml](https://github.com/grycap/im/blob/master/test/files/tosca_create.yml) + it launches one DB server and one Web server. If this TOSCA document is + sent as body of this POST function: + [tosca_add.yml](https://github.com/grycap/im/blob/master/test/files/tosca_add.yml), + a new web server will be added as the number of web servers has been + increased to two (count parameter of scalable capability). However if + this document is sent after the node addition (the number of web servers + will be two): + [tosca_remove.yml](https://github.com/grycap/im/blob/master/test/files/tosca_remove.yml) + , a web server (the VM with the ID 2 as specified in the removal_list + parameter) will be removed. + operationId: AddResources + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: context + in: query + description: >- + The context parameter is optional and is a flag to specify if the + contextualization step will be launched just after the VM addition. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'true' + requestBody: + content: + text/plain: + schema: + type: string + example: | + network net + system node + deploy node 1 + application/json: + schema: + type: string + example: + - class: network + id: net + reference: true + - class: system + id: node + reference: true + - class: deploy + system: wn + vm_number: 1 + text/yaml: + schema: + type: string + example: + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + web_server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: linux + distribution: CentOS + version: 8 + description: RADL (in plain RADL or in JSON formats) or TOSCA document (YAML). + required: true + responses: + '200': + description: successful operation + content: + text/uri-list: + examples: + response: + value: | + http://server.com:8800/infrastructures/inf_id/vms/0 + http://server.com:8800/infrastructures/inf_id/vms/1 + application/json: + examples: + response: + value: | + { + "uri-list": [ + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/0"}, + {"uri": "http://server.com:8800/infrastructures/inf_id/vms/1"} + ] + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + delete: + tags: + - infrastructures + summary: Delete an infrastructure. + security: + - IMAuth: [] + description: Undeploy the virtual machines associated to the infrastructure with ID. + operationId: DestroyInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: force + in: query + description: >- + The force parameter is optional and is a flag to specify that the + infra will be from the IM although not all resources are deleted. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + - name: async + in: query + description: >- + The async parameter is optional and is a flag to specify if the call + will not wait the infrastructure to be deleted. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/state: + get: + tags: + - infrastructures + summary: Get infrastructure state. + security: + - IMAuth: [] + description: Return a the state of the infrastructure with ID InfId. + operationId: GetInfrastructureState + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/InfrastructureState' + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/outputs: + get: + tags: + - infrastructures + summary: Get the infrastructure outputs. + security: + - IMAuth: [] + description: >- + In case of TOSCA documents it will return a JSON object with the outputs + of the TOSCA document. + operationId: GetInfrastructureOutpus + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: | + { + "output_name1": "output_value1", + "output_name2": "output_value2", + } + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/contmsg: + get: + tags: + - infrastructures + summary: Get a infrastructure contextualization message. + security: + - IMAuth: [] + description: >- + Return a string with the contextualization message. In case of + headeronly flag is set to ‘yes’, ‘true’ or ‘1’ only the initial part of + the infrastructure contextualization log will be returned (without any + VM contextualization log). + operationId: GetInfrastructureContmsg + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: headeronly + in: query + description: >- + In case of headeronly flag is set to `yes`, `true` or `1` only the + initial part of the infrastructure contextualization log will be + returned (without any VM contextualization log). + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: > + 2019-12-11 11:08:14.574891: Select master VM + + 2019-12-11 11:08:14.576685: Wait master VM to boot + + 2019-12-11 11:08:14.577905: Wait master VM to have the SSH + active. + application/json: + examples: + response: + value: + contmsg: | + 2019-12-11 11:08:14.574891: Select master VM + 2019-12-11 11:08:14.576685: Wait master VM to boot + 2019-12-11 11:08:14.577905: Wait master VM to have the SSH active. + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/data: + get: + tags: + - infrastructures + summary: Export an infrastructure. + security: + - IMAuth: [] + description: >- + Return the serialization of the infrastructure with ID InfId. This + function is useful to transfer the control of an infrastructure to other + IM server (See PUT /infrastructures). In case of delete flag is set to + ‘yes’, ‘true’ or ‘1’ the data not only will be exported but also the + infrastructure will be set deleted (the virtual infrastructure will not + be modified). + operationId: GetInfrastructureData + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: delete + in: query + description: >- + In case of delete flag is set to ‘yes’, ‘true’ or ‘1’ the data not + only will be exported but also the infrastructure will be set + deleted (the virtual infrastructure will not be modified). + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + vm_list: [] + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/radl: + get: + tags: + - infrastructures + summary: Get the infrastructure RADL used to create it. + security: + - IMAuth: [] + description: >- + Return a string with the original specified RADL of the infrastructure + with ID InfId. + operationId: GetInfrastructureRADL + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + network net (outbound = 'yes') + system node ( + ... + ) + application/json: + examples: + response: + value: + radl: "network net (outbound = 'yes')\nsystem node ( ... )" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/tosca: + get: + tags: + - infrastructures + summary: Get the TOSCA representation of the infrastructure. + security: + - IMAuth: [] + description: >- + Return a string with the TOSCA representation of the infrastructure with + ID InfId. + operationId: GetInfrastructureTOSCA + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + tosca: | + tosca_definitions_version: tosca_simple_yaml_1_0 + description: Some TOSCA template + ... + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/authorization: + get: + tags: + - infrastructures + summary: Get the list of infrastructure owners. + security: + - IMAuth: [] + description: >- + Return a list of strings with the set of users of the infrastructure + with ID InfId. + operationId: GetInfrastructureOwners + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + user1 + user2 + application/json: + examples: + response: + value: + owners: + - user1 + - user2 + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + post: + tags: + - infrastructures + summary: Change the authorization data of the infrastructure. + security: + - IMAuth: [] + description: >- + Change the authorization data of the infrastructure with ID InfId. + The body of the request contains the new authorization data in json + format. It can be a pair username-password or a token. In case of using + a token it must be a valid access token. + operationId: ChangeInfrastructureAuth + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: overwrite + in: query + description: >- + The overwrite parameter is optional and is a flag to specify if the + authorization data will be overwrited or will be appended. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + requestBody: + content: + application/json: + schema: + oneOf: + - type: object + properties: + username: + type: string + example: new_user + password: + type: string + example: new_password + - type: object + properties: + token: + type: string + example: acces_token_new_user + + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/stop: + put: + tags: + - infrastructures + summary: Stop an infrastructure. + security: + - IMAuth: [] + description: >- + Perform the stop operation in all the virtual machines in the + infrastructure. + operationId: StopInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/start: + put: + tags: + - infrastructures + summary: Start an infrastructure. + security: + - IMAuth: [] + description: >- + Perform the start operation in all the (previuosly stopped) virtual + machines in the infrastructure. + operationId: StartInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/reconfigure: + put: + tags: + - infrastructures + summary: Reconfigure an infrastructure. + security: + - IMAuth: [] + description: Re-start the contextualization process of the infrastructure. + operationId: ReconfigureInfrastructure + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: vm_list + in: query + description: >- + The vm_list parameter is optional and is a coma separated list of + IDs of the VMs to reconfigure. If not specified all the VMs will be + reconfigured + required: false + schema: + type: string + requestBody: + content: + text/plain: + schema: + type: string + example: | + configure node ( + @begin + --- + - tasks: + - debug: msg="Some message" + @end + ) + application/json: + schema: + type: string + example: + - class: configure + id: node + recipes: | + - tasks: + - debug: msg="Some message" + description: >- + Optional RADL (in plain RADL or in JSON formats) with new + configuration recipes. If not specified the current configuration + recipes will be executed again. + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + + /infrastructures/{InfId}/vms/{VMId}: + get: + tags: + - infrastructures + summary: Get VM info. + security: + - IMAuth: [] + description: >- + Return information about the virtual machine with ID VMId associated to + the infrastructure with ID InfId + operationId: GetVMInfo + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 2G and + net_interface.0.ip = '8.8.8.8' and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 2147483648 + net_interface.0.ip: 8.8.8.8 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + put: + tags: + - infrastructures + summary: Alter VM. + security: + - IMAuth: [] + description: >- + Change the features of the virtual machine with ID VMId in the + infrastructure with with ID infId. + operationId: AlterVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + requestBody: + content: + text/plain: + schema: + type: string + example: | + system node ( + memory.size >= 4G + ) + application/json: + schema: + type: string + example: + - class: system + id: node + memory.size_min: 4294967296 + description: RADL (in plain RADL or in JSON formats). + required: true + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: | + network net (outbound = 'yes') + system node ( + cpu.count >= 2 and + memory.size >= 4G and + net_interface.0.ip = '8.8.8.8' and + net_interface.0.connection = 'net' and + disk.0.image.url = 'one://someserver.com/123' + ) + application/json: + schema: + type: string + example: + - class: network + id: net + outbound: "yes" + - class: system + id: node + cpu.count_min: 2 + memory.size_min: 4294967296 + net_interface.0.ip: 8.8.8.8 + net_interface.0.connection: net + disk.0.image.url: "one://someserver.com/123" + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + '415': + description: Unsupported Media type + delete: + tags: + - infrastructures + summary: Delete a VM. + security: + - IMAuth: [] + description: >- + Undeploy the virtual machine VMId associated to the infrastructure with + ID infId + operationId: DestroyVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/contmsg: + get: + tags: + - infrastructures + summary: Get a VM contextualization message. + security: + - IMAuth: [] + description: >- + Return a string with the contextualization message associated to the + virtual machine with ID VMId in the infrastructure with with ID infId + operationId: GetVMContMsg + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: | + Launch task: basic + ... + application/json: + examples: + response: + value: + contmsg: | + Launch task: basic + ... + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/{Property}: + get: + tags: + - infrastructures + summary: Get a VM property. + security: + - IMAuth: [] + description: >- + Return a property associated to the virtual machine with ID VMId in the + infrastructure with with ID infId + operationId: GetVMProperty + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + - name: Property + in: path + description: The specific VM RADL property to get. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + text/plain: + examples: + response: + value: property_value + application/json: + examples: + response: + value: + property_name: property_value + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/stop: + put: + tags: + - infrastructures + summary: Stop a VM. + security: + - IMAuth: [] + description: >- + Perform the stop operation in the virtual machine VMId of the + infrastructure InfId. The default behavior is suspending the VM if this + operation is not available the VM will be stopped. + operationId: StopVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/start: + put: + tags: + - infrastructures + summary: Start a VM. + security: + - IMAuth: [] + description: >- + Perform the start operation in the (previuosly stopped) virtual machine + VMId of the infrastructure InfId. + operationId: StartVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/reboot: + put: + tags: + - infrastructures + summary: Reboot a VM. + security: + - IMAuth: [] + description: >- + Perform the reboot operation in the virtual machine VMId of the + infrastructure InfId. + operationId: RebootVM + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + responses: + '200': + description: successful operation + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /infrastructures/{InfId}/vms/{VMId}/disks/{diskNum}/snapshot: + put: + tags: + - infrastructures + summary: Create a disk snapshot. + security: + - IMAuth: [] + description: >- + Create a snapshot of the specified diskNum in the VM VMId of the + infrastructure with ID InfId and returns the image url of the new + created image in IM format. + operationId: CreateSnapShot + parameters: + - name: InfId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: VMId + in: path + description: The ID of the specific VM. + required: true + schema: + type: string + - name: diskNum + in: path + description: The number of the specific disk. + required: true + schema: + type: string + - name: image_name + in: query + description: The name to assing to the created snapshot. + required: true + schema: + type: string + - name: auto_delete + in: query + description: >- + The auto_delete flag specifies that the snapshot will be deleted + when the infrastructure is destroyed + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' + responses: + '200': + description: successful operation + content: + text/plain: + schema: + type: string + example: 'one://server.com/image_id' + '400': + description: Invalid status value + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: Not Found + + /clouds/{CloudId}/images: + get: + tags: + - clouds + summary: Get the list of available images in the specified cloud provider. + security: + - IMAuth: [] + description: > + Return a list of URIs referencing the available images in the specified + cloud provider. The id cloudId is relative to the id field in the + AUTHORIZATION header. + operationId: GetCloudImages + parameters: + - name: CloudId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + - name: filters + in: query + description: >- + The optional filters parameter enables filterin the list of images. + It is a comma separated list of keypair values + ('key1=val1,key2=value2'). + + This field is cloud provider specific (e.g. 'region=region_name' for + Amazon EC2, GCE or Azure). + required: false + schema: + type: string + default: '' + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + images: + - uri: ost://hostname/image-id1 + name: Image Name1 + - uri: ost://hostname/image-id2 + name: Image Name2 + '400': + description: Invalid status value + '401': + description: Unauthorized + + /clouds/{CloudId}/quotas: + get: + tags: + - clouds + summary: >- + Get available and used resources for the current user in the specified + cloud provider. + security: + - IMAuth: [] + description: > + Return a dictionary with available and used resources for the current + user in the specified cloud provider. The id cloudId is relative to the + id field in the AUTHORIZATION header. + operationId: GetCloudQuotas + parameters: + - name: CloudId + in: path + description: The ID of the specific infrastructure. + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + examples: + response: + value: + quotas: + cores: + used: 1 + limit: 10 + ram: + used: 1 + limit: 10 + instances: + used: 1 + limit: 10 + floating_ips: + used: 1 + limit: 10 + security_groups: + used: 1 + limit: 10 + '400': + description: Invalid status value + '401': + description: Unauthorized + +externalDocs: + description: Find out more about IM + url: 'http://www.grycap.upv.es/im' +servers: + - url: 'https://appsgrycap.i3m.upv.es:31443/im' +components: + schemas: + State: + type: string + enum: + - pending + - running + - configured + - unconfigured + - stopped + - 'off' + - failed + - unknown + - deleting + title: State + example: running + InfrastructureState: + type: object + properties: + state: + $ref: '#/components/schemas/State' + vm_states: + type: array + items: + $ref: '#/components/schemas/State' + example: + - running + - running + title: InfrastructureState + + securitySchemes: + IMAuth: + type: apiKey + in: header + name: Authorization + description: >- + The Authentication header must provide the content of the + [Authorization + File](https://imdocs.readthedocs.io/en/latest/client.html#auth-file), + but putting all the elements in one line using “\n” as separator. diff --git a/README.md b/README.md index c424ccfd..697a695d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ This software has received a gold badge according to the [Software Quality Baseline criteria](https://github.com/indigo-dc/sqa-baseline) defined by the [EOSC-Synergy](https://www.eosc-synergy.eu) project. -## 1 DOCKER IMAGE (Recommended Option) +## 1 DOCKER IMAGE The recommended option to use the Infrastructure Manager service is using the available docker image. A Docker image named `ghcr.io/grycap/im` has been @@ -106,193 +106,7 @@ helm install --namespace=im --create-namespace im grycap/IM All the information about this chart is available at the [IM chart README](https://github.com/grycap/helm-charts/blob/master/IM/README.md). -## 3 INSTALLATION - -### 3.1 REQUISITES - -IM is based on Python, so Python 2.7 or higher (Python 3.6 or higher -recommended) runtime and standard library must be installed in the system. - -If you use pip to install the IM, all the requisites will be installed. -However, if you install IM from sources you should install: - -* The RADL parser (), available in pip - as the ``RADL`` package. -* The paramiko ssh2 protocol library for python version 1.14 or later - (), typically available as the - ``python-paramiko`` package. -* The YAML library for Python, typically available as the ``python-yaml`` or - ``PyYAML`` package. -* The suds library for Python, typically available as the ``python-suds`` - package. -* The Netaddr library for Python, typically available as the ``python-netaddr`` - package. -* The Requests library for Python, typically available as the - ``python-requests`` package. -* TOSCA parser library for Python, available as the ``tosca-parser`` package in - pip. -* Ansible () to configure nodes in the - infrastructures. - In particular, Ansible 2.4+ must be installed. - To ensure the functionality the following values must be set in the - ansible.cfg file (usually found in /etc/ansible/): - -```yml -[defaults] -transport = smart -host_key_checking = False -nocolor = 1 - -become_user = root -become_method = sudo - -[paramiko_connection] - -record_host_keys=False - -[ssh_connection] - -# Only in systems with OpenSSH support to ControlPersist -ssh_args = -o ControlMaster=auto -o ControlPersist=900s -o UserKnownHostsFile=/dev/null -# In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) -#ssh_args = -o UserKnownHostsFile=/dev/null -pipelining = True -``` - -### 3.2 OPTIONAL PACKAGES - -The Flask framework () is used for the REST API. -It is typically available as the ``python3-flask`` and ``python3-werkzeug`` system -packages or ``flask`` and ``werkzeug`` pip packages. - -The CherryPy Web framework (), is needed for the REST -API. It is typically available as the ``python3-cherrypy`` system package or -``CherryPy`` pip package. In newer versions (9.0 and later) the functionality -has been moved to the ``cheroot`` library (). -It is typically available ``python3-cheroot`` system package or ``cheroot`` pip package. - -Apache-libcloud () 3.0 or later is used in the -LibCloud, OpenStack and GCE connectors. It is typically available as the -``python3-libcloud`` system package or ``apache-libcloud`` pip package. - -Boto () 2.29.0 or later is used as interface to -Amazon EC2. It is available as package named ``python3-boto`` in Debian based -distributions or ``boto`` pip package. It can also be downloaded from boto -GitHub repository (). -Download the file and copy the boto subdirectory into the IM install path. - -In case of using the a MySQL DB as the backend to store IM data. The Python -interface to MySQL must be installed, typically available as the package -``python-mysqldb`` or ``MySQL-python`` package. In case of using Python 3 use -the PyMySQL package, available as the package ``python3-pymysql`` on -debian systems or ``PyMySQL`` package in pip. - -In case of using the a MongoDB as the backend to store IM data. The Python -interface to MongoDB must be installed, typically available as the package -``python-pymongo``package in most distributions or ``pymongo`` pip package. - -In case of using the SSL secured version of the REST API pyOpenSSL -() must be installed. available as ``pyOpenSSL`` -package in pip. - -Azure python SDK () is used -to connect with the Microsoft Azure platform. The easiest way is to install all -the required packages with pip: - -```sh -pip install msrest msrestazure azure-common azure-mgmt-storage \ - azure-mgmt-compute azure-mgmt-network azure-mgmt-resource \ - azure-mgmt-dns azure-identity -``` - -The VMware vSphere API Python Bindings () -are needed by the vSphere connector. It is available as the package ``pyvmomi`` -at the pip repository. - -### 3.3 INSTALLING - -#### 3.3.1 From PIP - -First you need to install pip tool and some packages needed to compile some of -the IM requirements. To install them in Debian and Ubuntu based distributions, -do:: - -```sh -apt update -apt install -y gcc python3-dev libffi-dev libssl-dev python3-pip sshpass \ - default-libmysqlclient-dev -``` - -In Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, -Fedora, etc.), do: - -```sh -yum install -y epel-release -yum install -y which gcc python3-devel libffi-devel openssl-devel \ - python3-pip sshpass -``` - -Then you only have to call the install command of the pip tool with the IM -package: - -```sh -pip3 install IM -``` - -You can also install an specific branch of the Github repository: - -```sh -pip install git+https://github.com/grycap/im.git@master -``` - -Pip will also install the, non installed, pre-requisites needed. So Ansible 2.4 -or later will be installed in the system. Some of the optional packages are -also installed please check if some of IM features that you need requires to -install some of the packages of section OPTIONAL PACKAGES. - -You must also remember to modify the ansible.cfg file setting as specified in -the REQUISITES section. - -### 3.4 START IM ON BOOT - -In case that you want the IM service to be started at boot time, you must -execute the next set of commands: - -On Debian Systems: - -```sh -chkconfig im on -``` - -Or for newer systems like ubuntu 14.04: - -```sh -sysv-rc-conf im on -``` - -On RedHat Systems: - -```sh -update-rc.d im start 99 2 3 4 5 . stop 05 0 1 6 . -``` - -Or you can do it manually: - -```sh -ln -s /etc/init.d/im /etc/rc2.d/S99im -ln -s /etc/init.d/im /etc/rc3.d/S99im -ln -s /etc/init.d/im /etc/rc5.d/S99im -ln -s /etc/init.d/im /etc/rc1.d/K05im -ln -s /etc/init.d/im /etc/rc6.d/K05im -``` - -Adjust the installation path by setting the IMDAEMON variable at /etc/init.d/im -to the path where the IM im_service.py file is installed (e.g. -/usr/local/im/im_service.py), or set the name of the script file -(im_service.py) if the file is in the PATH (pip puts the im_service.py file in -the PATH as default). - -### 4 CONFIGURATION +### 3 CONFIGURATION Check the parameters in $IM_PATH/etc/im.cfg or /etc/im/im.cfg. See [IM Manual](https://imdocs.readthedocs.io/en/latest/manual.html#configuration) @@ -308,7 +122,7 @@ DATA_DB - must be set to the URL to access the database to store the IM data. SQLite: `sqlite:///etc/im/inf.dat` or MongoDB: `mongodb://username:password@server/db_name`, -#### 4.1 SECURITY +#### 3.1 SECURITY Security is disabled by default. Please notice that someone with local network access can "sniff" the traffic and get the messages with the IM with the diff --git a/doc/source/manual.rst b/doc/source/manual.rst index bf9a6d76..b3caa58a 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -43,152 +43,9 @@ Then install the IM chart (with Helm v3):: All the information about this chart is available at the `IM chart README `_. -IM Service Installation -======================= - -Prerequisites -------------- - -IM needs at least Python 2.7 (Python 3.6 or higher recommended) to run, as well as the next libraries: - -* `The RADL parser `_. - (Since IM version 1.5.3, it requires RADL version 1.1.0 or later). -* `The TOSCA parser `_. - A TOSCA YAML Spec 1.0 Parser. -* `paramiko `_, ssh2 protocol library for python - (version 1.14 or later). -* `PyYAML `_, a YAML parser. -* `suds `_, a full-featured SOAP library. -* `Netaddr `_, A Python library for representing - and manipulating network addresses. -* `Requests `_, A Python library for access REST APIs. - -Also, IM uses `Ansible `_ (2.4 or later) to configure the -infrastructure nodes. - -These components are usually available from the distribution repositories. - -Finally, check the next values in the Ansible configuration file -:file:`ansible.cfg`, (usually found in :file:`/etc/ansible`):: - - [defaults] - transport = smart - host_key_checking = False - nocolor = 1 - become_user = root - become_method = sudo - - [paramiko_connection] - - record_host_keys=False - - [ssh_connection] - - # Only in systems with OpenSSH support to ControlPersist - ssh_args = -o ControlMaster=auto -o ControlPersist=900s - # In systems with older versions of OpenSSH (RHEL 6, CentOS 6, SLES 10 or SLES 11) - #ssh_args = - pipelining = True - -Optional Packages ------------------ - -* `The Flask framework `_ is used for the REST API. - It is typically available as the 'python3-flask' and 'python3-werkzeug' packages. -* `The CherryPy Web framework `_, is needed for the REST API. - It is typically available as the 'python3-cherrypy' package. In newer versions (9.0 - and later) the functionality has been moved `the cheroot - library `_. It is typically available as the - 'python3-cherrot' package. -* `apache-libcloud `_ 3.0 or later is used in the - LibCloud, OpenStack, EGI and GCE connectors. -* `boto `_ 2.29.0 or later is used as interface to - Amazon EC2. It is available as package named 'python3-boto' in Debian based - distributions. It can also be downloaded from `boto GitHub repository `_. - Download the file and copy the boto subdirectory into the IM install path. -* `pyOpenSSL `_ is needed to secure the REST API - with SSL certificates (see :confval:`REST_SSL`). - pyOpenSSL can be installed using pip. -* `The Python interface to MySQL `_, is needed to access MySQL server as IM data - backend. It is typically available as the package 'python-mysqldb' or 'MySQL-python' package. In case of - using Python 3 use the PyMySQL package, available as the package 'python3-pymysql' on debian systems or PyMySQL - package in pip. -* `The Python interface to MongoDB `_, is needed to access MongoDB server as IM data - backend. It is typically available as the package 'python-pymongo' package in most distributions or pymongo - package in pip. -* `The Azure Python SDK `_, is needed by the Azure - connector. It is available as the package 'azure' at the pip repository. -* `The VMware vSphere API Python Bindings `_ are needed by the vSphere - connector. It is available as the package 'pyvmomi' at the pip repository. - - -Installation ------------- - -From Pip -^^^^^^^^ - -First you need to install pip tool and some packages needed to compile some of the IM requirements. -To install them in Debian and Ubuntu based distributions, do:: - - $ apt update - $ apt install gcc python3-dev libffi-dev libssl-dev python3-pip sshpass python3-requests - -In Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, -Fedora, etc.), do:: - - $ yum install epel-release - $ yum install which gcc python3-devel libffi-devel openssl-devel python3-pip sshpass default-libmysqlclient-dev - -Then you only have to call the install command of the pip tool with the IM package:: - - $ pip install IM - -You can also install an specific branch of the Github repository:: - - $ pip install git+https://github.com/grycap/im.git@master - -Pip will also install the, non installed, pre-requisites needed. So Ansible 2.4 or later will -be installed in the system. Some of the optional packages are also installed please check if some -of IM features that you need requires to install some of the packages of section "Optional Packages". - -You must also remember to modify the ansible.cfg file setting as specified in the -"Prerequisites" section. - Configuration -------------- - -If you want the IM Service to be started at boot time, do - -1. Update the value of the variable ``IMDAEMON`` in :file:`/etc/init.d/im` file - to the path where the IM im_service.py file is installed (e.g. /usr/local/im/im_service.py), - or set the name of the script file (im_service.py) if the file is in the PATH - (pip puts the im_service.py file in the PATH as default):: - - $ sudo sed -i 's/`IMDAEMON=.*/`IMDAEMON=/usr/local/IM-0.1/im_service.py'/etc/init.d/im - -2. Register the service. - -To do the last step on a Debian based distributions, execute:: - - $ sudo sysv-rc-conf im on - -if the package 'sysv-rc-conf' is not available in your distribution, execute:: - - $ sudo update-rc.d im start 99 2 3 4 5 . stop 05 0 1 6 . - -For Red Hat based distributions:: - - $ sudo chkconfig im on - -Alternatively, it can be done manually:: - - $ ln -s /etc/init.d/im /etc/rc2.d/S99im - $ ln -s /etc/init.d/im /etc/rc3.d/S99im - $ ln -s /etc/init.d/im /etc/rc5.d/S99im - $ ln -s /etc/init.d/im /etc/rc1.d/K05im - $ ln -s /etc/init.d/im /etc/rc6.d/K05im +============= IM reads the configuration from :file:`$IM_PATH/etc/im.cfg`, and if it is not available, does from ``/etc/im/im.cfg``. There is a template of :file:`im.cfg` @@ -338,21 +195,6 @@ Default Virtual Machine Options Contextualization ^^^^^^^^^^^^^^^^^ -.. confval:: CONTEXTUALIZATION_DIR - - Full path to the IM contextualization files. - The default value is :file:`/usr/share/im/contextualization`. - -.. confval:: RECIPES_DIR - - Full path to the Ansible recipes directory. - The default value is :file:`CONTEXTUALIZATION_DIR/AnsibleRecipes`. - -.. confval:: RECIPES_DB_FILE - - Full path to the Ansible recipes database file. - The default value is :file:`CONTEXTUALIZATION_DIR/recipes_ansible.db`. - .. confval:: MAX_CONTEXTUALIZATION_TIME Maximum time in seconds spent on contextualize a virtual machine before diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 69818681..101605f8 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -7,17 +7,31 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-distutils openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y patch wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ + pip3 install -U pip && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ - pip3 install urllib3==1.26.18 git+https://github.com/grycap/im@$BRANCH && \ + pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ + pip3 install git+https://github.com/micafer/libcloud@ost_nets_extra && \ + pip3 install apache-libcloud==3.8.0 git+https://github.com/grycap/im@$BRANCH && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Patch libcloud to add network extra +# untill this PR is merged and released +# https://github.com/apache/libcloud/pull/2016 +COPY ost.patch /tmp/ost.patch +RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch + +# Copy im configuration files +RUN mkdir /etc/im +RUN wget https://raw.githubusercontent.com/grycap/im/${BRANCH}/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/${BRANCH}/etc/logging.conf -O /etc/im/logging.conf + + # Set the VM_NUM_USE_CTXT_DIST to 3 for the tests RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/im.cfg @@ -28,4 +42,4 @@ COPY ansible.cfg /etc/ansible/ansible.cfg COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json # Start IM service -CMD im_service.py \ No newline at end of file +CMD /usr/local/bin/im_service \ No newline at end of file diff --git a/docker-devel/ost.patch b/docker-devel/ost.patch new file mode 100644 index 00000000..ba39ef96 --- /dev/null +++ b/docker-devel/ost.patch @@ -0,0 +1,14 @@ +@@ -3158,6 +3158,12 @@ + extra["router:external"] = obj.get("router:external") + if obj.get("subnets", None): + extra["subnets"] = obj.get("subnets") ++ if obj.get("tags", None): ++ extra["tags"] = obj.get("tags") ++ if obj.get("is_default", None) is not None: ++ extra["is_default"] = obj.get("is_default") ++ if obj.get("description", None): ++ extra["description"] = obj.get("description") + return OpenStackNetwork(id=obj["id"], name=obj["name"], cidr=None, driver=self, extra=extra) + + def ex_list_networks(self): + diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 8448cccb..e3d7f400 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,22 +1,38 @@ # Dockerfile to create a container with the IM service FROM ubuntu:22.04 + +ENV VERSION=1.17.0 + LABEL maintainer="Miguel Caballer " -LABEL version="1.17.0" +LABEL version="${VERSION}" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 # Ensure system is up to date with mandatory python packages installed -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-distutils openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ +RUN apt-get update && apt-get install --no-install-recommends -y patch wget python3 openssh-client sshpass vim libmysqlclient21 python3-mysqldb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ # Install IM RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ + pip3 install -U pip && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ - pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.7.0&& \ - pip3 install urllib3==1.26.18 IM==1.17.0 && \ + pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==8.7.0&& \ + pip3 install apache-libcloud==3.8.0 IM==${VERSION} && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Patch libcloud to add network extra +# untill this PR is merged and released +# https://github.com/apache/libcloud/pull/2016 +COPY ost.patch /tmp/ost.patch +RUN patch /usr/local/lib/python3.10/dist-packages/libcloud/compute/drivers/openstack.py < /tmp/ost.patch && rm /tmp/ost.patch + +# Copy im configuration files +RUN mkdir /etc/im +RUN mkdir /var/log/im +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/im.cfg -O /etc/im/im.cfg +RUN wget https://raw.githubusercontent.com/grycap/im/v${VERSION}/etc/logging.conf -O /etc/im/logging.conf + # Copy a ansible.cfg with correct minimum values COPY ansible.cfg /etc/ansible/ansible.cfg @@ -24,4 +40,4 @@ COPY ansible.cfg /etc/ansible/ansible.cfg COPY endpoints.json /usr/local/lib/python3.10/dist-packages/boto/endpoints.json # Start IM service -CMD im_service.py +CMD /usr/local/bin/im_service diff --git a/docker-py3/ost.patch b/docker-py3/ost.patch new file mode 100644 index 00000000..ba39ef96 --- /dev/null +++ b/docker-py3/ost.patch @@ -0,0 +1,14 @@ +@@ -3158,6 +3158,12 @@ + extra["router:external"] = obj.get("router:external") + if obj.get("subnets", None): + extra["subnets"] = obj.get("subnets") ++ if obj.get("tags", None): ++ extra["tags"] = obj.get("tags") ++ if obj.get("is_default", None) is not None: ++ extra["is_default"] = obj.get("is_default") ++ if obj.get("description", None): ++ extra["description"] = obj.get("description") + return OpenStackNetwork(id=obj["id"], name=obj["name"], cidr=None, driver=self, extra=extra) + + def ex_list_networks(self): + diff --git a/etc/im.cfg b/etc/im.cfg index be1b5b90..6e1d5ff5 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -85,9 +85,6 @@ REST_PORT = 8800 REST_ADDRESS = 0.0.0.0 # Contextualization data -CONTEXTUALIZATION_DIR = /usr/share/im/contextualization -RECIPES_DIR = %(CONTEXTUALIZATION_DIR)s/AnsibleRecipes -RECIPES_DB_FILE = %(CONTEXTUALIZATION_DIR)s/recipes_ansible.db MAX_CONTEXTUALIZATION_TIME = 7200 REMOTE_CONF_DIR = /var/tmp/.im # Interval to update the state of the contextualization process in the VMs (in secs) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fd6487a1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,103 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +[build-system] +requires = [ + "setuptools~=66.1", + "wheel~=0.37.1" +] +build-backend = "setuptools.build_meta" + +[project] +name = "IM" +description = "IM is a tool to manage virtual infrastructures on Cloud deployments" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Framework :: CherryPy", + "Framework :: Flask", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] +requires-python = ">=3.6" +dependencies = [ + "ansible >=2.4", + "paramiko >= 1.14", + "PyYAML", + "suds-community", + "cheroot", + "boto >= 2.29", + "apache-libcloud >= 3.2.0", + "RADL >= 1.3.3", + "flask", + "netaddr", + "requests >= 2.19", + "scp", + "tosca-parser", + "defusedxml", + "urllib3>=1.23", + "hvac", + "psutil", + "scar", + "requests-cache >= 1.0.0", + "packaging", + "werkzeug", + "xmltodict" +] +license = {text = "GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt"} +dynamic = ["version", "readme"] + +[project.urls] +Homepage = "https://www.grycap.upv.es/im" +Documentation = "https://imdocs.readthedocs.io" +Repository = "https://github.com/grycap/im" + +[project.optional-dependencies] +build = [ + "build==1.0.3" +] +publish = [ + "twine==4.0.2" +] +test = [ + "mock", + "coverage", +] + +[project.scripts] +im_service = "IM.im_service:main" + +[tool.setuptools] +packages = ["IM", "IM.ansible_utils", "IM.connectors", "IM.tosca", "IM.openid", "IM.tts", "contextualization"] + +[tool.setuptools.dynamic] +version = {attr = "IM.__version__"} +readme = {file = ["README.md"], content-type = "text/markdown"} + +[tool.setuptools.package-data] +IM = ["*.yaml"] + +[tool.distutils.bdist_wheel] +universal = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 2423968b..00000000 --- a/setup.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from IM import __version__ as version -from setuptools import setup -import os -import sys - -if not hasattr(sys, 'version_info') or sys.version_info < (2, 6): - raise SystemExit("IM requires Python version 2.6 or above.") - -suds_pkg = "suds" -sqlite_pkg = "pysqlite" -if sys.version_info > (3, 0): - suds_pkg = "suds-py3" - sqlite_pkg = "" - -# Avoid using wheel as it does not copy data_files to / dir -if 'bdist_wheel' in sys.argv: - raise RuntimeError("This setup.py does not support wheels") - -# Add contextualization dir files -install_path = '/usr/share/im' -datafiles = [(os.path.join(install_path, root), [os.path.join(root, f) for f in files]) - for root, dirs, files in os.walk("contextualization")] -# Add other special files -datafiles.append(('/etc/init.d', ['scripts/im'])) -datafiles.append(('/etc/systemd/system', ['scripts/im.service'])) -datafiles.append(('/etc/im', ['etc/im.cfg'])) -datafiles.append(('/etc/im', ['etc/logging.conf'])) -# force the im_service.py file to be allways in this path -datafiles.append(('/usr/bin', ['im_service.py'])) - -try: - long_desc = open('README.md').read() - long_desc_type = 'text/markdown' -except Exception as ex: - print("Error reading README: %s" % ex) - long_desc = "IM is a tool to manage virtual infrastructures on Cloud deployments" - long_desc_type = 'text/plain' - -setup(name="IM", version=version, - author='GRyCAP - Universitat Politecnica de Valencia', - author_email='micafer1@upv.es', - url='http://www.grycap.upv.es/im', - include_package_data=True, - packages=['IM', 'IM.ansible_utils', 'IM.connectors', 'IM.tosca', 'IM.openid', 'IM.tts'], - scripts=["im_service.py"], - data_files=datafiles, - license="GPL version 3, http://www.gnu.org/licenses/gpl-3.0.txt", - long_description=long_desc, - long_description_content_type=long_desc_type, - description="IM is a tool to manage virtual infrastructures on Cloud deployments", - platforms=["any"], - install_requires=["ansible >=2.4", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", - "boto >= 2.29", "apache-libcloud >= 3.2.0", "RADL >= 1.3.3", "flask", "netaddr", - "requests >= 2.19", "scp", "tosca-parser", 'defusedxml', 'urllib3>=1.23', 'hvac', - 'psutil', 'scar', 'requests-cache >= 1.0.0', 'packaging', 'werkzeug'] - ) diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index da9ef55a..3217f25e 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -272,7 +272,7 @@ def test_19_addresource(self): str(len(vm_ids)) + "). It must be 4")) all_configured = self.wait_inf_state( - self.inf_id, VirtualMachine.CONFIGURED, 2100) + self.inf_id, VirtualMachine.CONFIGURED, 2700) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") diff --git a/test/integration/TestREST_JSON.py b/test/integration/TestREST_JSON.py index 30b26124..fbbeddb4 100755 --- a/test/integration/TestREST_JSON.py +++ b/test/integration/TestREST_JSON.py @@ -131,6 +131,13 @@ def wait_inf_state(self, state, timeout, incorrect_states=None, vm_ids=None): return all_ok + def test_07_index(self): + resp = self.create_request("GET", "/") + self.assertEqual(resp.status_code, 200, + msg="ERROR getting IM index:" + resp.text) + res = json.loads(resp.text) + self.assertEqual(res['openapi'], '3.0.0') + def test_20_create(self): radl = read_file_as_string('../files/test_simple.json') resp = self.create_request("POST", "/infrastructures", diff --git a/test/unit/REST.py b/test/unit/REST.py index 48f7a168..56b24ba4 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -568,6 +568,11 @@ def test_GeVersion(self): res = self.client.get('/version') self.assertEqual(res.text, version) + def test_Index(self): + res = self.client.get('/') + self.assertEqual(res.json['openapi'], '3.0.0') + self.assertEqual(res.json['servers'][0]['url'], 'http://localhost/') + @patch("IM.InfrastructureManager.InfrastructureManager.CreateDiskSnapshot") def test_CreateDiskSnapshot(self, CreateDiskSnapshot): """Test REST StopVM."""