plugin.yaml

########
# Copyright (c) 2019 Atos Spain SA. All rights reserved.
#
# This file is part of Croupier.
#
# Croupier is free software: you can redistribute it and/or modify it
# under the terms of the Apache License, Version 2.0 (the License) License.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT ANY WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT, IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# See README file for full disclaimer information and LICENSE file for full
# license information in the project root.
#
# @author: Javier Carnero
#          Atos Research & Innovation, Atos Spain S.A.
#          e-mail: javier.carnero@atos.net
# @author: Jesus Gorronogoitia
#          Atos Research & Innovation, Atos Spain S.A.
#          e-mail: jesus.gorronogoitia@atos.net
# @author: Jesus Ramos Rivas
#          Atos Research & Innovation, Atos Spain S.A.
#          e-mail: jesus.2.ramos@atos.net
#
# plugin.yaml

plugins:
    croupier:
        executor: central_deployment_agent
        # URL to archive containing the plugin or name of directory containing
        # the plugin if it is included in the blueprint directory under the
        # "plugins" directory.
        source: https://github.com/ari-apc-lab/croupier/archive/master.zip
        install_arguments: -r plugin-requirements.txt
        package_name: croupier
        package_version: "3.3.0"

data_types:

    croupier.datatypes.wm_config:
        properties:
            infrastructure_interface:
                description: Scheduler type.
                type: string
                required: true
            country_tz:
                description: Timezone
                type: string
                required: false
                default: 'GMT'
            reservation_deletion_path:
                description: Path where the executable to delete a reservation is located
                type: string
                required: false
                default: '/opt/cesga/sistemas/reservas/sresdelete'


    croupier.datatypes.monitoring_options:
        properties:
            monitor_period:
                description: How often to scrape the WM for metrics
                type: integer
                required: false
                default: 15
            deployment_label:
                description: Deployment label to apply to the metrics to identify the deployment.
                type: string
                required: false
            hpc_label:
                description: HPC label to apply to the metrics to identify the HPC WM
                type: string
                required: false
            only_jobs:
                description: Set to true to not monitor the infrastructure metrics (queues/partitions)
                type: boolean
                required: false
                default: false

    croupier.datatypes.accounting_options:
        properties:
            reporting_user:
                type: string
                required: false

    croupier.datatypes.job_deployment_config:
        properties:
            inputs:
                default: []
                required: false
            bootstrap:
                default: ''
                required: false
                type: string
            revert:
                default: ''
                required: false
                type: string

    croupier.datatypes.dataset:
        properties:
            config:
                type: croupier.datatypes.dataset_config
                required: true
            dataset:
                type: croupier.datatypes.dataset_dataset
                required: true
            type:
                type: string
                required: true

    croupier.datatypes.dataset_config:
        properties:
            api_key:
              type: string

    croupier.datatypes.dataset_dataset:
        properties:
            name:
                type: string

    croupier.datatypes.ecmwf_query:
        properties:
            area:
                type: string
                required: true
            params:
                type: string
                required: true
            date:
                type: string
                required: false
                default: ""
            time:
                type: string
                required: false
                default: ""
            collection:
                type: string
                required: false
                default: ""
            members:
                type: string
                required: false
                default: ""
            ensemble:
                type: string
                required: false
                default: ""
            input:
                type: string
                required: false
                default: ""
            keep_input:
                type: string
                required: false
                default: ""
            algorithm:
                type: string
                required: false
                default: ""
            max_step:
                type: string
                required: false
                default: "1"

    croupier.datatypes.job.Options:
        properties:
            commands:
                description: 'List of commands to be included in job submission script. It is incompatible with script property'
                type: list
                required: false #Either commands or script must be present
            remote_script:
                description: 'Script, present in the HPC workdir, to be launched with job submission. It is incompatible with commands/local_script property'
                type: string
                required: false #Either commands or script must be present
            local_script:
                description: 'Script to be uploaded from the blueprint folder and launched with job submission. It is incompatible with commands/remote_script property'
                type: string
                required: false #Either commands or script must be present
            arguments:
                description: 'List of arguments to be sent to the remote_script or local_script'
                type: list
                required: false
            nodes:
                description: '(SLURM|TORQUE) Controls the minimum/maximum number of nodes allocated to the job'
                type: integer
                required: false
            tasks:
                description: '(SLURM) Controls the number of tasks to be created for the job'
                type: integer
                required: false
            tasks_per_node:
                description: '(SLURM) Controls the maximum number of tasks per allocated node. (TORQUE) Processors per node'
                type: integer
                required: false
            max_time:
                description: 'Walltime (TORQUE) or Maxtime (SLURM) for job'
                type: string
                required: false
            partition:
                description: '(SLURM) partition where to submit the job to'
                type: string
                required: false
            queue:
                description: '(TORQUE) partition where to submit the job to'
                type: string
                required: false
            scale:
                description: 'Specifies the task ids of a job array TORQUE (-t 0-XX), SLURM (--array=0-XX)'
                type: integer
                required: false
            memory:
                description: 'Maximum amount of physical memory used by the job TORQUE (-l mem=XX), SLURM (--mem=XX)'
                type: integer
                required: false
            mail_user:
                description: 'Declares the list of users to whom mail is sent by the execution server when it sends mail about the job. TORQUE (-M XX), SLURM (--mail-user=XX)'
                type: string
                required: false
            mail_type:
                description: 'Defines the set of conditions under which the execution server will send a mail message about the job. TORQUE (-m XX), SLURM (--mail-type=XX)'
                type: string
                required: false
            account:
                description: 'Defines the account string associated with the job TORQUE (-A XX), SLURM (--A XX)'
                type: string
                required: false
            stderr_file:
                description: 'Defines the path to be used for the standard error stream of the batch job TORQUE|SLURM (-e XX)'
                type: string
                required: false
            stdout_file:
                description: 'Defines the path to be used for the standard output stream of the batch job TORQUE|SLURM (-o XX)'
                type: string
                required: false
            group_name:
                description: 'FIXME: Not used, but read by TORQUE'
                type: string
                required: false
            scale_max_in_parallel:
                description: ''
                type: integer
                required: false
            reservation:
                description: 'To launch a job in the context of a reservation SLURM (--reservation=XX)'
                type: string
                required: false
            qos:
                description: 'The quality of service associated with a job SLURM (--qos=XX)'
                type: string
                required: false

# DATA MANAGEMENT

    croupier.datatypes.Credentials:
        properties:
            host:
                description: 'Host for which the credentials are for'
                type: string
                required: false
                default: ''
            user:
                description: 'Account username'
                type: string
                required: false
                default: ''
            password:
                description: 'Account password'
                type: string
                required: false
                default: ''
            auth-header:
                type: string
                required: false
                default: ''
                description: 'Auth header for http calls, for example API tokens'
            auth-header-label:
                type: string
                required: false
                default: 'Authorization'
                description: 'Label to use for authenticating calls to the API'
            private_key:
                description: 'User private key'
                type: string
                required: false
                default: ''
            private_key_password:
                description: 'Password for User private key'
                type: string
                required: false
                default: ''
            ssh_port:
                type: integer
                required: false
                default: 22
            ssh_tunnel:
                type: string
                required: false
                default: ''
            ssh_login_shell:
                type: string
                required: false
                default: ''


    croupier.datatypes.dm.GridFTPCredentials:
        derived_from: croupier.datatypes.Credentials
        properties:
            cert:
                description: "GridFTP user's certificate"
                type: string
                required: false
                default: ''
            cert_password:
                description: "Password for GripFTP user's certificate"
                type: string
                required: false
                default: ''

    croupier.datatypes.dm.HPCWorkspace:
        properties:
            name:
                description: 'workspace name'
                type: string
                required: true
            create:
                description: 'whether or not to create a workspace in the HPC where to copy data'
                type: boolean
                required: false
                default: false
            lifetime:
                description: 'defines the number of days between the creation of the workspace until its automatic deletion.'
                type: integer
                required: false
                default: 1

    croupier.datatypes.ckan_dataset:
        properties:
            name:
                type: string
                required: false
                default: ""
            package_id:
                type: string
                required: false
                default: ""
            title:
                type: string
                required: false
                default: ""
            private:
                type: boolean
                required: false
                default: true
            author:
                type: string
                required: false
                default: ""
            author_email:
                type: string
                required: false
                default: ""
            maintainer:
                type: string
                required: false
                default: ""
            maintainer_email:
                type: string
                required: false
                default: ""
            license_id:
                type: string
                required: false
                default: ""
            notes:
                type: string
                required: false
                default: ""
            version:
                type: string
                required: false
                default: ""
            tags:
                type: list
                required: false
                default: []
            owner_org:
                type: string
                required: false
                default: ""

    croupier.datatypes.ckan_resource:
        properties:
            name:
                type: string
                required: false
                default: ""
            url:
                type: string
                required: false
                default: ""
            description:
                type: string
                required: false
                default: ""
            type:
                type: string
                required: false
                default: ""
            format:
                type: string
                required: false
                default: ""

workflows:
    run_jobs:
        mapping: croupier.croupier_plugin.workflows.run_jobs
    croupier_install:
        mapping: croupier.croupier_plugin.workflows.croupier_install
    croupier_configure:
        mapping: croupier.croupier_plugin.workflows.croupier_configure

node_types:
    croupier.nodes.InfrastructureInterface:
        derived_from: cloudify.nodes.Compute
        properties:
            config:
                description: type
                type: croupier.datatypes.wm_config
            credentials:
                description: SSH configuration
                type: croupier.datatypes.Credentials
                default: {}
            job_prefix:
                description: Job name prefix for this HPC
                default: "croupier"
                type: string
            base_dir:
                description: Root directory of all executions
                default: "$HOME"
                type: string
            workdir_prefix:
                description: Prefix of the working directory instead of blueprint name
                default: ""
                type: string
            monitoring_options:
                description: Monitoring options
                default: {}
                type: croupier.datatypes.monitoring_options
            accounting_options:
                description: Accounting options
                default: {}
                type: croupier.datatypes.accounting_options
            simulate:
                description: Set to true to simulate job without sending it
                type: boolean
                default: False
            skip_cleanup:
                description: True to not clean all files at deployment removal
                type: boolean
                default: False
            recurring_workflow:
                description: True if the workflow is scheduled to run recurrently
                type : boolean
                default: False
                required: False
            internet_access:
                description: True if HPC has access to the internet
                type: boolean
                default: False
                required: False
            supported_prtocols:
                type: list
                default: []
                description: List of protocols supported for data management
            agent_config:
                default:
                    install_method: none
        interfaces:
            cloudify.interfaces.lifecycle:
                configure:
                    implementation: croupier.croupier_plugin.tasks.configure_execution
                    inputs:
                        config:
                            default: { get_property: [SELF, config] }
                        credentials:
                            default: { get_property: [SELF, credentials] }
                        base_dir:
                            default: { get_property: [SELF, base_dir] }
                        workdir_prefix:
                            default: { get_property: [SELF, workdir_prefix] }
                        monitoring_options:
                            default: { get_property: [SELF, monitoring_options] }
                        accounting_options:
                            default: { get_property: [SELF, accounting_options] }
                        recurring_workflow:
                            default: { get_property: [SELF, recurring_workflow] }
                        simulate:
                            default: { get_property: [SELF, simulate] }
                delete:
                    implementation: croupier.croupier_plugin.tasks.cleanup_execution
                    inputs:
                        config:
                            default: { get_property: [SELF, config] }
                        credentials:
                            default: { get_property: [SELF, credentials] }
                        skip:
                            default: { get_property: [SELF, skip_cleanup] }
                        simulate:
                            default: { get_property: [SELF, simulate] }
            cloudify.interfaces.monitoring:
                start:
                    implementation: croupier.croupier_plugin.tasks.start_monitoring_hpc
                    inputs:
                        config_infra:
                            default: { get_property: [SELF, config] }
                        credentials:
                            default: { get_property: [SELF, credentials] }
                        monitoring_options:
                            default: { fet_property: [SELF, monitoring_options]}
                        simulate:
                            default: { get_property: [SELF, simulate] }
                stop:
                    implementation: croupier.croupier_plugin.tasks.stop_monitoring_hpc
                    inputs:
                        credentials:
                            default: { get_property: [SELF, credentials] }
                        simulate:
                            default: { get_property: [SELF, simulate] }

    croupier.nodes.Job:
        derived_from: cloudify.nodes.Root
        properties:
            deployment:
                description: Deployment script and inputs
                default: {}
                type: croupier.datatypes.job_deployment_config
            job_options:
                description: Job main command and options
                type: croupier.datatypes.job.Options
            skip_cleanup:
                description: True to not clean after execution (debug purposes)
                type: boolean
                default: False
        interfaces:
            cloudify.interfaces.lifecycle:
                configure:
                    implementation: croupier.croupier_plugin.tasks.configure_job
                    inputs:
                        deployment:
                            description: Deployment scripts and inputs
                            default: { get_property: [SELF, deployment] }
                        skip_cleanup:
                            default: { get_property: [SELF, skip_cleanup] }
                        job_options:
                            default: { get_property: [SELF, job_options] }
                    max_retries: 0
                stop:
                    implementation: croupier.croupier_plugin.tasks.revert_job
                    inputs:
                        deployment:
                            description: Undeployment script and inputs
                            default: { get_property: [SELF, deployment] }
                        skip_cleanup:
                            default: { get_property: [SELF, skip_cleanup] }
            croupier.interfaces.lifecycle:
                queue:
                    implementation: croupier.croupier_plugin.tasks.send_job
                    inputs:
                        job_options:
                            default: { get_property: [SELF, job_options] }
                    max_retries: 0
                publish:
                    implementation: croupier.croupier_plugin.tasks.publish
                cleanup:
                    implementation: croupier.croupier_plugin.tasks.cleanup_job
                    inputs:
                        job_options:
                            default: { get_property: [SELF, job_options] }
                        skip:
                            default: { get_property: [SELF, skip_cleanup] }
                cancel:
                    implementation: croupier.croupier_plugin.tasks.stop_job
                    inputs:
                        job_options:
                            default: { get_property: [SELF, job_options] }
                delete_reservation:
                    implementation: croupier.croupier_plugin.tasks.delete_reservation

    croupier.nodes.SingularityJob:
        derived_from: croupier.nodes.Job

# DATA MANAGEMENT
    croupier.nodes.DataAccessInfrastructure:
        derived_from: cloudify.nodes.Root
        properties:
            endpoint:
                description: 'accessible (via protocol) endpoint of infrastructure'
                type: string
            supported_protocols:
                description: 'list of supported interfaces to access the data'
                type: list
            credentials:
                description: 'Credentials to access the data access infrastructure'
                type: croupier.datatypes.Credentials
                required: false
            internet_access:
                description: 'Whether of not this DM infrastructure has external access to Internet'
                type: boolean
                required: false
                default: false

    croupier.nodes.GridFTPDataAccessInfrastructure:
        derived_from: croupier.nodes.DataAccessInfrastructure
        properties:
            workspace_basepath:
                description: 'basepath location for workspace creating'
                type: string
                required: false
            ssh_endpoint:
                description: 'ssh endpoint for workspace management'
                type: string
                required: false
            credentials:
                description: 'Credentials to access the data access infrastructure'
                type: croupier.datatypes.dm.GridFTPCredentials
                required: false

    croupier.nodes.CKAN_dataset:
        derived_from: croupier.nodes.DataAccessInfrastructure
        properties:
            dataset_info:
                type: croupier.datatypes.ckan_dataset
                required: true
            supported_protocols:
                default:
                    - CKANAPI

    croupier.nodes.DataTransfer:
        derived_from: cloudify.nodes.Root
        properties:
            transfer_protocol:
                description: 'protocol to transfer data from source to target'
                type: string
        interfaces:
            cloudify.interfaces.lifecycle:
                configure:
                    implementation: croupier.croupier_plugin.tasks.configure_data_transfer

    croupier.nodes.DataSource:
        derived_from: cloudify.nodes.Root

    croupier.nodes.FileDataSource:
        derived_from: croupier.nodes.DataSource
        properties:
            filepath:
                description: 'location of data source in filesystem'
                type: string

    croupier.nodes.WebDataSource:
        derived_from: croupier.nodes.DataSource
        properties:
            resource:
                description: 'URI of data source in Web'
                type: string

    croupier.nodes.CKAN_resource:
        derived_from: croupier.nodes.DataSource
        properties:
            resource:
                type: croupier.datatypes.ckan_resource
                required: true

    croupier.nodes.ECMWFVerticalInterpolation:
        derived_from: croupier.nodes.WebDataSource
        properties:
            credentials:
                description: SSH credentials for the ECMWF VM. Fields- host, user, private_key
                type: croupier.datatypes.Credentials
            keycloak_credentials:
                description: login credentials for hidalgo keycloak. Fields- user, pw
                type: croupier.datatypes.Credentials
            query:
                description: ECMWF Query fields.
                type: croupier.datatypes.ecmwf_query
            server_port:
                type: string
                default: 9984
            cloudify_address:
                type: string
                default: ''
                required: False
            resource:
                required: false
                default: ''
        interfaces:
            cloudify.interfaces.lifecycle:
                configure:
                    implementation: croupier.croupier_plugin.tasks.ecmwf_vertical_interpolation
                    max_retries: 0
                    inputs:
                        query:
                            default: { get_property: [ SELF, query ] }
                        credentials:
                            default: { get_property: [ SELF, credentials ] }
                        keycloak_credentials:
                            default: { get_property: [ SELF, keycloak_credentials ] }
                        cloudify_address:
                            default: { get_property: [ SELF, cloudify_address ] }
                        server_port:
                            default: { get_property: [ SELF, server_port ] }



    croupier.nodes.HPCDataSource:
        derived_from: croupier.nodes.FileDataSource
        properties:
            workspace:
                description: 'Workspace where FileDataSource is located'
                type: croupier.datatypes.dm.HPCWorkspace
                required: false

#VAULT
    croupier.nodes.Vault:
        derived_from: cloudify.nodes.Root
        properties:
            token:
                type: string
                required: true
            user:
                type: string
                required: false
                default: ''
            address:
                type: string
                required: false
                default: ''
            cubbyhole:
                type: boolean
                required: false
                default: false
        interfaces:
            cloudify.interfaces.lifecycle:
                create:
                    implementation: croupier.croupier_plugin.tasks.create_vault
                    inputs:
                        address:
                            default: { get_property: [ SELF,address ] }

    croupier.nodes.HPCExporter:
        derived_from: cloudify.nodes.Root
        properties:
            address:
                type: string
                required: false
                default: ''
        interfaces:
            cloudify.interfaces.lifecycle:
                create:
                    implementation: croupier.croupier_plugin.tasks.create_monitor
                    inputs:
                        address:
                            default: { get_property: [SELF,address] }

relationships:
    interface_contained_in:
        derived_from: cloudify.relationships.contained_in
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                preconfigure:
                    implementation: croupier.croupier_plugin.tasks.preconfigure_interface
                    inputs:
                        credentials:
                            default: { get_property: [SOURCE, credentials] }
                        simulate:
                            default: { get_property: [SOURCE, simulate] }

    interface_monitored_by:
        derived_from: cloudify.relationships.depends_on
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                preconfigure:
                    implementation: croupier.croupier_plugin.tasks.preconfigure_interface_monitor

    task_managed_by_interface:
        derived_from: cloudify.relationships.contained_in
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                preconfigure:
                    implementation: croupier.croupier_plugin.tasks.preconfigure_task
                    inputs:
                        config:
                            default: { get_property: [TARGET, config] }
                        credentials:
                            default: { get_property: [TARGET, credentials] }
                        job_prefix:
                            default: { get_property: [TARGET, job_prefix] }
                        monitoring_options:
                            default: { get_property: [TARGET, monitoring_options] }
                        simulate:
                            default: { get_property: [TARGET, simulate] }

    ds_located_at:
        derived_from: cloudify.relationships.contained_in
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                preconfigure:
                    implementation: croupier.croupier_plugin.tasks.configure_data_source


    job_depends_on:
        derived_from: cloudify.relationships.depends_on

    from_source:
        derived_from: cloudify.relationships.connected_to
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                postconfigure:
                    implementation: croupier.croupier_plugin.tasks.configure_dt_ds_relationship

    to_target:
        derived_from: cloudify.relationships.connected_to
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                postconfigure:
                    implementation: croupier.croupier_plugin.tasks.configure_dt_ds_relationship


    input:
        derived_from: cloudify.relationships.connected_to

    output:
        derived_from: cloudify.relationships.connected_to

    retrieve_credentials_from_vault:
        derived_from: cloudify.relationships.depends_on
        source_interfaces:
            cloudify.interfaces.relationship_lifecycle:
                preconfigure:
                    implementation: croupier.croupier_plugin.tasks.download_vault_credentials
                    inputs:
                        token:
                            default: { get_property: [TARGET, token  ] }
                        user:
                            default: { get_property: [TARGET, user   ] }
                        cubbyhole:
                            default: { get_property: [TARGET, cubbyhole ] }