From 7dff729d1e589157eabcd48697e52b50e70a9368 Mon Sep 17 00:00:00 2001 From: Mark Ryan Date: Fri, 20 Oct 2017 16:40:29 +0000 Subject: [PATCH] ciao-down: Remove ciao-down ciao-down is now known as Configurable Cloud VM and can be found in its new repo: https://github.com/intel/ccloudvm Signed-off-by: Mark Ryan --- .gitignore | 1 - k8s/kubicle/create.go | 1 - testutil/ciao-down/README.md | 520 +--------------- testutil/ciao-down/ciao_down.go | 584 ------------------ testutil/ciao-down/deps.go | 48 -- testutil/ciao-down/download.go | 182 ------ testutil/ciao-down/host.go | 22 - testutil/ciao-down/host_linux.go | 47 -- testutil/ciao-down/instance.go | 295 --------- testutil/ciao-down/mock_test.go | 91 --- testutil/ciao-down/prepare.go | 371 ----------- testutil/ciao-down/vm.go | 287 --------- testutil/ciao-down/workload.go | 296 --------- testutil/ciao-down/workload_test.go | 84 --- .../ciao-down/workloads/ciao-fedora25.yaml | 206 ------ testutil/ciao-down/workloads/ciao.yaml | 207 ------- .../workloads/clearcontainers-2.x.yaml | 199 ------ .../workloads/clearcontainers-3.x.yaml | 249 -------- testutil/ciao-down/workloads/fedora25.yaml | 57 -- testutil/ciao-down/workloads/xenial.yaml | 62 -- testutil/singlevm/setup.sh | 4 +- 21 files changed, 3 insertions(+), 3810 deletions(-) delete mode 100644 testutil/ciao-down/ciao_down.go delete mode 100644 testutil/ciao-down/deps.go delete mode 100644 testutil/ciao-down/download.go delete mode 100644 testutil/ciao-down/host.go delete mode 100644 testutil/ciao-down/host_linux.go delete mode 100644 testutil/ciao-down/instance.go delete mode 100644 testutil/ciao-down/mock_test.go delete mode 100644 testutil/ciao-down/prepare.go delete mode 100644 testutil/ciao-down/vm.go delete mode 100644 testutil/ciao-down/workload.go delete mode 100644 testutil/ciao-down/workload_test.go delete mode 100644 testutil/ciao-down/workloads/ciao-fedora25.yaml delete mode 100644 testutil/ciao-down/workloads/ciao.yaml delete mode 100644 testutil/ciao-down/workloads/clearcontainers-2.x.yaml delete mode 100644 testutil/ciao-down/workloads/clearcontainers-3.x.yaml delete mode 100644 testutil/ciao-down/workloads/fedora25.yaml delete mode 100644 testutil/ciao-down/workloads/xenial.yaml diff --git a/.gitignore b/.gitignore index b793f4c15..304a8ffef 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,3 @@ networking/libsnnet/tests/cncicli/cncicli networking/libsnnet/tests/cncli/cncli networking/libsnnet/tests/snnetcli/snnetcli test-cases/test-cases -testutil/ciao-down/ciao-down diff --git a/k8s/kubicle/create.go b/k8s/kubicle/create.go index 3a9f8f251..19647d6b3 100644 --- a/k8s/kubicle/create.go +++ b/k8s/kubicle/create.go @@ -87,7 +87,6 @@ func init() { workloadTmpl = template.Must(template.New("workloadTmpl").Parse(workloadTemplate)) } -// TODO: Code copied from ciao-down needs to be refactored func getProxy(upper, lower string) (string, error) { proxy := os.Getenv(upper) if proxy == "" { diff --git a/testutil/ciao-down/README.md b/testutil/ciao-down/README.md index fc8fd35ca..e8064f08e 100644 --- a/testutil/ciao-down/README.md +++ b/testutil/ciao-down/README.md @@ -1,519 +1 @@ -# ciao-down - -## Introduction - -ciao-down is a small utility that simplifies the task of creating, -managing and connecting to virtual machines. All you need to have -installed on your machine is: - -- Go 1.8 or greater - -Then simply type - -``` -go get github.com/ciao-project/ciao/testutil/ciao-down -$GOPATH/bin/ciao-down create xenial -``` - -to create a new Ubuntu 16.04 VM. ciao-down will install some needed -dependencies on your local PC such as qemu and xorriso. It will then -download an Ubuntu Cloud Image and create a VM based on this image. -It will boot the VM, create an account for you, with the same -name as your account on your host, and optionally update the default -packages. - -Once it's finished you'll be able to connect to the the VM via SSH -using the following command. - -``` -ciao-down connect -``` - -The command above assumes that either $GOPATH/bin or ~/go/bin is in -your PATH. - -ciao-down will cache the Ubuntu image locally so the next time you -create another xenial based VM you won't have to wait very long. -It also knows about HTTP proxies and will mirror your host computer's proxy -settings inside the VMs it creates. - -You can delete the VM you've just created by running, - -``` -ciao-down delete -``` - -## Workloads - -Each VM is created from a workload. A workload is a text file which contains -a set of instructions for initialising the VM. A workload, among other things, -can specify: - -- The hostname of the VM. -- The resources to be consumed by the VM -- The base image from which the VM is to be created -- The folders that should be shared between the host and the VM -- Any file backed storage that should appear as a device in the VM -- An annotated cloud-init file that contains the set of instructions - to run on the first boot of the VM. This file is used to create - user accounts, install packages and configure the VM. - -ciao-down ships with a number of workloads for creating VMs based on standard images, -such as Ubuntu 16.04 and Fedora 25. Users are also free to create their own workloads. -Standard workloads are stored in $GOPATH/src/github.com/ciao-project/ciao/testutil/ciao-down/workloads. -User created workloads are stored in ~/.ciao-down/workloads. ciao-down always checks the -~/.ciao-down/workloads directory first so if a workload exists in both directories -with the same name, ciao-down will use the workload in ~/.ciao-down/workloads. - -Which workload to use is specified when creating the VM with the create -command. The workload can be a file without the .yaml extension or a URI pointing to -a local or remote file. If the workload is a file without the .yaml extension then it -must be present in either of the two directories mentioned above. For example, -the create command in the introduction section creates an Ubuntu 16.04 -workload (xenial). This caused ciao-down to load the workload definition in -$GOPATH/src/github.com/ciao-project/ciao/testutil/ciao-down/workloads/xenial.yaml. -In the case of a remote file the supported schemes are http, https and file. Be aware -of the remote file will not be saved hence each time this option is used, ciao-down -will try to get it from the remote location. For example, to create a workload using the -file /home/x/workload.yaml we have two options. - -Using the file scheme: - -``` -ciao-down create file:///home/ciao-down/workload.yaml -``` - -or its absolute path: - -``` -ciao-down create /home/ciao-down/workload.yaml -``` - - -As the majority of the ciao-down workloads are cloud-init files, ciao-down only -works with images that are designed to run cloud-init on their first boot. Typically, -these are the cloud images, e.g.,. the [Ubuntu Cloud Images](https://cloud-images.ubuntu.com/). - -## Creating new Workloads - -ciao-down workloads are multi-doc YAML files. The first document contains invariants -about the VM created from the workload such as the image off which it is based and its -hostname. The second contains dynamic instance data that can be altered on every boot, -such as the number of CPUs the VM uses and the ports mapped. The final document contains -the cloud-init file. If only one section is present it is assumed to the be cloud-init -file and default values are used for the other two sections. - -An example workload is shown below: - -``` ---- -base_image_url: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img -base_image_name: Ubuntu 16.04 -... ---- -mem_gib: 2 -cpus: 2 -... ---- -#cloud-config -package_upgrade: false -runcmd: - - {{finished .}} - -users: - - name: {{.User}} - gecos: CIAO Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} -... -``` - -### Templates - -[Go templates](https://golang.org/pkg/text/template/) can be used in each -document in a workload yaml file. These templates have access to an instance -of a structure called Workspace that contains some useful information about the -host's environment and the instance that is being created. The following pieces -of information are available via the workload structure. - - - GoPath : The GoPath on the host or ~/go if GOPATH is not defined - - Home : The home directory of the user who ran ciao-down - - HTTPProxy : The hosts HTTP proxy if set - - HTTPSProxy : The hosts HTTPS proxy if set - - NoProxy : The value of the host's no_proxy variable - - User : The user who invoked ciao-down - - PublicKey : The public key that will be used to access the instance over SSH - - GitUserName : The git user name of the user who ran ciao-down - - GitEmail : The git email address of the user who ran ciao-down - - Mounts : A slice of mounts, each describing a path to be shared between guest and host - - Hostname : The instance hostname - - UUID : A UUID for the new instance - - PackageUpgrade : Indicates whether package upgrade should be performed during the first boot. - -As an example consider the third document in the workload definition above. The User and -PublicKey fields are accessed via the {{.User}} and {{.PublicKey}} Go templates. Abstracting -all of the user and instance specific information via templates in this way allows us to keep our -workload definitions generic. - -### The Instance Specification Document - -The first yaml document is called the instance specification document. It defines the fixed -characteristics of instances created from the workload which cannot be altered. Three -fields are currently defined: - -- base_image_url : The URL of the qcow2 image upon which instances of the workload should be based. -- base_image_name : Friendly name for the base image. This is optional. -- hostname : The hostname of instances created from this workload. Hostname is also optional and defaults to singlevm if not provided. - -### The Instance Data Document - -The second document is called the instance data document. It contains information that is -set when an instance is first created but may be modified at a later date, without deleting -the instance. Four fields are currently defined: - -- mem_gib : Number of Gigabytes to assign to the VM. Defaults to half the memory on your machine -- cpus : Number of CPUs to assign to the VM. Defaults to half the cores on your machine -- ports : Slice of port objects which map host ports on 127.0.0.1 to guest ports -- mounts : Slice of mount objects which describe the folders shared between the host and the guest -- drives : Slice of drive objects which identify resources accessible on the host to be made available as block devices on the guest. - -Each port object has two members, host and guest. They are both integers and they specify -the mapping of port numbers from host to guest. A default mapping of 10022 to 22 is always -added so that users can connect to their VMs via ciao-down connect. An example port -mapping is shown below: - -``` -ports: -- host: 10022 - guest: 22 -``` - -Mount objects can be used to share folders between the host and guest. Folders are shared -using the 9p protocol. Each mount object has three pieces of information. - -- tag : An id for the mount. This information is needed when mounting the shared folder in the guest -- security_model : The 9p security model to use -- path : The path of the host folder to share - -An example of a mount is given below. - -``` -mounts: -- tag: hostgo - security_model: passthrough - path: /home/user/go -``` - -Note that specifying a mount in the instance data document only creates a -9p device which is visible inside the guest. To actually access the shared -folder from inside the guest you need to mount the folder. This can be -done in the cloud-init file discussed below. - -Drive objects allow the user to make a resource accessible from the host -available as a block device in the guest VM. Currently, these resources -are restricted to being file backed storage located on the host. Each -drive object has three pieces of information. - -- path : The path of the resource as accessed from the host -- format : The format of the resource, e.g., qcow2 -- options : A comma separate list of options. This field is optional. - -An example of a drive is given below. - -``` - drives: - - path: /tmp/img.qcow2 - format: qcow2 - options: aio=native -``` - - -### The Cloudinit document - -The third document contains a cloud-init user data file that can be used -to create users, mount folders, install and update packages and perform -general configuration tasks on the newly created VM. The user is -expected to be familiar with the -[cloudinit file format](https://cloudinit.readthedocs.io/en/latest/). - -Like the instance specification and data documents, the cloudinit document -is processed by the Go template engine before being passed to cloudinit -running inside the guest VM. The template engine makes a number of -functions available to the cloudinit document. These functions are used -to perform common tasks such as configuring proxies and communicating with -ciao-down running on the host. Each function must be passed the -workspace object as the first parameter. The following functions are -available: - -#### proxyVars - -Generates a string containing variable definitions for all the proxy -variables. It is useful for prepending to commands inside the runcmd: -section of the cloudinit document that require proxies to be set, e.g, - -``` -- {{proxyvars .}} wget https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz" "/tmp/go1.8.linux-amd64.tar.gz -``` - -may expand to something like after being processed by the template engine - -``` -- https_proxy=http://myproxy.mydomain.com:911 wget https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz" "/tmp/go1.8.linux-amd64.tar.gz -``` - -#### proxyEnv - -Generates proxy definitions in a format that is suitable for the /etc/environment file. -It should be used as follows: - -``` -write_files: -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end}} -``` - -#### download - -Download should be used to download files from inside the guest. Download is preferable -to directly downloading files inside the cloudinit document using curl or wget for -example, as the downloaded files are cached by ciao-down. This means that the next time -an instance of this workload is created the file will not have to be retrieved from -the Internet, resulting in quicker boot times for the new VM. - -download takes three parameters - -- The workspace object -- The URL of the file to download -- The location on the guest where the downloaded file should be stored - -An example of its usage is - -``` -- {{download . "https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz" "/tmp/go1.8.linux-amd64.tar.gz"}} -``` - -#### The task functions - -There are four functions that can be used to provide visual information back to the user -who invoked ciao-down on the host. These functions should always be issued in pairs. -The first function invoked should always be beginTask. - -beginTask takes two parameters. The first is the workspace object. The second is a -string to present to the user. beginTask is typically followed by a command and -then a call to either endTaskOk, endTaskFail or endTaskCheck. - -endTaskOk prints "[OK]" to the screen, endTaskFail prints "[FAIL]" to the screen and -endTaskCheck prints "[OK]" or "[FAIL]" depending on the result of the previous command. - -For example, - -``` - - {{beginTask . "Unpacking Go" }} - - tar -C /usr/local -xzf /tmp/go1.8.linux-amd64.tar.gz - - {{endTaskCheck .}} -``` - -will print - -Unpacking Go : [OK] - -if the tar command succeeds - -and - -Unpacking Go : [FAIL] - -if it fails. - -Reporting a failure back to ciao-down does not cause the create command to exit. The -failure is presented to the user and the setup of the VM continues. - -#### Finished - -Every cloudinit document must contain a runcmd: section and that section must contain -a call to the finished function. The function takes one parameter, the workspace -parameter. A call to - -``` - - {{finished .}} -``` - -is usually the last command in the runcmd section. Remember it's currently mandatory. - -### Automatically mounting shared folders - -As previously mentioned mounts specified in the instance data document will only -create 9p devices in the guest. In order to access the files in the shared folder -you need to arrange to have these devices mounted. The way you do this might -differ from distro to distro. On Ubuntu it is done by adding a mount: section -to the cloudinit document, e.g., - -``` -mounts: -{{range .Mounts}} - [{{.Tag}}, {{.Path}}, 9p, "x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L", "0", "0"] -{{end -}} -``` - -The above command will arrange for all mounts specified in the instance data document or on -the create command line to be mounted to the same location that they are mounted on the -host. - -Mounts added later via the start command will need to be mounted manually. - -## Commands - -### create - -ciao-down create creates and configures a new ciao-down VM. Currently, -only one VM can be configured at any one time. All the files associated -with the VM are stored in ~/.ciao-down. - -An example of ciao-down create is given below: - -``` -$ ./ciao-down create xenial -Booting VM with 7 GB RAM and 4 cpus -Booting VM : [OK] -Downloading Go : [OK] -Unpacking Go : [OK] -Installing apt-transport-https and ca-certificates : [OK] -Add docker GPG key : [OK] -Retrieving updated list of packages : [OK] -Upgrading : [OK] -Installing Docker : [OK] -Installing GCC : [OK] -Installing Make : [OK] -Installing QEMU : [OK] -Installing xorriso : [OK] -Installing ceph-common : [OK] -Installing Openstack client : [OK] -Auto removing unused components : [OK] -Building ciao : [OK] -Installing Go development utils : [OK] -Pulling ceph/demo : [OK] -Downloading Fedora-Cloud-Base-24-1.2.x86_64.qcow2 : [OK] -Downloading CNCI image : [OK] -Downloading latest clear cloud image : [OK] -Setting git user.name : [OK] -Setting git user.email : [OK] -VM successfully created! -Type ciao-down connect to start using it. -``` - -By default, ciao-down will assign half of your host's resources to the VM -that it creates and launches. If you have 8 CPUs and 8 GB of RAM on your -host, ciao-down will assign 4GB of RAM and 4 VCPUs to the guest VM. You -can control this behaviour by using the --mem and --cpu options. For -example, - -ciao-down create --cpus 2 -mem 2 ciao - -Creates and boots a VM with 2 VCPUs and 2 GB of RAM. - -The --package-upgrade option can be used to provide a hint to workloads -indicating whether packages contained within the base image should be updated or not -during the first boot. Updating packages can be quite time consuming -and may not be necessary if the user just wants to create a throw away -VM to test something under a particular distro. This hint only works if -the workloads define the following in their cloudinit documents. - -``` -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}false{{end}} -``` - -#### Port mappings, Mounts and Drives - -By default, ciao-down creates one port mapping for new VMs, 10022-22 for SSH -access. You can specify additional port mappings, mounts or drives or even override -the default settings on the command line. - -For example, - -./ciao-down create --mount docs,passthrough,$HOME/Documents --port 10000-80 xenial - -shares the host directory, $HOME/Documents, with the ciao-down VM using the 9p -passthrough security model. The directory can be mounted inside the VM using -the docs tag. The command also adds a new port mapping. 127.0.0.1:10000 on -the host now maps to port 80 on the guest. - -New file backed storage devices can be added to the guest using the -drive option. --drive requires at least two parameters. The first is -the location of the file backed storage, e.g., the location on the -host of a qcow2 file. The second indicates the format of that storage. -A user can specify additional options which are passed straight -through to the underlying hypervisor. For example, let's suppose we -have a qcow2 file called $HOME/img.qcow2. We could make this file -accessible as a block device in our VM as follows. - -./ciao-down create --drive $HOME/img.qcow2,qcow2,aio=threads - -The drive will appear as a device, e.g., /dev/vdc in the VM. - -Multiple --mount, --drive and --port options can be provided and it's -also possible to override the existing values. Existing mounts, ones -specified in the instance data document, can be overridden by -specifying an existing tag with new options, existing ports can be -overridden by mapping a new host port to an existing guest port and -existing drives can be overridden by providing a new set of options for -an existing drive path. For example, - -./ciao-down create --mount hostgo,none,$HOME/go -port 10023-22 xenial - -changes the security model of the mount with the hostgo tag and makes the instance -available via ssh on 127.0.0.1:10023. - - -### delete - -ciao-down delete, shuts down and deletes all the files associated with the VM. - -### status - -ciao-down status provides information about the current ciao-down VM, e.g., whether -it is running, and how to connect to it. For example, - -``` -$ ciao-down status -Status : ciao up -SSH : ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/user/.ciao-down/id_rsa 127.0.0.1 -p 10022 -``` - -### stop - -ciao-down stop is used to power down the ciao-down VM cleanly. - -### start - -ciao-down start boots a previously created but not running ciao-down VM. -The start command also supports the --mem and --cpu options. So it's -possible to change the resources assigned to the guest VM by stopping it -and restarting it, specifying --mem and --cpu. It's also possible to -specify additional port mappings or mounts via the start command. See -the section on port mappings below. - -Any parameters you pass to the start command override the parameters -you originally passed to create. These settings are also persisted. -For example, if you were to run - -``` -./ciao-down create --mem=2 xenial -./ciao-down stop -./ciao-down start --mem=1 -./ciao-down stop -./ciao-down start -``` - -The final ciao-down instance would boot with 1GB of RAM even though no mem -parameter was provided. - -### quit - -ciao-down quit terminates the VM immediately. It does not shut down the OS -running in the VM cleanly. - +This directory has been moved to the [Configurable Cloud VM](https://github.com/intel/ccloudvm) repository. diff --git a/testutil/ciao-down/ciao_down.go b/testutil/ciao-down/ciao_down.go deleted file mode 100644 index faef54ec8..000000000 --- a/testutil/ciao-down/ciao_down.go +++ /dev/null @@ -1,584 +0,0 @@ -/* -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -*/ - -/* TODO - -5. Install kernel -12. Make most output from osprepare optional -*/ - -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "os" - "os/exec" - "os/signal" - "path" - "strconv" - "strings" - "syscall" - "time" -) - -// Different types of virtual development environments -// we support. -const ( - CIAO = "ciao" - CLEARCONTAINERS = "clearcontainers" -) - -func init() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "%s [create|start|stop|quit|status|connect|delete]\n\n", os.Args[0]) - fmt.Fprintln(os.Stderr, "- create : creates a new VM") - fmt.Fprintln(os.Stderr, "- start : boots a stopped VM") - fmt.Fprintln(os.Stderr, "- stop : cleanly powers down a running VM") - fmt.Fprintln(os.Stderr, "- quit : quits a running VM") - fmt.Fprintln(os.Stderr, "- status : prints status information about the ciao-down VM") - fmt.Fprintln(os.Stderr, "- connect : connects to the VM via SSH") - fmt.Fprintln(os.Stderr, "- delete : shuts down and deletes the VM") - } -} - -type mounts []mount - -func (m *mounts) String() string { - return fmt.Sprint(*m) -} - -func (m *mounts) Set(value string) error { - components := strings.Split(value, ",") - if len(components) != 3 { - return fmt.Errorf("--mount parameter should be of format tag,security_model,path") - } - *m = append(*m, mount{ - Tag: components[0], - SecurityModel: components[1], - Path: components[2], - }) - return nil -} - -type ports []portMapping - -func (p *ports) String() string { - return fmt.Sprint(*p) -} - -func (p *ports) Set(value string) error { - components := strings.Split(value, "-") - if len(components) != 2 { - return fmt.Errorf("--port parameter should be of format host-guest") - } - host, err := strconv.Atoi(components[0]) - if err != nil { - return fmt.Errorf("host port must be a number") - } - guest, err := strconv.Atoi(components[1]) - if err != nil { - return fmt.Errorf("guest port must be a number") - } - *p = append(*p, portMapping{ - Host: host, - Guest: guest, - }) - return nil -} - -type packageUpgrade string - -func (p *packageUpgrade) String() string { - return string(*p) -} - -func (p *packageUpgrade) Set(value string) error { - if value != "true" && value != "false" { - return fmt.Errorf("--package-update parameter should be true or false") - } - *p = packageUpgrade(value) - return nil -} - -type drives []drive - -func (d *drives) String() string { - return fmt.Sprint(*d) -} - -func (d *drives) Set(value string) error { - components := strings.Split(value, ",") - if len(components) < 2 { - return fmt.Errorf("--drive parameter should be of format path,format[,option]*") - } - _, err := os.Stat(components[0]) - if err != nil { - return fmt.Errorf("Unable to access %s : %v", components[1], err) - } - *d = append(*d, drive{ - Path: components[0], - Format: components[1], - Options: strings.Join(components[2:], ","), - }) - return nil -} - -func vmFlags(fs *flag.FlagSet, memGB, CPUs *int, m *mounts, p *ports, d *drives) { - fs.IntVar(memGB, "mem", *memGB, "Gigabytes of RAM allocated to VM") - fs.IntVar(CPUs, "cpus", *CPUs, "VCPUs assigned to VM") - fs.Var(m, "mount", "directory to mount in guest VM via 9p. Format is tag,security_model,path") - fs.Var(d, "drive", "Host accessible resource to appear as block device in guest VM. Format is path,format[,option]*") - fs.Var(p, "port", "port mapping. Format is host_port-guest_port, e.g., -port 10022-22") -} - -func checkDirectory(dir string) error { - if dir == "" { - return nil - } - - if !path.IsAbs(dir) { - return fmt.Errorf("%s is not an absolute path", dir) - } - - fi, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("Unable to stat %s : %v", dir, err) - } - - if !fi.IsDir() { - return fmt.Errorf("%s is not a directory", dir) - } - - return nil -} - -func createFlags(ctx context.Context, ws *workspace) (*workload, bool, error) { - var debug bool - var m mounts - var p ports - var d drives - var memGiB, CPUs int - var update packageUpgrade - - fs := flag.NewFlagSet("create", flag.ExitOnError) - fs.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s create \n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " \tName of the workload to create\n\n") - fs.PrintDefaults() - } - vmFlags(fs, &memGiB, &CPUs, &m, &p, &d) - fs.BoolVar(&debug, "debug", false, "Enables debug mode") - fs.Var(&update, "package-upgrade", - "Hint to enable or disable update of VM packages. Should be true or false") - - if err := fs.Parse(flag.Args()[1:]); err != nil { - return nil, false, err - } - - if fs.NArg() != 1 { - fs.Usage() - return nil, false, errors.New("no workload specified") - } - workloadName := fs.Arg(0) - - for i := range m { - if err := checkDirectory(m[i].Path); err != nil { - return nil, false, err - } - } - - wkl, err := createWorkload(ctx, ws, workloadName) - if err != nil { - return nil, false, err - } - - in := &wkl.spec.VM - if memGiB != 0 { - in.MemGiB = memGiB - } - if CPUs != 0 { - in.CPUs = CPUs - } - - in.mergeMounts(m) - in.mergePorts(p) - in.mergeDrives(d) - - ws.Mounts = in.Mounts - ws.Hostname = wkl.spec.Hostname - if ws.NoProxy != "" { - ws.NoProxy = fmt.Sprintf("%s,%s", ws.Hostname, ws.NoProxy) - } else if ws.HTTPProxy != "" || ws.HTTPSProxy != "" { - ws.NoProxy = ws.Hostname - } - ws.PackageUpgrade = string(update) - - return wkl, debug, nil -} - -func startFlags(in *VMSpec) error { - var m mounts - var p ports - var d drives - - CPUs := in.CPUs - memGiB := in.MemGiB - - fs := flag.NewFlagSet("start", flag.ExitOnError) - vmFlags(fs, &memGiB, &CPUs, &m, &p, &d) - if err := fs.Parse(flag.Args()[1:]); err != nil { - return err - } - - for i := range m { - if err := checkDirectory(m[i].Path); err != nil { - return err - } - } - - in.CPUs = CPUs - in.MemGiB = memGiB - in.mergeMounts(m) - in.mergePorts(p) - in.mergeDrives(d) - - return nil -} - -func create(ctx context.Context, errCh chan error) { - var err error - - defer func() { - errCh <- err - }() - - ws, err := prepareEnv(ctx) - if err != nil { - return - } - - wkld, debug, err := createFlags(ctx, ws) - if err != nil { - return - } - - if wkld.spec.NeedsNestedVM && !hostSupportsNestedKVM() { - err = fmt.Errorf("nested KVM is not enabled. Please enable and try again") - return - } - - in := &wkld.spec.VM - _, err = os.Stat(ws.instanceDir) - if err == nil { - err = fmt.Errorf("instance already exists") - return - } - - fmt.Println("Installing host dependencies") - installDeps(ctx) - - err = os.MkdirAll(ws.instanceDir, 0755) - if err != nil { - err = fmt.Errorf("unable to create cache dir: %v", err) - return - } - - defer func() { - if err != nil { - _ = os.RemoveAll(ws.instanceDir) - } - }() - - err = wkld.save(ws) - if err != nil { - err = fmt.Errorf("Unable to save instance state : %v", err) - return - } - - err = prepareSSHKeys(ctx, ws) - if err != nil { - return - } - - fmt.Printf("Downloading %s\n", wkld.spec.BaseImageName) - qcowPath, err := downloadFile(ctx, wkld.spec.BaseImageURL, ws.ciaoDir, downloadProgress) - if err != nil { - return - } - - err = buildISOImage(ctx, ws.instanceDir, wkld.userData, ws, debug) - if err != nil { - return - } - - err = createRootfs(ctx, qcowPath, ws.instanceDir) - if err != nil { - return - } - - fmt.Printf("Booting VM with %d GB RAM and %d cpus\n", in.MemGiB, in.CPUs) - - err = bootVM(ctx, ws, in) - if err != nil { - return - } - - err = manageInstallation(ctx, ws.ciaoDir, ws.instanceDir, ws) - if err != nil { - return - } - fmt.Println("VM successfully created!") - fmt.Println("Type ciao-down connect to start using it.") -} - -func start(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - wkld, err := restoreWorkload(ws) - if err != nil { - errCh <- err - return - } - in := &wkld.spec.VM - - memGiB, CPUs := getMemAndCpus() - if in.MemGiB == 0 { - in.MemGiB = memGiB - } - if in.CPUs == 0 { - in.CPUs = CPUs - } - - err = startFlags(in) - if err != nil { - errCh <- err - return - } - - if wkld.spec.NeedsNestedVM && !hostSupportsNestedKVM() { - errCh <- fmt.Errorf("nested KVM is not enabled. Please enable and try again") - return - } - - if err := wkld.save(ws); err != nil { - fmt.Printf("Warning: Failed to update instance state: %v", err) - } - - fmt.Printf("Booting VM with %d GB RAM and %d cpus\n", in.MemGiB, in.CPUs) - - err = bootVM(ctx, ws, in) - if err != nil { - errCh <- err - return - } - - fmt.Println("VM Started") - - errCh <- err -} - -func stop(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - err = stopVM(ctx, ws.instanceDir) - if err != nil { - errCh <- err - return - } - - fmt.Println("VM Stopped") - - errCh <- err -} - -func quit(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - err = quitVM(ctx, ws.instanceDir) - if err != nil { - errCh <- err - return - } - - fmt.Println("VM Quit") - - errCh <- err -} - -func status(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - wkld, err := restoreWorkload(ws) - if err != nil { - errCh <- fmt.Errorf("Unable to load instance state: %v", err) - return - } - in := &wkld.spec.VM - - sshPort, err := in.sshPort() - if err != nil { - errCh <- fmt.Errorf("Instance does not have SSH port open. Unable to determine status") - return - } - - statusVM(ctx, ws.instanceDir, ws.keyPath, wkld.spec.WorkloadName, - sshPort) - errCh <- err -} - -func connect(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - wkld, err := restoreWorkload(ws) - if err != nil { - errCh <- fmt.Errorf("Unable to load instance state: %v", err) - return - } - in := &wkld.spec.VM - - path, err := exec.LookPath("ssh") - if err != nil { - errCh <- fmt.Errorf("Unable to locate ssh binary") - return - } - - sshPort, err := in.sshPort() - if err != nil { - errCh <- err - return - } - - if !sshReady(ctx, sshPort) { - fmt.Printf("Waiting for VM to boot ") - DONE: - for { - select { - case <-time.After(time.Second): - case <-ctx.Done(): - errCh <- fmt.Errorf("Cancelled") - return - } - - if sshReady(ctx, sshPort) { - break DONE - } - - fmt.Print(".") - } - fmt.Println() - } - - err = syscall.Exec(path, []string{path, - "-q", "-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking=no", - "-o", "IdentitiesOnly=yes", - "-i", ws.keyPath, - "127.0.0.1", "-p", strconv.Itoa(sshPort)}, - os.Environ()) - errCh <- err -} - -func delete(ctx context.Context, errCh chan error) { - ws, err := prepareEnv(ctx) - if err != nil { - errCh <- err - return - } - - _ = quitVM(ctx, ws.instanceDir) - err = os.RemoveAll(ws.instanceDir) - if err != nil { - errCh <- fmt.Errorf("unable to delete instance: %v", err) - return - } - - errCh <- nil -} - -func runCommand(signalCh <-chan os.Signal) error { - var err error - - errCh := make(chan error) - ctx, cancelFunc := context.WithCancel(context.Background()) - switch os.Args[1] { - case "create": - go create(ctx, errCh) - case "start": - go start(ctx, errCh) - case "stop": - go stop(ctx, errCh) - case "quit": - go quit(ctx, errCh) - case "status": - go status(ctx, errCh) - case "connect": - go connect(ctx, errCh) - case "delete": - go delete(ctx, errCh) - } - select { - case <-signalCh: - cancelFunc() - err = <-errCh - case err = <-errCh: - cancelFunc() - } - - return err -} - -func main() { - flag.Parse() - if len(os.Args) < 2 || - !(os.Args[1] == "create" || os.Args[1] == "start" || os.Args[1] == "stop" || - os.Args[1] == "quit" || os.Args[1] == "status" || - os.Args[1] == "connect" || os.Args[1] == "delete") { - flag.Usage() - os.Exit(1) - } - - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - - if err := runCommand(signalCh); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/testutil/ciao-down/deps.go b/testutil/ciao-down/deps.go deleted file mode 100644 index 116a5785e..000000000 --- a/testutil/ciao-down/deps.go +++ /dev/null @@ -1,48 +0,0 @@ -// -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import "github.com/ciao-project/ciao/osprepare" - -var ciaoDevClearDeps = []osprepare.PackageRequirement{ - {BinaryName: "/usr/bin/qemu-system-x86_64", PackageName: "cloud-control"}, - {BinaryName: "/usr/bin/xorriso", PackageName: "cloud-control"}, - {BinaryName: "/usr/bin/ssh", PackageName: "openssh-server"}, - {BinaryName: "/usr/bin/ssh-keygen", PackageName: "openssh-server"}, -} - -var ciaoDevFedoraDeps = []osprepare.PackageRequirement{ - {BinaryName: "/usr/bin/qemu-system-x86_64", PackageName: "qemu-system-x86"}, - {BinaryName: "/usr/bin/qemu-img", PackageName: "qemu-img"}, - {BinaryName: "/usr/bin/xorriso", PackageName: "xorriso"}, - {BinaryName: "/usr/bin/ssh", PackageName: "openssh-clients"}, - {BinaryName: "/usr/bin/ssh-keygen", PackageName: "openssh-clients"}, -} - -var ciaoDevUbuntuDeps = []osprepare.PackageRequirement{ - {BinaryName: "/usr/bin/qemu-system-x86_64", PackageName: "qemu-system-x86"}, - {BinaryName: "/usr/bin/qemu-img", PackageName: "qemu-utils"}, - {BinaryName: "/usr/bin/xorriso", PackageName: "xorriso"}, - {BinaryName: "/usr/bin/ssh", PackageName: "openssh-client"}, - {BinaryName: "/usr/bin/ssh-keygen", PackageName: "openssh-client"}, -} - -var ciaoDevDeps = map[string][]osprepare.PackageRequirement{ - "clearlinux": ciaoDevClearDeps, - "fedora": ciaoDevFedoraDeps, - "ubuntu": ciaoDevUbuntuDeps, -} diff --git a/testutil/ciao-down/download.go b/testutil/ciao-down/download.go deleted file mode 100644 index a7b41cef0..000000000 --- a/testutil/ciao-down/download.go +++ /dev/null @@ -1,182 +0,0 @@ -/* -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - "os" - "os/exec" - "path" - "path/filepath" -) - -type downloadInfo struct { - imageName string - imageTmpName string -} - -type progressCB func(p progress) - -type progress struct { - downloadedMB int - totalMB int -} - -type progressReader struct { - downloaded int64 - totalMB int - reader io.Reader - cb progressCB -} - -func (pr *progressReader) Read(p []byte) (int, error) { - read, err := pr.reader.Read(p) - if err == nil { - oldMB := pr.downloaded / 10000000 - pr.downloaded += int64(read) - newMB := pr.downloaded / 10000000 - if newMB > oldMB { - pr.cb(progress{downloadedMB: int(newMB * 10), totalMB: pr.totalMB}) - } - } - return read, err -} - -func makeDownloadInfo(URL string) (downloadInfo, error) { - u, err := url.Parse(URL) - if err != nil { - return downloadInfo{}, fmt.Errorf("unable to parse %s:%v", err, URL) - } - di := downloadInfo{ - imageName: filepath.Base(u.Path), - } - di.imageTmpName = di.imageName + ".part" - return di, nil -} - -func getFile(ctx context.Context, URL string, dest io.WriteCloser, cb progressCB) (err error) { - defer func() { - err1 := dest.Close() - if err == nil && err1 != nil { - err = err1 - } - }() - req, err := http.NewRequest("GET", URL, nil) - if err != nil { - return - } - req = req.WithContext(ctx) - cli := &http.Client{Transport: http.DefaultTransport} - resp, err := cli.Do(req) - if err != nil { - return - } - - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("Failed to download %s : %s", URL, resp.Status) - return - } - - pr := &progressReader{reader: resp.Body, cb: cb} - if resp.ContentLength == -1 { - pr.totalMB = -1 - } else { - pr.totalMB = int(resp.ContentLength / 1000000) - } - - buf := make([]byte, 1<<20) - _, err = io.CopyBuffer(dest, pr, buf) - - if err == nil && int(pr.downloaded/1000000)%10 != 0 { - cb(progress{downloadedMB: pr.totalMB, totalMB: pr.totalMB}) - } - - return -} - -func downloadProgress(p progress) { - if p.totalMB >= 0 { - fmt.Printf("Downloaded %d MB of %d\n", p.downloadedMB, p.totalMB) - } else { - fmt.Printf("Downloaded %d MB\n", p.downloadedMB) - } -} - -func downloadFile(ctx context.Context, URL, ciaoDir string, cb progressCB) (string, error) { - di, err := makeDownloadInfo(URL) - if err != nil { - return "", err - } - - cacheDir := path.Join(ciaoDir, "cache") - imgPath := path.Join(cacheDir, di.imageName) - - if _, err := os.Stat(imgPath); err == nil { - return imgPath, nil - } - - if err := os.MkdirAll(cacheDir, 0755); err != nil { - return "", fmt.Errorf("Unable to create directory %s : %v", - cacheDir, err) - } - - // Handles legacy code in which the ubuntu image used to be stored in - // the root of the ~/.ciao_down directory. We don't want to move the - // old image as this would break any existing VMs based off it. - - oldImgPath := path.Join(ciaoDir, di.imageName) - if err := exec.Command("cp", oldImgPath, imgPath).Run(); err == nil { - return imgPath, nil - } - - tmpImgPath := path.Join(cacheDir, di.imageTmpName) - - if _, err := os.Stat(imgPath); err == nil { - return imgPath, nil - } - - if _, err := os.Stat(tmpImgPath); err == nil { - _ = os.Remove(tmpImgPath) - } - - f, err := os.Create(tmpImgPath) - if err != nil { - return "", fmt.Errorf("Unable to create download file: %v", err) - } - - err = getFile(ctx, URL, f, cb) - if err != nil { - _ = os.Remove(tmpImgPath) - return "", fmt.Errorf("Unable download file %s: %v", - URL, err) - } - - err = os.Rename(tmpImgPath, imgPath) - if err != nil { - _ = os.Remove(tmpImgPath) - return "", fmt.Errorf("Unable move downloaded file to %s: %v", - imgPath, err) - } - - return imgPath, nil -} diff --git a/testutil/ciao-down/host.go b/testutil/ciao-down/host.go deleted file mode 100644 index b182fbc5e..000000000 --- a/testutil/ciao-down/host.go +++ /dev/null @@ -1,22 +0,0 @@ -// -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build !linux - -package main - -func getMemAndCpus() (mem int, cpus int) { - return 4, 2 -} diff --git a/testutil/ciao-down/host_linux.go b/testutil/ciao-down/host_linux.go deleted file mode 100644 index 78d1f3f82..000000000 --- a/testutil/ciao-down/host_linux.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// +build linux - -package main - -import "github.com/ciao-project/ciao/deviceinfo" - -func getOnlineCPUs() int { - return deviceinfo.GetOnlineCPUs() -} - -func getTotalMemory() int { - total, _ := deviceinfo.GetMemoryInfo() - total /= 1024 - return total -} - -func getMemAndCpus() (mem int, cpus int) { - cpus = getOnlineCPUs() / 2 - if cpus < 0 { - cpus = 1 - } else if cpus > 8 { - cpus = 8 - } - - mem = getTotalMemory() / 2 - if mem < 0 { - mem = 1 - } else if mem > 8 { - mem = 8 - } - - return mem, cpus -} diff --git a/testutil/ciao-down/instance.go b/testutil/ciao-down/instance.go deleted file mode 100644 index 681e31028..000000000 --- a/testutil/ciao-down/instance.go +++ /dev/null @@ -1,295 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "net/url" - "path" - "path/filepath" - "strings" - "text/template" - - yaml "gopkg.in/yaml.v2" -) - -// Constants for the Guest image used by ciao-down - -const ( - guestDownloadURL = "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img" - guestImageFriendlyName = "Ubuntu 16.04" - defaultHostname = "singlevm" -) - -type portMapping struct { - Host int `yaml:"host"` - Guest int `yaml:"guest"` -} - -func (p portMapping) String() string { - return fmt.Sprintf("%d-%d", p.Host, p.Guest) -} - -type mount struct { - Tag string `yaml:"tag"` - SecurityModel string `yaml:"security_model"` - Path string `yaml:"path"` -} - -func (m mount) String() string { - return fmt.Sprintf("%s,%s,%s", m.Tag, m.SecurityModel, m.Path) -} - -type drive struct { - Path string - Format string - Options string -} - -func (d drive) String() string { - return fmt.Sprintf("%s,%s,%s", d.Path, d.Format, d.Options) -} - -// VMSpec holds the per-VM state. -type VMSpec struct { - MemGiB int `yaml:"mem_gib"` - CPUs int `yaml:"cpus"` - PortMappings []portMapping `yaml:"ports"` - Mounts []mount `yaml:"mounts"` - Drives []drive `yaml:"drives"` -} - -type workloadSpec struct { - BaseImageURL string `yaml:"base_image_url"` - BaseImageName string `yaml:"base_image_name"` - Hostname string `yaml:"hostname"` - WorkloadName string `yaml:"workload"` - NeedsNestedVM bool `yaml:"needs_nested_vm"` - VM VMSpec `yaml:"vm"` -} - -// This function creates a default instanceData object for legacy ciao-down -// ciao VMs. These old VMs did not store information about mounts and -// mapped ports as this information was hard-coded into ciao-down itself. -// Consequently, when migrating one of these old VMs we need to fill in -// the missing information. -func (in *VMSpec) loadLegacyInstance(ws *workspace) error { - // Check for legacy state files. - - data, err := ioutil.ReadFile(path.Join(ws.instanceDir, "vmtype.txt")) - if err == nil { - vmType := string(data) - if vmType != CIAO && vmType != CLEARCONTAINERS { - err := fmt.Errorf("Unsupported vmType %s. Should be one of "+CIAO+"|"+CLEARCONTAINERS, vmType) - return err - } - } - - uiPath := "" - data, err = ioutil.ReadFile(path.Join(ws.instanceDir, "ui_path.txt")) - if err == nil { - uiPath = string(data) - } - - in.Mounts = []mount{ - { - Tag: "hostgo", - SecurityModel: "passthrough", - Path: ws.GoPath, - }, - } - - in.PortMappings = []portMapping{ - { - Host: 10022, - Guest: 22, - }, - } - - if uiPath != "" { - in.Mounts = append(in.Mounts, mount{ - Tag: "hostui", - SecurityModel: "mapped", - Path: filepath.Clean(uiPath), - }) - } - - return nil -} - -func (in *VMSpec) unmarshal(data []byte) error { - err := yaml.Unmarshal(data, in) - if err != nil { - return fmt.Errorf("Unable to unmarshal instance state : %v", err) - } - - for i := range in.Mounts { - if err := checkDirectory(in.Mounts[i].Path); err != nil { - return fmt.Errorf("Bad mount %s specified: %v", - in.Mounts[i].Path, err) - } - } - - var memDef, cpuDef int - if in.MemGiB == 0 || in.CPUs == 0 { - memDef, cpuDef = getMemAndCpus() - if in.MemGiB == 0 { - in.MemGiB = memDef - } - if in.CPUs == 0 { - in.CPUs = cpuDef - } - } - - var i int - for i = 0; i < len(in.PortMappings); i++ { - if in.PortMappings[i].Guest == 22 { - break - } - } - if i == len(in.PortMappings) { - in.PortMappings = append(in.PortMappings, - portMapping{ - Host: 10022, - Guest: 22, - }) - } - - return nil -} - -func (in *VMSpec) unmarshalWithTemplate(ws *workspace, data string) error { - tmpl, err := template.New("instance-data").Parse(string(data)) - if err != nil { - return fmt.Errorf("Unable to parse instance data template: %v", err) - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, ws) - if err != nil { - return fmt.Errorf("Unable to execute instance data template: %v", err) - } - return in.unmarshal(buf.Bytes()) -} - -func (in *VMSpec) mergeMounts(m mounts) { - mountCount := len(in.Mounts) - for _, mount := range m { - var i int - for i = 0; i < mountCount; i++ { - if mount.Tag == in.Mounts[i].Tag { - break - } - } - - if i == mountCount { - in.Mounts = append(in.Mounts, mount) - } else { - in.Mounts[i] = mount - } - } -} - -func (in *VMSpec) mergePorts(p ports) { - portCount := len(in.PortMappings) - for _, port := range p { - var i int - for i = 0; i < portCount; i++ { - if port.Guest == in.PortMappings[i].Guest { - break - } - } - - if i == portCount { - in.PortMappings = append(in.PortMappings, port) - } else { - in.PortMappings[i] = port - } - } -} - -func (in *VMSpec) mergeDrives(d drives) { - driveCount := len(in.Drives) - for _, drive := range d { - var i int - for i = 0; i < driveCount; i++ { - if drive.Path == in.Drives[i].Path { - break - } - } - - if i == driveCount { - in.Drives = append(in.Drives, drive) - } else { - in.Drives[i] = drive - } - } -} - -func (in *VMSpec) sshPort() (int, error) { - for _, p := range in.PortMappings { - if p.Guest == 22 { - return p.Host, nil - } - } - return 0, fmt.Errorf("No SSH port configured") -} - -func (ins *workloadSpec) unmarshal(data []byte) error { - err := yaml.Unmarshal(data, ins) - if err != nil { - return fmt.Errorf("Unable to unmarshal instance specification : %v", err) - } - - if ins.BaseImageURL == "" { - ins.BaseImageURL = guestDownloadURL - ins.BaseImageName = guestImageFriendlyName - } else { - url, err := url.Parse(ins.BaseImageURL) - if err != nil { - return fmt.Errorf("Unable to parse url %s : %v", - ins.BaseImageURL, err) - } - if ins.BaseImageName == "" { - lastSlash := strings.LastIndex(url.Path, "/") - if lastSlash == -1 { - ins.BaseImageName = url.Path - } else { - ins.BaseImageName = url.Path[lastSlash+1:] - } - } - } - - if ins.Hostname == "" { - ins.Hostname = defaultHostname - } - return nil -} - -func (ins *workloadSpec) unmarshalWithTemplate(ws *workspace, data string) error { - tmpl, err := template.New("instance-spec").Parse(string(data)) - if err != nil { - return fmt.Errorf("Unable to parse instance data template: %v", err) - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, ws) - if err != nil { - return fmt.Errorf("Unable to execute instance data template: %v", err) - } - return ins.unmarshal(buf.Bytes()) -} diff --git a/testutil/ciao-down/mock_test.go b/testutil/ciao-down/mock_test.go deleted file mode 100644 index 28cee060e..000000000 --- a/testutil/ciao-down/mock_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "io/ioutil" - "path" - "testing" - - "os" - - "github.com/stretchr/testify/assert" -) - -const xenialWorkloadSpecNoVM = ` -base_image_url: ` + guestDownloadURL + ` -base_image_name: ` + guestImageFriendlyName + ` -` - -const sampleVMSpec = ` -mem_gib: 3 -cpus: 2 -ports: -- host: 10022 - guest: 22 -mounts: [] -` - -const xenialWorkloadSpec = ` -base_image_url: ` + guestDownloadURL + ` -base_image_name: ` + guestImageFriendlyName + ` -vm: - mem_gib: 3 - cpus: 2 - ports: - - host: 10022 - guest: 22 - mounts: [] -` - -var mockVMSpec = VMSpec{ - MemGiB: 3, - CPUs: 2, - PortMappings: []portMapping{{Host: 10022, Guest: 22}}, - Mounts: []mount{}, -} - -const sampleCloudInit = ` -` - -const sampleWorkload3Docs = "---\n" + xenialWorkloadSpecNoVM + "...\n---\n" + sampleVMSpec + "...\n---\n" + sampleCloudInit + "...\n" -const sampleWorkload = "---\n" + xenialWorkloadSpec + "...\n---\n" + sampleCloudInit + "...\n" - -func createMockWorkSpaceWithWorkload(t *testing.T, workload string) *workspace { - ciaoDir, err := ioutil.TempDir("", "ciao-down-tests-") - assert.Nil(t, err) - - instanceDir := path.Join(ciaoDir, "foo") - err = os.Mkdir(instanceDir, 0750) - assert.Nil(t, err) - - ws := &workspace{ - ciaoDir: ciaoDir, - instanceDir: instanceDir, - } - - workloadFile := path.Join(ws.instanceDir, "state.yaml") - err = ioutil.WriteFile(workloadFile, []byte(workload), 0640) - assert.Nil(t, err) - - return ws -} - -func cleanupMockWorkspace(t *testing.T, ws *workspace) { - err := os.RemoveAll(ws.ciaoDir) - assert.Nil(t, err) -} diff --git a/testutil/ciao-down/prepare.go b/testutil/ciao-down/prepare.go deleted file mode 100644 index 8dc6fcd3b..000000000 --- a/testutil/ciao-down/prepare.go +++ /dev/null @@ -1,371 +0,0 @@ -// -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bytes" - "context" - "fmt" - "io/ioutil" - "net/url" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "text/template" - - "github.com/ciao-project/ciao/osprepare" - "github.com/ciao-project/ciao/qemu" - "github.com/ciao-project/ciao/uuid" -) - -const metaDataTemplate = ` -{ - "uuid": "{{.UUID}}", - "hostname": "{{.Hostname}}" -} -` - -type logger struct{} - -func (l logger) V(int32) bool { - return false -} - -func (l logger) Infof(s string, args ...interface{}) { - out := fmt.Sprintf(s, args...) - fmt.Print(out) - if !strings.HasSuffix(out, "\n") { - fmt.Println() - } -} - -func (l logger) Warningf(s string, args ...interface{}) { - l.Infof(s, args) -} - -func (l logger) Errorf(s string, args ...interface{}) { - l.Infof(s, args) -} - -type workspace struct { - GoPath string - Home string - HTTPProxy string - HTTPSProxy string - NoProxy string - User string - PublicKey string - HTTPServerPort int - GitUserName string - GitEmail string - Mounts []mount - Hostname string - UUID string - PackageUpgrade string - ciaoDir string - instanceDir string - keyPath string - publicKeyPath string -} - -func (w *workspace) MountPath(tag string) string { - for _, m := range w.Mounts { - if m.Tag == tag { - return m.Path - } - } - - return "" -} - -func installDeps(ctx context.Context) { - osprepare.InstallDeps(ctx, ciaoDevDeps, logger{}) -} - -func hostSupportsNestedKVMIntel() bool { - data, err := ioutil.ReadFile("/sys/module/kvm_intel/parameters/nested") - if err != nil { - return false - } - - return strings.TrimSpace(string(data)) == "Y" -} - -func hostSupportsNestedKVMAMD() bool { - data, err := ioutil.ReadFile("/sys/module/kvm_amd/parameters/nested") - if err != nil { - return false - } - - return strings.TrimSpace(string(data)) == "1" -} - -func hostSupportsNestedKVM() bool { - return hostSupportsNestedKVMIntel() || hostSupportsNestedKVMAMD() -} - -func prepareSSHKeys(ctx context.Context, ws *workspace) error { - _, privKeyErr := os.Stat(ws.keyPath) - _, pubKeyErr := os.Stat(ws.publicKeyPath) - - if pubKeyErr != nil || privKeyErr != nil { - err := exec.CommandContext(ctx, "ssh-keygen", - "-f", ws.keyPath, "-t", "rsa", "-N", "").Run() - if err != nil { - return fmt.Errorf("Unable to generate SSH key pair : %v", err) - } - } - - publicKey, err := ioutil.ReadFile(ws.publicKeyPath) - if err != nil { - return fmt.Errorf("Unable to read public ssh key: %v", err) - } - - ws.PublicKey = string(publicKey) - return nil -} - -func getProxy(upper, lower string) (string, error) { - proxy := os.Getenv(upper) - if proxy == "" { - proxy = os.Getenv(lower) - } - - if proxy == "" { - return "", nil - } - - if proxy[len(proxy)-1] == '/' { - proxy = proxy[:len(proxy)-1] - } - - proxyURL, err := url.Parse(proxy) - if err != nil { - return "", fmt.Errorf("Failed to parse %s : %v", proxy, err) - } - return proxyURL.String(), nil -} - -func prepareEnv(ctx context.Context) (*workspace, error) { - var err error - - ws := &workspace{HTTPServerPort: 8080} - data, err := exec.Command("go", "env", "GOPATH").Output() - if err == nil { - ws.GoPath = filepath.Clean(strings.TrimSpace(string(data))) - } - ws.Home = os.Getenv("HOME") - if ws.Home == "" { - return nil, fmt.Errorf("HOME is not defined") - } - ws.User = os.Getenv("USER") - if ws.User == "" { - return nil, fmt.Errorf("USER is not defined") - } - - ws.HTTPProxy, err = getProxy("HTTP_PROXY", "http_proxy") - if err != nil { - return nil, err - } - - ws.HTTPSProxy, err = getProxy("HTTPS_PROXY", "https_proxy") - if err != nil { - return nil, err - } - - if ws.HTTPSProxy != "" { - u, _ := url.Parse(ws.HTTPSProxy) - u.Scheme = "http" - ws.HTTPSProxy = u.String() - } - - ws.NoProxy = os.Getenv("no_proxy") - ws.ciaoDir = path.Join(ws.Home, ".ciao-down") - ws.instanceDir = path.Join(ws.ciaoDir, "instance") - ws.keyPath = path.Join(ws.ciaoDir, "id_rsa") - ws.publicKeyPath = fmt.Sprintf("%s.pub", ws.keyPath) - - data, err = exec.Command("git", "config", "--global", "user.name").Output() - if err == nil { - ws.GitUserName = strings.TrimSpace(string(data)) - } - - data, err = exec.Command("git", "config", "--global", "user.email").Output() - if err == nil { - ws.GitEmail = strings.TrimSpace(string(data)) - } - - ws.UUID = uuid.Generate().String() - - return ws, nil -} - -func createCloudInitISO(ctx context.Context, instanceDir string, userData, metaData []byte) error { - isoPath := path.Join(instanceDir, "config.iso") - return qemu.CreateCloudInitISO(ctx, instanceDir, isoPath, userData, metaData) -} - -func downloadFN(ws *workspace, URL, location string) string { - url := url.URL{ - Scheme: "http", - Host: fmt.Sprintf("10.0.2.2:%d", ws.HTTPServerPort), - Path: "download", - } - q := url.Query() - q.Set(urlParam, URL) - url.RawQuery = q.Encode() - return fmt.Sprintf("wget %s -O %s", url.String(), location) -} - -func beginTaskFN(ws *workspace, message string) string { - const infoStr = `curl -X PUT -d "%s" 10.0.2.2:%d` - return fmt.Sprintf(infoStr, message, ws.HTTPServerPort) -} - -func endTaskCheckFN(ws *workspace) string { - const checkStr = `if [ $? -eq 0 ] ; then ret="OK" ; else ret="FAIL" ; fi ; ` + - `curl -X PUT -d $ret 10.0.2.2:%d` - return fmt.Sprintf(checkStr, ws.HTTPServerPort) -} - -func endTaskOkFN(ws *workspace) string { - const okStr = `curl -X PUT -d "OK" 10.0.2.2:%d` - return fmt.Sprintf(okStr, ws.HTTPServerPort) -} - -func endTaskFailFN(ws *workspace) string { - const failStr = `curl -X PUT -d "FAIL" 10.0.2.2:%d` - return fmt.Sprintf(failStr, ws.HTTPServerPort) -} - -func finishedFN(ws *workspace) string { - const finishedStr = `curl -X PUT -d "FINISHED" 10.0.2.2:%d` - return fmt.Sprintf(finishedStr, ws.HTTPServerPort) -} - -func proxyVarsFN(ws *workspace) string { - var buf bytes.Buffer - if ws.NoProxy != "" { - buf.WriteString("no_proxy=") - buf.WriteString(ws.NoProxy) - buf.WriteString(" ") - buf.WriteString("NO_PROXY=") - buf.WriteString(ws.NoProxy) - buf.WriteString(" ") - } - if ws.HTTPProxy != "" { - buf.WriteString("http_proxy=") - buf.WriteString(ws.HTTPProxy) - buf.WriteString(" HTTP_PROXY=") - buf.WriteString(ws.HTTPProxy) - buf.WriteString(" ") - } - if ws.HTTPSProxy != "" { - buf.WriteString("https_proxy=") - buf.WriteString(ws.HTTPSProxy) - buf.WriteString(" HTTPS_PROXY=") - buf.WriteString(ws.HTTPSProxy) - buf.WriteString(" ") - } - return strings.TrimSpace(buf.String()) -} - -func proxyEnvFN(ws *workspace, indent int) string { - var buf bytes.Buffer - spaces := strings.Repeat(" ", indent) - if ws.NoProxy != "" { - buf.WriteString(spaces) - buf.WriteString(`no_proxy="`) - buf.WriteString(ws.NoProxy) - buf.WriteString(`"` + "\n") - buf.WriteString(spaces) - buf.WriteString(`NO_PROXY="`) - buf.WriteString(ws.NoProxy) - buf.WriteString(`"` + "\n") - } - if ws.HTTPProxy != "" { - buf.WriteString(spaces) - buf.WriteString(`http_proxy="`) - buf.WriteString(ws.HTTPProxy) - buf.WriteString(`"` + "\n") - buf.WriteString(spaces) - buf.WriteString(`HTTP_PROXY="`) - buf.WriteString(ws.HTTPProxy) - buf.WriteString(`"` + "\n") - } - if ws.HTTPSProxy != "" { - buf.WriteString(spaces) - buf.WriteString(`https_proxy="`) - buf.WriteString(ws.HTTPSProxy) - buf.WriteString(`"` + "\n") - buf.WriteString(spaces) - buf.WriteString(`HTTPS_PROXY="`) - buf.WriteString(ws.HTTPSProxy) - buf.WriteString(`"`) - } - return buf.String() -} - -func buildISOImage(ctx context.Context, instanceDir, tmpl string, ws *workspace, debug bool) error { - funcMap := template.FuncMap{ - "proxyVars": proxyVarsFN, - "proxyEnv": proxyEnvFN, - "download": downloadFN, - "beginTask": beginTaskFN, - "endTaskCheck": endTaskCheckFN, - "endTaskOk": endTaskOkFN, - "endTaskFail": endTaskFailFN, - "finished": finishedFN, - } - - udt := template.Must(template.New("user-data").Funcs(funcMap).Parse(tmpl)) - var udBuf bytes.Buffer - err := udt.Execute(&udBuf, ws) - if err != nil { - return fmt.Errorf("Unable to execute user data template : %v", err) - } - - mdt := template.Must(template.New("meta-data").Parse(metaDataTemplate)) - - var mdBuf bytes.Buffer - err = mdt.Execute(&mdBuf, ws) - if err != nil { - return fmt.Errorf("Unable to execute user data template : %v", err) - } - - if debug { - fmt.Println(string(udBuf.Bytes())) - fmt.Println(string(mdBuf.Bytes())) - } - - return createCloudInitISO(ctx, instanceDir, udBuf.Bytes(), mdBuf.Bytes()) -} - -// TODO: Code copied from launcher. Needs to be moved to qemu - -func createRootfs(ctx context.Context, backingImage, instanceDir string) error { - vmImage := path.Join(instanceDir, "image.qcow2") - if _, err := os.Stat(vmImage); err == nil { - _ = os.Remove(vmImage) - } - params := make([]string, 0, 32) - params = append(params, "create", "-f", "qcow2", "-o", "backing_file="+backingImage, - vmImage, "60000M") - return exec.CommandContext(ctx, "qemu-img", params...).Run() -} diff --git a/testutil/ciao-down/vm.go b/testutil/ciao-down/vm.go deleted file mode 100644 index 85d93d24c..000000000 --- a/testutil/ciao-down/vm.go +++ /dev/null @@ -1,287 +0,0 @@ -// -// Copyright (c) 2016 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bufio" - "bytes" - "context" - "fmt" - "io" - "net" - "net/http" - "os" - "path" - "strings" - "text/tabwriter" - "time" - - "github.com/ciao-project/ciao/qemu" -) - -// Download Parameters - -const ( - friendlyNameParam = "name" - urlParam = "url" -) - -func bootVM(ctx context.Context, ws *workspace, in *VMSpec) error { - disconnectedCh := make(chan struct{}) - socket := path.Join(ws.instanceDir, "socket") - qmp, _, err := qemu.QMPStart(ctx, socket, qemu.QMPConfig{}, disconnectedCh) - if err == nil { - qmp.Shutdown() - return fmt.Errorf("VM is already running") - } - - vmImage := path.Join(ws.instanceDir, "image.qcow2") - isoPath := path.Join(ws.instanceDir, "config.iso") - memParam := fmt.Sprintf("%dG", in.MemGiB) - CPUsParam := fmt.Sprintf("cpus=%d", in.CPUs) - args := []string{ - "-qmp", fmt.Sprintf("unix:%s,server,nowait", socket), - "-m", memParam, "-smp", CPUsParam, - "-drive", fmt.Sprintf("file=%s,if=virtio,aio=threads,format=qcow2", vmImage), - "-drive", fmt.Sprintf("file=%s,if=virtio,media=cdrom", isoPath), - "-daemonize", "-enable-kvm", "-cpu", "host", - "-net", "nic,model=virtio", - "-device", "virtio-rng-pci", - } - - for i, m := range in.Mounts { - fsdevParam := fmt.Sprintf("local,security_model=%s,id=fsdev%d,path=%s", - m.SecurityModel, i, m.Path) - devParam := fmt.Sprintf("virtio-9p-pci,id=fs%[1]d,fsdev=fsdev%[1]d,mount_tag=%s", - i, m.Tag) - args = append(args, "-fsdev", fsdevParam, "-device", devParam) - } - - for _, d := range in.Drives { - options := strings.TrimSpace(d.Options) - if options != "" { - options = "," + options - } - driveParam := fmt.Sprintf("file=%s,if=virtio,format=%s%s", d.Path, - d.Format, options) - args = append(args, "-drive", driveParam) - } - - var b bytes.Buffer - if len(in.PortMappings) > 0 { - i := 0 - p := in.PortMappings[i] - b.WriteString(fmt.Sprintf("user,hostfwd=tcp::%d-:%d", p.Host, p.Guest)) - for i = i + 1; i < len(in.PortMappings); i++ { - p := in.PortMappings[i] - b.WriteString(fmt.Sprintf(",hostfwd=tcp::%d-:%d", p.Host, p.Guest)) - } - } - - netParam := b.String() - if len(netParam) > 0 { - args = append(args, "-net", netParam) - } - - args = append(args, "-display", "none", "-vga", "none") - - output, err := qemu.LaunchCustomQemu(ctx, "", args, nil, nil) - if err != nil { - return fmt.Errorf("Failed to launch qemu : %v, %s", err, output) - } - return nil -} - -func executeQMPCommand(ctx context.Context, instanceDir string, - cmd func(ctx context.Context, q *qemu.QMP) error) error { - socket := path.Join(instanceDir, "socket") - disconnectedCh := make(chan struct{}) - qmp, _, err := qemu.QMPStart(ctx, socket, qemu.QMPConfig{}, disconnectedCh) - if err != nil { - return fmt.Errorf("Failed to connect to VM : %v", err) - } - defer qmp.Shutdown() - - err = qmp.ExecuteQMPCapabilities(ctx) - if err != nil { - return fmt.Errorf("Unable to query QEMU caps : %v", err) - } - - err = cmd(ctx, qmp) - if err != nil { - return fmt.Errorf("Unable to execute vm command : %v", err) - } - - return nil -} - -func stopVM(ctx context.Context, instanceDir string) error { - return executeQMPCommand(ctx, instanceDir, func(ctx context.Context, q *qemu.QMP) error { - return q.ExecuteSystemPowerdown(ctx) - }) -} - -func quitVM(ctx context.Context, instanceDir string) error { - return executeQMPCommand(ctx, instanceDir, func(ctx context.Context, q *qemu.QMP) error { - return q.ExecuteQuit(ctx) - }) -} - -func sshReady(ctx context.Context, sshPort int) bool { - dialer := net.Dialer{} - conn, err := dialer.DialContext(ctx, "tcp", - fmt.Sprintf("127.0.0.1:%d", sshPort)) - if err != nil { - return false - } - _ = conn.SetReadDeadline(time.Now().Add(time.Millisecond * 500)) - scanner := bufio.NewScanner(conn) - retval := scanner.Scan() - _ = conn.Close() - return retval -} - -func statusVM(ctx context.Context, instanceDir, keyPath, workloadName string, sshPort int) { - status := "ciao down" - ssh := "N/A" - if sshReady(ctx, sshPort) { - status = "ciao up" - ssh = fmt.Sprintf("ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i %s 127.0.0.1 -p %d", keyPath, sshPort) - } - - w := new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(w, "Workload\t:\t%s\n", workloadName) - fmt.Fprintf(w, "Status\t:\t%s\n", status) - fmt.Fprintf(w, "SSH\t:\t%s\n", ssh) - w.Flush() -} - -func serveLocalFile(ctx context.Context, ciaoDir string, w http.ResponseWriter, - r *http.Request) { - params := r.URL.Query() - URL := params.Get(urlParam) - - path, err := downloadFile(ctx, URL, ciaoDir, func(progress) {}) - if err != nil { - // May not be the correct error code but the error message is only going - // to end up in cloud-init's logs. - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - f, err := os.Open(path) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - _, err = io.Copy(w, f) - _ = f.Close() - if err != nil { - http.Error(w, fmt.Sprintf("Failed to copy %s : %v", friendlyNameParam, err), - http.StatusInternalServerError) - return - } -} - -func startHTTPServer(ctx context.Context, ciaoDir string, listener net.Listener, - errCh chan error) { - finished := false - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - var b bytes.Buffer - _, err := io.Copy(&b, r.Body) - if err != nil { - // TODO: Figure out what to do here - return - } - line := string(b.Bytes()) - if line == "FINISHED" { - _ = listener.Close() - finished = true - return - } - if line == "OK" || line == "FAIL" { - fmt.Printf("[%s]\n", line) - } else { - fmt.Printf("%s : ", line) - } - }) - - http.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) { - serveLocalFile(ctx, ciaoDir, w, r) - }) - - server := &http.Server{} - go func() { - _ = server.Serve(listener) - if finished { - errCh <- nil - } else { - errCh <- fmt.Errorf("HTTP server exited prematurely") - } - }() -} - -func manageInstallation(ctx context.Context, ciaoDir, instanceDir string, ws *workspace) error { - socket := path.Join(instanceDir, "socket") - disconnectedCh := make(chan struct{}) - - qmp, _, err := qemu.QMPStart(ctx, socket, qemu.QMPConfig{}, disconnectedCh) - if err != nil { - return fmt.Errorf("Unable to connect to VM : %v", err) - } - - qemuShutdown := true - defer func() { - if qemuShutdown { - ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) - _ = qmp.ExecuteQuit(ctx) - <-disconnectedCh - cancelFn() - } - qmp.Shutdown() - }() - - err = qmp.ExecuteQMPCapabilities(ctx) - if err != nil { - return fmt.Errorf("Unable to query QEMU caps") - } - - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", ws.HTTPServerPort)) - if err != nil { - return fmt.Errorf("Unable to create listener: %v", err) - } - - errCh := make(chan error) - startHTTPServer(ctx, ciaoDir, listener, errCh) - select { - case <-ctx.Done(): - _ = listener.Close() - <-errCh - return ctx.Err() - case err := <-errCh: - if err == nil { - qemuShutdown = false - } - return err - case <-disconnectedCh: - qemuShutdown = false - _ = listener.Close() - <-errCh - return fmt.Errorf("Lost connection to QEMU instance") - } -} diff --git a/testutil/ciao-down/workload.go b/testutil/ciao-down/workload.go deleted file mode 100644 index c89cec8d4..000000000 --- a/testutil/ciao-down/workload.go +++ /dev/null @@ -1,296 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bufio" - "bytes" - "context" - "fmt" - "go/build" - "io/ioutil" - "net/url" - "os" - "path" - "path/filepath" - "regexp" - "time" - - yaml "gopkg.in/yaml.v2" -) - -const ciaoDownPkg = "github.com/ciao-project/ciao/testutil/ciao-down" - -var indentedRegexp *regexp.Regexp - -func init() { - indentedRegexp = regexp.MustCompile("\\s+.*") -} - -type workload struct { - spec workloadSpec - userData string -} - -func (wkld *workload) save(ws *workspace) error { - var buf bytes.Buffer - - _, _ = buf.WriteString("---\n") - data, err := yaml.Marshal(wkld.spec) - if err != nil { - return fmt.Errorf("Unable to marshal instance specification : %v", err) - } - _, _ = buf.Write(data) - _, _ = buf.WriteString("...\n") - - _, _ = buf.WriteString("---\n") - _, _ = buf.WriteString(wkld.userData) - _, _ = buf.WriteString("...\n") - - err = ioutil.WriteFile(path.Join(ws.instanceDir, "state.yaml"), - buf.Bytes(), 0600) - if err != nil { - return fmt.Errorf("Unable to write instance state : %v", err) - } - - return nil -} - -func workloadFromURL(ctx context.Context, u url.URL) ([]byte, error) { - var workloadPath string - - switch u.Scheme { - case "http", "https": - workloadFile, err := ioutil.TempFile("", ".workload") - if err != nil { - return nil, fmt.Errorf("Failed to create a temporal file: %s", err) - } - - workloadPath = workloadFile.Name() - defer func() { _ = os.Remove(workloadPath) }() - - // 60 seconds should be enough to download the workload file - ctx, cancelFunc := context.WithTimeout(ctx, 60*time.Second) - err = getFile(ctx, u.String(), workloadFile, downloadProgress) - cancelFunc() - if err != nil { - return nil, fmt.Errorf("Unable download workload file from %s: %v", u.String(), err) - } - case "file": - workloadPath = u.Path - - default: - return nil, fmt.Errorf("Unable download workload file %s: unsupported scheme", u.String()) - } - - return ioutil.ReadFile(workloadPath) -} - -func loadWorkloadData(ctx context.Context, ws *workspace, workloadName string) ([]byte, error) { - u, err := url.Parse(workloadName) - if err != nil { - return nil, fmt.Errorf("Unable to parse workload name %s: %s", workloadName, err) - } - - // Absolute means that it has a non-empty scheme - if u.IsAbs() { - return workloadFromURL(ctx, *u) - } - - wkld, err := ioutil.ReadFile(workloadName) - if err == nil { - return wkld, nil - } - - localPath := filepath.Join(ws.Home, ".ciao-down", "workloads", - fmt.Sprintf("%s.yaml", workloadName)) - wkld, err = ioutil.ReadFile(localPath) - if err == nil { - return wkld, nil - } - - p, err := build.Default.Import(ciaoDownPkg, "", build.FindOnly) - if err != nil { - return nil, fmt.Errorf("Unable to locate ciao-down workload directory: %v", err) - } - workloadPath := filepath.Join(p.Dir, "workloads", fmt.Sprintf("%s.yaml", workloadName)) - wkld, err = ioutil.ReadFile(workloadPath) - if err != nil { - return nil, fmt.Errorf("Unable to load workload %s", workloadPath) - } - - return wkld, nil -} - -func unmarshalWorkload(ws *workspace, wkld *workload, spec, VMData, - userData string) error { - err := wkld.spec.unmarshalWithTemplate(ws, spec) - if err != nil { - return err - } - - err = wkld.spec.VM.unmarshalWithTemplate(ws, VMData) - if err != nil { - return err - } - - wkld.userData = userData - - return nil -} - -func createWorkload(ctx context.Context, ws *workspace, workloadName string) (*workload, error) { - data, err := loadWorkloadData(ctx, ws, workloadName) - if err != nil { - return nil, err - } - - var wkld workload - var spec, VMData, userData string - docs := splitYaml(data) - if len(docs) == 1 { - userData = string(docs[0]) - } else if len(docs) == 2 { - spec = string(docs[0]) - userData = string(docs[1]) - } else if len(docs) >= 3 { - spec = string(docs[0]) - VMData = string(docs[1]) - userData = string(docs[2]) - } else { - return nil, fmt.Errorf("Invalid workload") - } - - err = unmarshalWorkload(ws, &wkld, spec, VMData, userData) - if err != nil { - return nil, err - } - if wkld.spec.WorkloadName == "" { - wkld.spec.WorkloadName = workloadName - } - return &wkld, nil -} - -func restoreWorkload(ws *workspace) (*workload, error) { - var wkld workload - data, err := ioutil.ReadFile(path.Join(ws.instanceDir, "state.yaml")) - if err != nil { - if err = wkld.spec.VM.loadLegacyInstance(ws); err != nil { - return nil, err - } - return &wkld, nil - } - - docs := splitYaml(data) - if len(docs) == 0 { - return nil, fmt.Errorf("Invalid workload") - } - if len(docs) == 1 { - // Older versions of ciao-down just stored the VM data and not the - // entire workload. - if err = wkld.spec.VM.unmarshalWithTemplate(ws, string(docs[0])); err != nil { - return nil, err - } - return &wkld, nil - } - if len(docs) == 2 { - err = unmarshalWorkload(ws, &wkld, string(docs[0]), "", string(docs[1])) - return &wkld, err - } - - // 3 or more documents. - err = unmarshalWorkload(ws, &wkld, string(docs[0]), string(docs[1]), string(docs[2])) - return &wkld, err -} - -func findDocument(lines [][]byte) ([]byte, int) { - var realStart int - var realEnd int - docStartFound := false - docEndFound := false - - start := len(lines) - 1 - line := lines[start] - if bytes.HasPrefix(line, []byte("...")) { - docEndFound = true - realEnd = start - start-- - } - - for ; start >= 0; start-- { - line := lines[start] - if bytes.HasPrefix(line, []byte("---")) { - docStartFound = true - break - } - if bytes.HasPrefix(line, []byte("...")) { - start++ - break - } - } - - if docStartFound { - realStart = start + 1 - for start = start - 1; start >= 0; start-- { - line := lines[start] - if !bytes.HasPrefix(line, []byte{'%'}) { - break - } - } - start++ - } else { - if start < 0 { - start = 0 - } - realStart = start - } - - if !docEndFound { - realEnd = len(lines) - } - - var buf bytes.Buffer - for _, line := range lines[realStart:realEnd] { - _, _ = buf.Write(line) - _ = buf.WriteByte('\n') - } - - return buf.Bytes(), start -} - -func splitYaml(data []byte) [][]byte { - lines := make([][]byte, 0, 256) - docs := make([][]byte, 0, 3) - - reader := bytes.NewReader(data) - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Bytes() - lineC := make([]byte, len(line)) - _ = copy(lineC, line) - lines = append(lines, lineC) - } - - endOfNextDoc := len(lines) - for endOfNextDoc > 0 { - var doc []byte - doc, endOfNextDoc = findDocument(lines[:endOfNextDoc]) - docs = append([][]byte{doc}, docs...) - } - - return docs -} diff --git a/testutil/ciao-down/workload_test.go b/testutil/ciao-down/workload_test.go deleted file mode 100644 index b9ef0d743..000000000 --- a/testutil/ciao-down/workload_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// -// Copyright (c) 2017 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -const document1 = `# Just a simple document -foo: "a string" -bar: True -` - -const document2 = `# A list -- foo -- bar -` - -const twoDocuments = "---\n" + document1 + "...\n----\n" + document2 + "...\n" - -func TestSplitYaml(t *testing.T) { - tests := []struct { - content string - documents [][]byte - }{ - {"", [][]byte{}}, - {document1, [][]byte{[]byte(document1)}}, - {twoDocuments, [][]byte{[]byte(document1), []byte(document2)}}, - } - - for i := range tests { - test := &tests[i] - - documents := splitYaml([]byte(test.content)) - assert.Equal(t, test.documents, documents) - } -} - -func TestRestoreWorkload(t *testing.T) { - tests := []struct { - checkSpec bool // Should we check the workload.spec content? - workload string - }{ - // 1 document: per-VM data (legacy) - {workload: sampleVMSpec}, - // 2 documents: spec, cloud init file - {workload: sampleWorkload, checkSpec: true}, - // 3 documents: spec, per-VM data, cloud init file (legacy) - {workload: sampleWorkload3Docs, checkSpec: true}, - } - - for i := range tests { - test := &tests[i] - - ws := createMockWorkSpaceWithWorkload(t, test.workload) - - workload, err := restoreWorkload(ws) - assert.Nil(t, err) - assert.Equal(t, mockVMSpec, workload.spec.VM) - if test.checkSpec { - assert.Equal(t, guestDownloadURL, workload.spec.BaseImageURL) - assert.Equal(t, guestImageFriendlyName, workload.spec.BaseImageName) - assert.Equal(t, defaultHostname, workload.spec.Hostname) - } - - cleanupMockWorkspace(t, ws) - } -} diff --git a/testutil/ciao-down/workloads/ciao-fedora25.yaml b/testutil/ciao-down/workloads/ciao-fedora25.yaml deleted file mode 100644 index d8026495b..000000000 --- a/testutil/ciao-down/workloads/ciao-fedora25.yaml +++ /dev/null @@ -1,206 +0,0 @@ ---- -base_image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/Fedora-Cloud-Base-25-1.3.x86_64.qcow2 -base_image_name: Fedora 25 -hostname: singlevm -needs_nested_vm: true -vm: -{{with .GoPath}} - mounts: - - tag: hostgo - security_model: passthrough - path: {{.}} -{{end}} -... ---- -{{ define "GOPATH" }}{{with .GoPath}}{{$.MountPath "hostgo"}}{{else}}/home/{{.User}}/go{{end}}{{end}} -#cloud-config -write_files: -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end -}} -{{with .HTTPProxy}} - - content: | - [main] - gpgcheck=1 - installonly_limit=3 - clean_requirements_on_remove=True - proxy={{.}} - path: /etc/dnf/dnf.conf -{{end}} -{{- if len $.HTTPProxy }} - - content: | - [Service] - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - path: /etc/systemd/system/docker.service.d/http-proxy.conf -{{- end}} - - content: | - - To run Single VM: - - cd {{template "GOPATH" .}}/src/github.com/ciao-project/ciao/testutil/singlevm - ./setup.sh - path: /etc/motd - permissions: '0755' - -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}true{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - hostnamectl set-hostname singlevm - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - {{beginTask . "Installing DNF plugins core"}} - - dnf -y install dnf-plugins-core - - {{endTaskCheck .}} - - - {{beginTask . "Setting up docker repo"}} - - dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo - - {{endTaskCheck .}} - - - {{beginTask . "Updating dnf cache"}} - - dnf makecache fast - - {{endTaskCheck .}} - - - {{beginTask . "Installing docker"}} - - dnf install -y docker-ce - - {{endTaskCheck .}} - - - groupadd docker - - gpasswd -a {{.User}} docker - - - {{beginTask . "Starting docker"}} - - systemctl start docker - - {{endTaskCheck .}} - - - {{beginTask . "Installing wget"}} - - dnf install -y wget - - {{endTaskCheck .}} - - - {{beginTask . "Installing Git"}} - - dnf install -y git - - {{endTaskCheck .}} - - - {{beginTask . "Downloading Go" }} - - {{download . "https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz" "/tmp/go1.9.linux-amd64.tar.gz"}} - - {{endTaskCheck .}} - - - {{beginTask . "Unpacking Go" }} - - tar -C /usr/local -xzf /tmp/go1.9.linux-amd64.tar.gz - - {{endTaskCheck .}} - - - rm /tmp/go1.9.linux-amd64.tar.gz - - - {{beginTask . "Installing GCC"}} - - dnf install -y gcc - - {{endTaskCheck .}} - - - {{beginTask . "Installing Make"}} - - dnf install -y make - - {{endTaskCheck .}} - - - {{beginTask . "Installing QEMU"}} - - dnf install -y qemu-system-x86 qemu-img - - {{endTaskCheck .}} - - - {{beginTask . "Installing xorriso"}} - - dnf install -y xorriso - - {{endTaskCheck .}} - - - {{beginTask . "Installing dnsmasq"}} - - dnf install -y dnsmasq - - {{endTaskCheck .}} - - - {{beginTask . "Installing ceph-common"}} - - dnf install -y ceph-common - - {{endTaskCheck .}} - - - {{beginTask . "Installing Python devel"}} - - dnf install -y python-devel python-pip - - {{endTaskCheck .}} - - - {{beginTask . "Redhat RPM configrpm"}} - - dnf install -y redhat-rpm-config - - {{endTaskCheck .}} - - - {{beginTask . "Building ciao" }} - - sudo -u {{.User}} {{proxyVars .}} /usr/local/go/bin/go get github.com/ciao-project/ciao/... - - {{endTaskCheck .}} - - - {{beginTask . "Installing Go development utils"}} - - sudo -u {{.User}} {{proxyVars .}} /usr/local/go/bin/go get github.com/fzipp/gocyclo github.com/gordonklaus/ineffassign github.com/golang/lint/golint github.com/client9/misspell/cmd/misspell - - {{endTaskCheck .}} - - - chown {{.User}}:{{.User}} -R {{.User}}/go - - - {{beginTask . "Pulling ceph/demo" }} - - {{proxyVars .}} docker pull ceph/demo - - {{endTaskCheck .}} - - - mkdir -p /home/{{.User}}/local - - - {{beginTask . "Downloading Fedora-Cloud-Base-24-1.2.x86_64.qcow2"}} - - {{download . "https://download.fedoraproject.org/pub/fedora/linux/releases/24/CloudImages/x86_64/images/Fedora-Cloud-Base-24-1.2.x86_64.qcow2" (printf "/home/%s/local/Fedora-Cloud-Base-24-1.2.x86_64.qcow2" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading xenial-server-cloudimg-amd64-disk1.img"}} - - {{download . "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img" (printf "/home/%s/local/xenial-server-cloudimg-amd64-disk1.img" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading CNCI image" }} - - {{download . "https://download.clearlinux.org/demos/ciao/clear-8260-ciao-networking.img.xz" (printf "/home/%s/local/clear-8260-ciao-networking.img.xz" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading latest clear cloud image" }} - - LATEST=$({{proxyVars .}} wget -q -O - https://download.clearlinux.org/latest) && {{proxyVars .}} wget https://download.clearlinux.org/releases/"$LATEST"/clear/clear-"$LATEST"-cloud.img.xz -O /home/{{.User}}/local/clear-"$LATEST"-cloud.img.xz - - {{endTaskCheck .}} - - - cd /home/{{.User}}/local && xz -T0 --decompress *.xz - - - chown {{.User}}:{{.User}} -R /home/{{.User}}/local - -{{if len .GitUserName}} - - {{beginTask . "Setting git user.name"}} - - sudo -u {{.User}} git config --global user.name "{{.GitUserName}}" - - {{endTaskCheck .}} -{{end}} - -{{if len .GitEmail}} - - {{beginTask . "Setting git user.email"}} - - sudo -u {{.User}} git config --global user.email {{.GitEmail}} - - {{endTaskCheck .}} -{{end}} - - - echo "export GOPATH={{template "GOPATH" . }}" >> /home/{{.User}}/.profile - - echo "export PATH=$PATH:{{template "GOPATH" . }}/bin:/usr/local/go/bin" >> /home/{{.User}}/.profile - - echo "source /home/{{.User}}/.profile" >> /home/{{.User}}/.bashrc - - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CC Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} - diff --git a/testutil/ciao-down/workloads/ciao.yaml b/testutil/ciao-down/workloads/ciao.yaml deleted file mode 100644 index 44c0e3622..000000000 --- a/testutil/ciao-down/workloads/ciao.yaml +++ /dev/null @@ -1,207 +0,0 @@ ---- -base_image_url: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img -base_image_name: Ubuntu 16.04 -needs_nested_vm: true -vm: -{{with .GoPath}} - mounts: - - tag: hostgo - security_model: passthrough - path: {{.}} -{{end}} -... ---- -{{- define "ENV" -}} -{{proxyVars .}} -{{- print " DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true " -}} -{{end}} -{{ define "GOPATH" }}{{with .GoPath}}{{$.MountPath "hostgo"}}{{else}}/home/{{.User}}/go{{end}}{{end}} -#cloud-config -write_files: -{{- if len $.HTTPProxy }} - - content: | - [Service] - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - path: /etc/systemd/system/docker.service.d/http-proxy.conf -{{- end}} -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end}} - - content: | - #!/bin/sh - printf "\n" - printf "To run Single VM:\n" - printf "\n" - printf "cd {{template "GOPATH" .}}/src/github.com/ciao-project/ciao/testutil/singlevm\n" - printf "./setup.sh\n" - printf "\n" - path: /etc/update-motd.d/10-ciao-help-text - permissions: '0755' - - content: | - deb https://apt.dockerproject.org/repo ubuntu-xenial main - path: /etc/apt/sources.list.d/docker.list - - content: | - deb http://apt.kubernetes.io/ kubernetes-xenial main - path: /etc/apt/sources.list.d/kubernetes.list - -apt: -{{- if len $.HTTPProxy }} - proxy: "{{$.HTTPProxy}}" -{{- end}} -{{- if len $.HTTPSProxy }} - https_proxy: "{{$.HTTPSProxy}}" -{{- end}} -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}true{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - chown {{.User}}:{{.User}} /home/{{.User}} -{{- with .MountPath "hostui"}} - - chown {{$.User}}:{{$.User}} {{.}} -{{- end}} - - rm /etc/update-motd.d/10-help-text /etc/update-motd.d/51-cloudguest - - rm /etc/update-motd.d/90-updates-available - - rm /etc/legal - - - {{beginTask . "Downloading Go" }} - - {{download . "https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz" "/tmp/go1.9.linux-amd64.tar.gz"}} - - {{endTaskCheck .}} - - - {{beginTask . "Unpacking Go" }} - - tar -C /usr/local -xzf /tmp/go1.9.linux-amd64.tar.gz - - {{endTaskCheck .}} - - - rm /tmp/go1.9.linux-amd64.tar.gz - - - groupadd docker - - sudo gpasswd -a {{.User}} docker - - - {{beginTask . "Installing apt-transport-https and ca-certificates" }} - - {{template "ENV" .}}apt-get install apt-transport-https ca-certificates - - {{endTaskCheck .}} - - - {{beginTask . "Add docker GPG key" }} - - {{template "ENV" .}}apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - - {{endTaskCheck .}} - - - {{beginTask . "Add Google GPG key" }} - - {{template "ENV" .}}curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - - {{endTaskCheck .}} - - - {{beginTask . "Retrieving updated list of packages"}} - - {{template "ENV" .}}apt-get update - - {{endTaskCheck .}} - - - {{beginTask . "Installing Docker"}} - - {{template "ENV" .}}apt-get install docker-engine -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing kubectl"}} - - {{template "ENV" .}}apt-get install kubectl -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing GCC"}} - - {{template "ENV" .}}apt-get install gcc -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Make"}} - - {{template "ENV" .}}apt-get install make -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing QEMU"}} - - {{template "ENV" .}}apt-get install qemu-system-x86 -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing xorriso" }} - - {{template "ENV" .}}apt-get install xorriso -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing ceph-common"}} - - {{template "ENV" .}}apt-get install ceph-common -y - - {{endTaskCheck .}} - - - {{beginTask . "Auto removing unused components"}} - - {{template "ENV" .}}apt-get auto-remove -y - - {{endTaskCheck .}} - - - {{beginTask . "Building ciao" }} - - sudo -u {{.User}} {{template "ENV" .}} GOPATH={{template "GOPATH" .}} /usr/local/go/bin/go get github.com/ciao-project/ciao/... - - {{endTaskCheck .}} - - - {{beginTask . "Installing Go development utils"}} - - sudo -u {{.User}} {{template "ENV" .}} GOPATH={{template "GOPATH" .}} /usr/local/go/bin/go get github.com/fzipp/gocyclo github.com/gordonklaus/ineffassign github.com/golang/lint/golint github.com/client9/misspell/cmd/misspell - - {{endTaskCheck .}} - - - chown {{.User}}:{{.User}} -R {{template "GOPATH" .}} - - - {{beginTask . "Pulling ceph/demo" }} - - {{template "ENV" .}} docker pull ceph/demo - - {{endTaskCheck .}} - - - mkdir -p /home/{{.User}}/local - - - {{beginTask . "Downloading Fedora-Cloud-Base-24-1.2.x86_64.qcow2"}} - - {{download . "https://download.fedoraproject.org/pub/fedora/linux/releases/24/CloudImages/x86_64/images/Fedora-Cloud-Base-24-1.2.x86_64.qcow2" (printf "/home/%s/local/Fedora-Cloud-Base-24-1.2.x86_64.qcow2" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading xenial-server-cloudimg-amd64-disk1.img"}} - - {{download . "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img" (printf "/home/%s/local/xenial-server-cloudimg-amd64-disk1.img" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading CNCI image" }} - - {{download . "https://download.clearlinux.org/demos/ciao/clear-8260-ciao-networking.img.xz" (printf "/home/%s/local/clear-8260-ciao-networking.img.xz" .User)}} - - {{endTaskCheck .}} - - - {{beginTask . "Downloading latest clear cloud image" }} - - LATEST=$({{template "ENV" .}} curl -s https://download.clearlinux.org/latest) && {{template "ENV" .}} wget https://download.clearlinux.org/releases/"$LATEST"/clear/clear-"$LATEST"-cloud.img.xz -O /home/{{.User}}/local/clear-"$LATEST"-cloud.img.xz - - {{endTaskCheck .}} - - - cd /home/{{.User}}/local && xz -T0 --decompress *.xz - - - chown {{.User}}:{{.User}} -R /home/{{.User}}/local - -{{if len .GitUserName}} - - {{beginTask . "Setting git user.name"}} - - sudo -u {{.User}} git config --global user.name "{{.GitUserName}}" - - {{endTaskCheck .}} -{{end}} - -{{if len .GitEmail}} - - {{beginTask . "Setting git user.email"}} - - sudo -u {{.User}} git config --global user.email {{.GitEmail}} - - {{endTaskCheck .}} -{{end}} - - - echo "export GOPATH={{template "GOPATH" . }}" >> /home/{{.User}}/.profile - - echo "export PATH=$PATH:{{template "GOPATH" . }}/bin:/usr/local/go/bin" >> /home/{{.User}}/.profile - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CIAO Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} -... diff --git a/testutil/ciao-down/workloads/clearcontainers-2.x.yaml b/testutil/ciao-down/workloads/clearcontainers-2.x.yaml deleted file mode 100644 index 79989e0eb..000000000 --- a/testutil/ciao-down/workloads/clearcontainers-2.x.yaml +++ /dev/null @@ -1,199 +0,0 @@ ---- -base_image_url: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img -base_image_name: Ubuntu 16.04 -hostname: singlevm -needs_nested_vm: true -vm: -{{with .GoPath}} - mounts: - - tag: hostgo - security_model: none - path: {{.}} -{{end}} -... ---- -{{- define "ENV" -}} -{{proxyVars .}} -{{- print " DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true " -}} -{{end}} -{{ define "GOPATH" }}{{with .GoPath}}{{$.MountPath "hostgo"}}{{else}}/home/{{.User}}/go{{end}}{{end}} -#cloud-config -write_files: -{{- if len $.HTTPProxy }} - - content: | - [Service] - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - path: /etc/systemd/system/docker.service.d/http-proxy.conf -{{- end}} -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end}} - - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -D --add-runtime cor=/usr/bin/cc-oci-runtime --default-runtime=cor - path: /etc/systemd/system/docker.service.d/clr-containers.conf - - content: | - #!/bin/sh - printf "\n" - printf "\n" - printf "Your go code is at {{template "GOPATH" .}}\n" - printf "You can also edit your code on your host system \n" - printf "To build cc-oci-runtime from sources \n" - printf "go get -d github.com/01org/cc-oci-runtime/... \n" - printf "cd {{template "GOPATH" .}}/src/github.com/01org/cc-oci-runtime \n" - printf "./autogen.sh --with-cc-kernel=/usr/share/clear-containers/vmlinux.container --with-cc-image=/usr/share/clear-containers/clear-containers.img \n" - printf "make \n" - printf "make check \n" - printf "make install \n" - printf "\n" - printf "\n" - path: /etc/update-motd.d/10-ciao-help-text - permissions: '0755' - - content: | - deb https://apt.dockerproject.org/repo ubuntu-xenial main - path: /etc/apt/sources.list.d/docker.list - -apt: -{{- if len $.HTTPProxy }} - proxy: "{{$.HTTPProxy}}" -{{- end}} -{{- if len $.HTTPSProxy }} - https_proxy: "{{$.HTTPSProxy}}" -{{- end}} -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}true{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - chown {{.User}}:{{.User}} /home/{{.User}} - - rm /etc/update-motd.d/10-help-text /etc/update-motd.d/51-cloudguest - - rm /etc/update-motd.d/90-updates-available - - rm /etc/legal - - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - echo "GOPATH=\"{{template "GOPATH" .}}\"" >> /etc/environment - - echo "PATH=\"$PATH:/usr/local/go/bin:{{template "GOPATH" .}}/bin\"" >> /etc/environment - - - {{beginTask . "Downloading Go" }} - - {{download . "https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz" "/tmp/go1.7.4.linux-amd64.tar.gz"}} - - {{endTaskCheck .}} - - - {{beginTask . "Unpacking Go" }} - - tar -C /usr/local -xzf /tmp/go1.7.4.linux-amd64.tar.gz - - {{endTaskCheck .}} - - - rm /tmp/go1.7.4.linux-amd64.tar.gz - - - groupadd docker - - sudo gpasswd -a {{.User}} docker - - - {{beginTask . "Installing apt-transport-https and ca-certificates" }} - - {{template "ENV" .}}sudo apt-get -y install apt-transport-https ca-certificates - - {{endTaskCheck .}} - - - {{beginTask . "Add docker GPG key" }} - - {{template "ENV" .}}curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - {{endTaskCheck .}} - - - {{beginTask . "Adding docker repo"}} - - {{template "ENV" .}} sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - {{endTaskCheck .}} - - - {{beginTask . "Add Clear Containers OBS Repository "}} - - sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/clearlinux:/preview:/clear-containers-2.1/xUbuntu_16.04/ /' >> /etc/apt/sources.list.d/cc-oci-runtime.list" - - {{template "ENV" .}}curl -fsSL http://download.opensuse.org/repositories/home:clearlinux:preview:clear-containers-2.1/xUbuntu_16.04/Release.key | sudo apt-key add - - - {{endTaskCheck .}} - - - {{beginTask . "Retrieving updated list of packages"}} - - {{template "ENV" .}}sudo apt-get update - - {{endTaskCheck .}} - - - {{beginTask . "Installing Clear Containers Runtime"}} - - {{template "ENV" .}}sudo apt-get install cc-oci-runtime -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Docker"}} - - {{template "ENV" .}}sudo apt-get install -y --allow-downgrades --allow-unauthenticated docker-engine=1.12.1-0~xenial - - {{endTaskCheck .}} - - - {{beginTask . "Start Clear Containers Runtime"}} - - sudo systemctl daemon-reload - - sudo systemctl restart docker - - sudo systemctl enable cc-proxy.socket - - sudo systemctl start cc-proxy.socket - - {{endTaskCheck .}} - - - {{beginTask . "Installing GCC"}} - - {{template "ENV" .}}apt-get install gcc -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Make"}} - - {{template "ENV" .}}apt-get install make -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing QEMU" }} - - {{template "ENV" .}}apt-get install qemu-system-x86 -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing xorriso"}} - - {{template "ENV" .}}apt-get install xorriso -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Clear Containers development tools"}} - - {{template "ENV" .}}apt-get install build-essential python zlib1g-dev libcap-ng-dev libglib2.0-dev libpixman-1-dev libattr1-dev libcap-dev autoconf libtool libjson-glib-dev uuid-dev check bats libdevmapper-dev file apt-utils wget valgrind lcov libmnl-dev cppcheck libtap-formatter-html-perl -y - - {{endTaskCheck .}} - - - {{beginTask . "Auto removing unused development components"}} - - {{template "ENV" .}}apt-get auto-remove -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Go development utils"}} - - sudo -u {{.User}} {{template "ENV" .}} GOPATH={{template "GOPATH" .}} /usr/local/go/bin/go get github.com/mattn/goveralls golang.org/x/tools/cmd/cover github.com/pierrre/gotestcover github.com/fzipp/gocyclo github.com/gordonklaus/ineffassign github.com/golang/lint/golint github.com/client9/misspell/cmd/misspell github.com/ciao-project/ciao/test-cases github.com/opencontainers/runc/libcontainer/configs - - {{endTaskCheck .}} - - - chown {{.User}}:{{.User}} -R {{template "GOPATH" .}} - -{{if len .GitUserName}} - - {{beginTask . "Setting git user.name"}} - - sudo -u {{.User}} git config --global user.name "{{.GitUserName}}" - - {{endTaskCheck .}} -{{end}} - -{{if len .GitEmail}} - - {{beginTask . "Setting git user.email"}} - - sudo -u {{.User}} git config --global user.email {{.GitEmail}} - - {{endTaskCheck .}} -{{end}} - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CC Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} -... diff --git a/testutil/ciao-down/workloads/clearcontainers-3.x.yaml b/testutil/ciao-down/workloads/clearcontainers-3.x.yaml deleted file mode 100644 index 5ebea3b7e..000000000 --- a/testutil/ciao-down/workloads/clearcontainers-3.x.yaml +++ /dev/null @@ -1,249 +0,0 @@ ---- -base_image_url: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img -base_image_name: Ubuntu 16.04 -hostname: singlevm -needs_nested_vm: true -vm: -{{with .GoPath}} - mounts: - - tag: hostgo - security_model: none - path: {{.}} -{{end}} -... ---- -{{- define "ENV" -}} -{{proxyVars .}} -{{- print " DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true " -}} -{{end}} -{{ define "GOPATH" }}{{with .GoPath}}{{$.MountPath "hostgo"}}{{else}}/home/{{.User}}/go{{end}}{{end}} -#cloud-config -write_files: -{{- if len $.HTTPProxy }} - - content: | - [Service] - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - path: /etc/systemd/system/docker.service.d/http-proxy.conf -{{- end}} -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end}} - - content: | - [Service] - ExecStart= - ExecStart=/usr/bin/dockerd -D --add-runtime cor=/usr/bin/cc-runtime --default-runtime=cor - path: /etc/systemd/system/docker.service.d/clr-containers.conf - - content: | - #!/bin/sh - printf "\n" - printf "\n" - printf "Your go code is at {{template "GOPATH" .}}\n" - printf "You can also edit your code on your host system \n" - printf "To build cc-runtime from sources \n" - printf "go get -d github.com/clearcontainers/runtime... \n" - printf "cd {{template "GOPATH" .}}/src/github.com/clearcontainers/runtime \n" - printf "make build-cc-system\n" - printf "sudo -E PATH=$PATH make install-cc-system \n" - printf "\n" - printf "\n" - path: /etc/update-motd.d/10-ciao-help-text - permissions: '0755' - - content: | - [Unit] - Description=OCI-based implementation of Kubernetes Container Runtime Interface - Documentation=https://github.com/kubernetes-incubator/cri-o - - [Service] - ExecStart=/usr/bin/crio --debug - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - Restart=on-failure - RestartSec=5 - - [Install] - WantedBy=multi-user.target - path: /etc/systemd/system/crio.service - - content: | - deb https://apt.dockerproject.org/repo ubuntu-xenial main - path: /etc/apt/sources.list.d/docker.list - -apt: -{{- if len $.HTTPProxy }} - proxy: "{{$.HTTPProxy}}" -{{- end}} -{{- if len $.HTTPSProxy }} - https_proxy: "{{$.HTTPSProxy}}" -{{- end}} -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}true{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - chown {{.User}}:{{.User}} /home/{{.User}} - - rm /etc/update-motd.d/10-help-text /etc/update-motd.d/51-cloudguest - - rm /etc/update-motd.d/90-updates-available - - rm /etc/legal - - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - echo "GOPATH=\"{{template "GOPATH" .}}\"" >> /etc/environment - - echo "PATH=\"$PATH:/usr/local/go/bin:{{template "GOPATH" .}}/bin\"" >> /etc/environment - - - {{beginTask . "Downloading Go" }} - - {{download . "https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz" "/tmp/go1.8.3.linux-amd64.tar.gz"}} - - {{endTaskCheck .}} - - - {{beginTask . "Unpacking Go" }} - - tar -C /usr/local -xzf /tmp/go1.8.3.linux-amd64.tar.gz - - {{endTaskCheck .}} - - - rm /tmp/go1.8.3.linux-amd64.tar.gz - - - groupadd docker - - sudo gpasswd -a {{.User}} docker - - - {{beginTask . "Installing apt-transport-https and ca-certificates" }} - - {{template "ENV" .}}sudo apt-get -y install apt-transport-https ca-certificates - - {{endTaskCheck .}} - - - {{beginTask . "Add docker GPG key" }} - - {{template "ENV" .}}curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - {{endTaskCheck .}} - - - {{beginTask . "Adding docker repo"}} - - {{template "ENV" .}} sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - {{endTaskCheck .}} - - - {{beginTask . "Add Clear Containers OBS Repository "}} - - sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' >> /etc/apt/sources.list.d/clear-containers.list" - - {{template "ENV" .}}curl -fsSL http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/Release.key | sudo apt-key add - - - {{endTaskCheck .}} - - - {{beginTask . "Add CRI-O OS dependency repo"}} - - {{template "ENV" .}}sudo add-apt-repository "ppa:alexlarsson/flatpak" - - {{endTaskCheck .}} - - - {{beginTask . "Add CRI-O repo"}} - - {{template "ENV" .}}sudo add-apt-repository "ppa:projectatomic/ppa" - - {{endTaskCheck .}} - - - {{beginTask . "Add K8S repo"}} - - sudo sh -c "echo 'deb http://apt.kubernetes.io/ kubernetes-xenial-unstable main' >> /etc/apt/sources.list.d/kubernetes.list" - - {{template "ENV" .}}curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - - - {{beginTask . "Retrieving updated list of packages"}} - - {{template "ENV" .}}sudo apt-get update - - {{endTaskCheck .}} - - - {{beginTask . "Installing Clear Containers Runtime"}} - - {{template "ENV" .}}sudo apt-get install -y cc-runtime cc-proxy cc-shim - - {{endTaskCheck .}} - - - {{beginTask . "Installing Docker"}} - - {{template "ENV" .}}sudo apt-get install -y --allow-downgrades --allow-unauthenticated docker-engine=1.12.1-0~xenial - - {{endTaskCheck .}} - - - {{beginTask . "Start Clear Containers Runtime"}} - - sudo systemctl daemon-reload - - sudo systemctl restart docker - - sudo systemctl enable cc-proxy.socket - - sudo systemctl start cc-proxy.socket - - {{endTaskCheck .}} - - - {{beginTask . "Installing GCC"}} - - {{template "ENV" .}}apt-get install gcc -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Make"}} - - {{template "ENV" .}}apt-get install make -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing QEMU" }} - - {{template "ENV" .}}apt-get install qemu-system-x86 -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing xorriso"}} - - {{template "ENV" .}}apt-get install xorriso -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing Clear Containers development tools"}} - - {{template "ENV" .}}apt-get install build-essential python zlib1g-dev libcap-ng-dev libglib2.0-dev libpixman-1-dev libattr1-dev libcap-dev autoconf libtool libjson-glib-dev uuid-dev check bats libdevmapper-dev file apt-utils wget valgrind lcov libmnl-dev cppcheck libtap-formatter-html-perl -y - - {{endTaskCheck .}} - - - {{beginTask . "Installing CRI-O dependencies"}} - - {{template "ENV" .}}apt-get install -y socat libgpgme11 libostree-1-1 - - {{endTaskCheck .}} - - - {{beginTask . "Installing K8S"}} - - {{template "ENV" .}}apt-get install -y kubelet=1.7.5-00 kubeadm=1.7.5-00 kubectl=1.7.5-00 - - {{template "ENV" .}}sudo apt-mark hold kubelet kubeadm kubectl - - {{endTaskCheck .}} - - - {{beginTask . "Installing CRI-O"}} - - {{template "ENV" .}}apt-get install -y cri-o - - {{endTaskCheck .}} - - - {{beginTask . "Auto removing unused development components"}} - - {{template "ENV" .}}apt-get auto-remove -y - - {{endTaskCheck .}} - - - {{beginTask . "Configuring CRIO for Clear Containers"}} - - sudo sed -i 's/runtime_untrusted_workload = ""/runtime_untrusted_workload = "\/usr\/bin\/cc-runtime"/' "/etc/crio/crio.conf" - - {{endTaskCheck .}} - - - {{beginTask . "Configuring K8S for CRIO"}} - - sudo sed -i '/KUBELET_CADVISOR_ARGS/a Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=/var/run/crio.sock --runtime-request-timeout=30m"' "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - - {{endTaskCheck .}} - - - {{beginTask . "Start CRI-O daemon"}} - - sudo systemctl enable crio - - sudo systemctl start crio - - {{endTaskCheck .}} - - - {{beginTask . "Installing Go development utils"}} - - sudo -u {{.User}} {{template "ENV" .}} GOPATH={{template "GOPATH" .}} /usr/local/go/bin/go get github.com/mattn/goveralls golang.org/x/tools/cmd/cover github.com/pierrre/gotestcover github.com/fzipp/gocyclo github.com/gordonklaus/ineffassign github.com/golang/lint/golint github.com/client9/misspell/cmd/misspell github.com/ciao-project/ciao/test-cases github.com/opencontainers/runc/libcontainer/configs - - {{endTaskCheck .}} - - - chown {{.User}}:{{.User}} -R {{template "GOPATH" .}} - -{{if len .GitUserName}} - - {{beginTask . "Setting git user.name"}} - - sudo -u {{.User}} git config --global user.name "{{.GitUserName}}" - - {{endTaskCheck .}} -{{end}} - -{{if len .GitEmail}} - - {{beginTask . "Setting git user.email"}} - - sudo -u {{.User}} git config --global user.email {{.GitEmail}} - - {{endTaskCheck .}} -{{end}} - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CC Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} -... diff --git a/testutil/ciao-down/workloads/fedora25.yaml b/testutil/ciao-down/workloads/fedora25.yaml deleted file mode 100644 index 12397c690..000000000 --- a/testutil/ciao-down/workloads/fedora25.yaml +++ /dev/null @@ -1,57 +0,0 @@ ---- -base_image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/Fedora-Cloud-Base-25-1.3.x86_64.qcow2 -base_image_name: Fedora 25 -hostname: singlevm -... ---- -#cloud-config -write_files: -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end -}} -{{with .HTTPProxy}} - - content: | - [main] - gpgcheck=1 - installonly_limit=3 - clean_requirements_on_remove=True - proxy={{.}} - path: /etc/dnf/dnf.conf -{{end}} - -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}false{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - hostnamectl set-hostname {{.Hostname}} - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CC Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} - diff --git a/testutil/ciao-down/workloads/xenial.yaml b/testutil/ciao-down/workloads/xenial.yaml deleted file mode 100644 index c1a2e71ce..000000000 --- a/testutil/ciao-down/workloads/xenial.yaml +++ /dev/null @@ -1,62 +0,0 @@ ---- -base_image_url: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img -base_image_name: Ubuntu 16.04 -... ---- -{{- define "ENV" -}} -{{proxyVars .}} -{{- print " DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true " -}} -{{end}} -#cloud-config -write_files: -{{- if len $.HTTPProxy }} - - content: | - [Service] - Environment="HTTP_PROXY={{$.HTTPProxy}}"{{if len .HTTPSProxy}} "HTTPS_PROXY={{.HTTPSProxy}}{{end}}"{{if len .NoProxy}} "NO_PROXY={{.NoProxy}},{{.Hostname}}{{end}}" - path: /etc/systemd/system/docker.service.d/http-proxy.conf -{{- end}} -{{with proxyEnv . 5}} - - content: | -{{.}} - path: /etc/environment -{{end}} - -apt: -{{- if len $.HTTPProxy }} - proxy: "{{$.HTTPProxy}}" -{{- end}} -{{- if len $.HTTPSProxy }} - https_proxy: "{{$.HTTPSProxy}}" -{{- end}} -package_upgrade: {{with .PackageUpgrade}}{{.}}{{else}}false{{end}} - -runcmd: - - {{beginTask . "Booting VM"}} - - {{endTaskOk . }} - - - {{beginTask . (printf "Adding %s to /etc/hosts" .Hostname) }} - - echo "127.0.0.1 {{.Hostname}}" >> /etc/hosts - - {{endTaskCheck .}} - -{{range .Mounts}} - - mkdir -p {{.Path}} - - sudo chown {{$.User}}:{{$.User}} {{.Tag}} - - echo "{{.Tag}} {{.Path}} 9p x-systemd.automount,x-systemd.device-timeout=10,nofail,trans=virtio,version=9p2000.L 0 0" >> /etc/fstab -{{end}} -{{range .Mounts}} - - {{beginTask $ (printf "Mounting %s" .Path) }} - - mount {{.Path}} - - {{endTaskCheck $}} -{{end}} - - - {{finished .}} - -users: - - name: {{.User}} - gecos: CIAO Demo User - lock-passwd: true - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: - - {{.PublicKey}} -... diff --git a/testutil/singlevm/setup.sh b/testutil/singlevm/setup.sh index 21eeab98e..521c586fe 100755 --- a/testutil/singlevm/setup.sh +++ b/testutil/singlevm/setup.sh @@ -140,12 +140,12 @@ if [ -x "$(command -v ip)" ]; then sudo ip link set dev "$ciao_bridge" up sudo ip -d link show "$ciao_bridge" sudo iptables -A FORWARD -p all -i ciaovlan -j ACCEPT - #Do this only in the case of ciao-down as it can potentially + #Do this only in the case of ccloudvm as it can potentially #open up the machine. On bare metal the user will need to explicitly #add this rule if [ "$ciao_host" == "singlevm" ]; then sudo iptables -A FORWARD -p all -i "$ext_int" -j ACCEPT - #NAT out all the traffic departing ciao-down + #NAT out all the traffic departing ccloudvm sudo iptables -t nat -A POSTROUTING -o "$ext_int" -j MASQUERADE fi