Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
rkoster committed Jul 4, 2017
2 parents 04747cf + 0d758fb commit 02e2efb
Show file tree
Hide file tree
Showing 15 changed files with 250 additions and 70 deletions.
17 changes: 12 additions & 5 deletions bin/bucc
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ repo_root=$(dirname $(realpath "$0") | rev | cut -c5- | rev)
state=${repo_root}/state
mkdir -p $state/manifests
manifest="${repo_root}/src/bosh-deployment/bosh.yml"
ssh_private_key="${state}/ssh.key"
state_arg="--state ${state}/state.json"
vars_store=${state}/creds.yml
vars_store_arg="--vars-store ${vars_store}"
Expand Down Expand Up @@ -45,7 +44,7 @@ lite() {
}

cpis() {
find ${repo_root}/ops/cpis -mindepth 1 -maxdepth 1 ! -type l -exec basename {} +;
find ${repo_root}/ops/cpis -mindepth 1 -maxdepth 1 ! -type l -exec basename -a {} +;
}

validate_vars_file() {
Expand Down Expand Up @@ -236,6 +235,9 @@ env() {
echo "export BOSH_CA_CERT='$(cat $(ca_cert))'"
echo "export BOSH_CLIENT=$(get_var bosh_client)"
echo "export BOSH_CLIENT_SECRET=$(get_var bosh_client_secret)"
echo "export BOSH_GW_USER=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name)"
echo "export BOSH_GW_HOST=$(ip)"
echo "export BOSH_GW_PRIVATE_KEY=$(ssh_private_key)"
}

routes() {
Expand Down Expand Up @@ -324,11 +326,16 @@ ip() {
int --path /networks/name=default/subnets/0/static/0
}

_ssh() {
local user=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name)
ssh_private_key() {
local ssh_private_key="${state}/ssh.key"
store_int --path /jumpbox_ssh/private_key > $ssh_private_key
chmod 600 $ssh_private_key
ssh -i $ssh_private_key $user@$(ip) -t 'command; sudo su -'
echo $ssh_private_key
}

_ssh() {
local user=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name)
ssh -i $(ssh_private_key) $user@$(ip) -t 'command; sudo su -'
}

header() {
Expand Down
121 changes: 91 additions & 30 deletions ci/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ meta:
owner: (( param "Please specify the name of the user / organization that owns the Github repository" ))
repo: (( param "Please specify the name of the Github repository" ))
branch: develop
master-branch: master
private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" ))
access_token: (( param "Please generate a Personal Access Token and specify it here" ))

Expand Down Expand Up @@ -63,14 +64,13 @@ jobs:
- { get: bucc-ci }
- task: bucc-manifests
file: bucc-ci/ci/tasks/bucc-manifests/task.yml

# on_failure:
# put: notify
# params:
# channel: (( grab meta.slack.channel ))
# username: (( grab meta.slack.username ))
# icon_url: (( grab meta.slack.icon ))
# text: '(( concat meta.slack.fail_url " " meta.pipeline ": test job failed" ))'
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": sanity-check job failed" ))'

- name: test
public: true
Expand Down Expand Up @@ -104,6 +104,14 @@ jobs:
ensure:
task: bucc-down-vsphere
file: bucc-ci/ci/tasks/bucc-down/task.yml
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": test job failed" ))'


- name: rc
public: true
Expand All @@ -114,13 +122,13 @@ jobs:
- { get: version, trigger: true, params: {pre: rc} }
- put: version
params: {file: version/number}
# on_failure:
# put: notify
# params:
# channel: (( grab meta.slack.channel ))
# username: (( grab meta.slack.username ))
# icon_url: (( grab meta.slack.icon ))
# text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))'
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))'

- name: bump-patch
public: true
Expand Down Expand Up @@ -174,6 +182,9 @@ jobs:
- { get: version, passed: [rc], params: {bump: final} }
- { get: bucc, passed: [rc] }
- { get: bucc-ci }
- { get: bucc-stable }
- task: generate-release-notes
file: bucc-ci/ci/tasks/generate-release-notes/task.yml
- task: release
file: bucc-ci/ci/tasks/shipit/task.yml
- put: version
Expand All @@ -182,20 +193,20 @@ jobs:
params:
rebase: true
repository: bucc-out
tag: github/tag
tag: release/tag
- name: github
put: github
params:
name: github/name
tag: github/tag
body: github/notes.md
# on_failure:
# put: notify
# params:
# channel: (( grab meta.slack.channel ))
# username: (( grab meta.slack.username ))
# icon_url: (( grab meta.slack.icon ))
# text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))'
name: release/name
tag: release/tag
body: release-notes/notes.md
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))'

- name: compile-releases
public: true
Expand All @@ -218,6 +229,13 @@ jobs:
- try:
put: compiled-releases-credhub
params: { file: "compiled-releases/credhub*.tgz", acl: public-read }
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": compiled-releases job failed" ))'

- name: concourse-compiled-release-ops-file
public: true
Expand All @@ -238,6 +256,13 @@ jobs:
params:
rebase: true
repository: bucc-out
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-compiled-release-ops-file job failed" ))'

- name: os-conf-compiled-release-ops-file
public: true
Expand All @@ -258,6 +283,13 @@ jobs:
params:
rebase: true
repository: bucc-out
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": os-conf-compiled-release-ops-file job failed" ))'

- name: credhub-compiled-release-ops-file
public: true
Expand All @@ -278,6 +310,13 @@ jobs:
params:
rebase: true
repository: bucc-out
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-compiled-release-ops-file job failed" ))'

- name: bosh-deployment-repo
public: true
Expand All @@ -297,6 +336,13 @@ jobs:
params:
rebase: true
repository: bucc-out
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": bosh-deployment-repo job failed" ))'

- name: concourse-release
public: true
Expand All @@ -310,7 +356,7 @@ jobs:
trigger: true
params:
tarball: false
- task: bosh-deployment
- task: update-concourse-release
file: bucc-ci/ci/tasks/update-concourse-release/task.yml
privileged: true
- task: bucc-manifests
Expand All @@ -319,6 +365,13 @@ jobs:
params:
rebase: true
repository: bucc-out
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-release job failed" ))'

resource_types:
- name: bosh-deployment
Expand All @@ -344,24 +397,32 @@ resources:
type: git
source:
uri: (( grab meta.github.uri ))
branch: master
branch: (( grab meta.github.master-branch ))
private_key: (( grab meta.github.private_key ))
ignore_paths: [ci/*]
submodules: all

- name: bucc-stable
type: git
source:
uri: (( grab meta.github.uri ))
branch: (( grab meta.github.master-branch ))
private_key: (( grab meta.github.private_key ))
tag_filter: "v*"

- name: bucc-ci
type: git
source:
uri: (( grab meta.github.uri ))
branch: (( grab meta.github.branch ))
private_key: (( grab meta.github.private_key ))
paths: [ci/*]
disable_ci_skip: true

- name: bosh-deployment
type: git
source:
uri: https://github.com/cloudfoundry/bosh-deployment.git
branch: master
branch: (( grab meta.github.master-branch ))
ignore_paths: [ci/*]

- name: version
Expand Down
4 changes: 2 additions & 2 deletions ci/settings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@ meta:
access_key_id: (( vault "secret/pipelines/bucc/aws:access_key_id" ))
secret_access_key: (( vault "secret/pipelines/bucc/aws:secret_access_key_id" ))
slack:
webhook: (( vault "secret/pipelines/habitat-boshrelease/slack:webhook" ))
channel: "#cf-community-pipeline"
webhook: (( vault "secret/pipelines/bucc/slack:webhook" ))
channel: "#bucc"
3 changes: 3 additions & 0 deletions ci/tasks/compile-releases/task
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@ generate_compile_manifest() {
continue
fi
if [[ ${compiled_release_url} != *"${version}"*"${stemcell_version}"* ]]; then
already_exist=$(curl -s "https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/?prefix${name}" | grep "${version}-ubuntu-trusty-${stemcell_version}")
if [ -z $alread_exist]; then
releases_to_compile+="- $( echo ${release} | jq -c '.version = "latest"' )\n"
fi
fi
done

Expand Down
83 changes: 83 additions & 0 deletions ci/tasks/generate-release-notes/task
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env bash

set -e
set -o pipefail

custom_notes=$(pwd)/bucc/ci/release_notes.md
notes=$(pwd)/release-notes/notes.md

touch ${custom_notes}
cp ${custom_notes} ${notes}

curl -L >$(which bosh2) https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-2.0.10-linux-amd64 && chmod 755 $(which bosh2)

get_releases() {
mkdir -p state
echo "virtualbox" > state/cpi
echo "foo: bar" > vars.yml
ops=$(find ops/cpis/*/*.yml | grep -v 'virtualbox\|google' | sed 's/^/-o /' | xargs)
spruce json <(echo "releases:"; ./bin/bucc int ${ops} --path /releases) | jq -c -r -M '.releases[]'
}

get_stemcell_version() {
mkdir -p state
echo "virtualbox" > state/cpi
echo "foo: bar" > vars.yml
spruce json <(echo "releases:"; ./bin/bucc int --path /releases) | jq '.releases[0].url' \
| tr '-' '\n' | grep trusty -A 1 | tail -1
}

get_release_url() {
# sort results to prefere cloudfoundry over cloudfoundry-{incubator,community}
local path=$(curl -s bosh.io/releases/ | grep '<a href="/releases/' | cut -d'"' -f2 | cut -d'?' -f1 | grep "/${1}$\|/${1}-release$\|/${1}-boshrelease$" | sort -n | tail -n 1)

if [ ! -z "${path}" ]; then
echo "https://bosh.io${path}?version=${2}"
fi
}

pushd bucc-stable > /dev/null
stable_releases=$(get_releases)
stable_stemcell=$(get_stemcell_version)
popd > /dev/null

pushd bucc > /dev/null
new_releases=$(get_releases)
new_stemcell=$(get_stemcell_version)
popd > /dev/null

notes_releases=""
for release in $(echo ${new_releases} | jq -c '.'); do
name=$(echo ${release} | jq -r '.name')
version=$(echo ${release} | jq -r '.version')
stable_version=$(echo ${stable_releases} | jq -s -r --arg name "${name}" '.[] | select(.name == $name).version')

release_url=$(get_release_url ${name} ${version})
if [ ! -z "${release_url}" ]; then
notes_releases+="| ${name} | [${version}](${release_url}) |"
else
notes_releases+="| ${name} | ${version} |"
fi

if [[ "${version}" != "${stable_version}" ]]; then
notes_releases+=" yes |\n"
else
notes_releases+=" no |\n"
fi
done

if [ ! -z "${notes_releases}" ]; then
echo "## Release updates:" >> ${notes}
echo "| Name | Version | Changed |" >> ${notes}
echo "| ---- | ------- | ------- |" >> ${notes}
echo -e "${notes_releases}" >> ${notes}
fi

if [[ ${stemcell} != ${new_stemcell} ]]; then
echo "## Stemcell update:" >> ${notes}
stemcell_url="https://github.com/cloudfoundry/bosh-linux-stemcell-builder/releases/tag/v${new_stemcell}"
echo -e "Bumped to [${new_stemcell}](${stemcell_url})" >> ${notes}
fi

cat ${notes}

Loading

0 comments on commit 02e2efb

Please sign in to comment.