diff --git a/.circleci/config.yml b/.circleci/config.yml index d4f7491..9ac835b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,140 +1,28 @@ version: 2.1 -parameters: - trigger-project: - type: string - default: "" - trigger-url: - type: string - default: "" - trigger-sha: - type: string - default: "" - trigger-description: - type: string - default: "A Satis build was triggered for an unknown reason." jobs: - build: + test: &job-test docker: - - image: composer/satis - environment: - SATIS: /satis/bin/satis - SATIS_BUILD: /tmp/satis-build/app + - image: docker.io/govcms/govcms-ci steps: - checkout - - run: - name: Build or prepare satis /app - command: | - # If satis build was automated, it would happen here. Currently just copy the committed /app to the build directory. - mkdir -p "${SATIS_BUILD}" - cp -R app/* "${SATIS_BUILD}" - - persist_to_workspace: - root: /tmp/satis-build - paths: - - . + - run: ./.circleci/test.sh - test: - docker: - - image: quay.io/govcms/govcms-ci + test_master: + <<: *job-test environment: - SATIS_BUILD: /tmp/satis-build/app - GOVCMS_SCAFFOLD: /tmp/govcms-build - steps: - - checkout - - attach_workspace: - at: /tmp/satis-build - - run: - name: Test that GovCMS can be built - command: | - # Get package versions required for testing stable release. - VERSION_GOVCMS=$(cat ./satis-config/govcms-stable.json | jq -r '.require | .["govcms/govcms"]') - VERSION_TOOLING=$(cat ./satis-config/govcms-stable.json | jq -r '.require | .["govcms/scaffold-tooling"]') - VERSION_DEV=$(cat ./satis-config/govcms-stable.json | jq -r '.require | .["govcms/require-dev"]') - - echo "--> Starting satis web server on http://localhost:4141" - php -S localhost:4141 -t "${SATIS_BUILD}" > /tmp/phpd.log 2>&1 & - echo "--> Cloning govcms8-scaffold-paas into ${GOVCMS_SCAFFOLD}" - composer create-project --no-install --quiet govcms/govcms8-scaffold-paas "${GOVCMS_SCAFFOLD}" - cd "${GOVCMS_SCAFFOLD}" - composer config secure-http false - - for branch in {"","develop","master"}; do - echo - echo "--> --------------------------------------------------------" - echo "--> Test build GovCMS against http://localhost:4141/${branch}" - echo + SATIS_BRANCH: master - rm -fR vendor && rm -f composer.lock - composer config repositories.govcms composer http://localhost:4141/"${branch}" - if [ "${branch}" = "master" ] || [ "${branch}" = "develop" ] ; then - composer require --no-update \ - govcms/govcms:1.x \ - govcms/scaffold-tooling:dev-"${branch}" \ - govcms/require-dev:dev-"${branch}" \ - symfony/event-dispatcher:"v4.3.11 as v3.4.35" # @todo: remove once govcms/govcms no longer requires "symfony/event-dispatcher:v4.3.11 as v3.4.35" which only works at the root composer.json level. - else - echo -e "--> Expected stable versions, based on the satis config: \n - govcms/govcms:${VERSION_GOVCMS} \n - govcms/scaffold-tooling:${VERSION_TOOLING} \n - govcms/require-dev:${VERSION_DEV}" - composer require --no-update \ - govcms/govcms:"${VERSION_GOVCMS}" \ - govcms/scaffold-tooling:"${VERSION_TOOLING}" \ - govcms/require-dev:"${VERSION_DEV}" \ - symfony/event-dispatcher:"v4.3.11 as v3.4.35" # @todo: remove once govcms/govcms no longer requires "symfony/event-dispatcher:v4.3.11 as v3.4.35" which only works at the root composer.json level. - fi - cat composer.json | jq .require - composer -n --quiet --no-suggest --no-scripts update - composer info | grep ^govcms - done - - deploy: - docker: - - image: quay.io/govcms/govcms-ci + test_develop: + <<: *job-test environment: - SATIS_BUILD: /tmp/satis-build/app - steps: - - checkout - - attach_workspace: - at: /tmp/satis-build - - run: - name: Update github develop branch. - command: | - # If builds were automated, this would push the new satis back to git. - exit 1 - - # Currently just testing a push to a test branch. - git checkout -b test-"${CIRCLE_SHA1}" - rm -Rf app - cp -Rf "${SATIS_BUILD}" . - git add app - git config --global user.email "govhosting@finance.gov.au" - git config --global user.name "GovCMS service account" - # Insecure proof of concept. Key removed between testing. - # @todo use ssh key fingerprint https://github.com/integratedexperts/drupal-dev/blob/8.x/.circleci/config.yml#L127 - git remote set-url origin https://simesy:"${GITHUB_READ_WRITE_TOKEN}"@github.com/govCMS/satis.git - git commit -m"[skip ci] test-commit" - git push origin test-"${CIRCLE_SHA1}" + SATIS_BRANCH: develop workflows: version: 2.1 update: jobs: - - build: - filters: - tags: - ignore: /.*/ - - test: - filters: - tags: - ignore: /.*/ - requires: - - build - - deploy: - filters: - tags: - ignore: /.*/ - branches: - only: - - upstream_changes - - develop - requires: - - test + - test + - test_master + - test_develop diff --git a/.circleci/test.sh b/.circleci/test.sh new file mode 100755 index 0000000..cbc8b5b --- /dev/null +++ b/.circleci/test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +## +## Test that GovCMS scaffold can be built using Satis. +## + +# shellcheck disable=SC2002,SC2015 + +# Satis application directory. +APP_DIR="${APP_DIR:-./app}" + +# The branch of Satis to test against. +SATIS_BRANCH="${SATIS_BRANCH:-}" + +# Directory where scaffold is installed into. +GOVCMS_SCAFFOLD_DIR="${GOVCMS_SCAFFOLD_DIR:-/tmp/govcms-build}" + +#------------------------------------------------------------------------------- + +echo "--> Starting Satis web server on http://localhost:4141." +killall -9 php > /dev/null 2>&1 || true +php -S localhost:4141 -t "${APP_DIR}" > /tmp/phpd.log 2>&1 & +sleep 4 # Waiting for the server to be ready. +netstat_opts='-tulpn'; [ "$(uname)" == "Darwin" ] && netstat_opts='-anv' || true; +netstat "${netstat_opts[@]}" | grep -q 4141 || (echo "ERROR: Unable to start inbuilt PHP server" && cat /tmp/php.log && exit 1) +curl -s -o /dev/null -w "%{http_code}" -L -I http://localhost:4141 | grep -q 200 || (echo "ERROR: Server is started, but site cannot be served" && exit 1) + +echo "--> Cloning govcms8-scaffold-paas into ${GOVCMS_SCAFFOLD_DIR}." +rm -Rf "${GOVCMS_SCAFFOLD_DIR}" +composer create-project --no-install --quiet govcms/govcms8-scaffold-paas "${GOVCMS_SCAFFOLD_DIR}" + +composer --working-dir="${GOVCMS_SCAFFOLD_DIR}" config secure-http false + +echo +echo "--> Test build GovCMS against http://localhost:4141/${SATIS_BRANCH}." +echo + +echo "--> Add http://localhost:4141/${SATIS_BRANCH} as a repository." +composer --working-dir="${GOVCMS_SCAFFOLD_DIR}" config repositories.govcms composer http://localhost:4141/"${SATIS_BRANCH}" + +if [ "${SATIS_BRANCH}" = "master" ] || [ "${SATIS_BRANCH}" = "develop" ] ; then + php -d memory_limit=-1 "$(command -v composer)" --working-dir="${GOVCMS_SCAFFOLD_DIR}" require --no-update \ + govcms/govcms:1.x \ + govcms/scaffold-tooling:dev-"${SATIS_BRANCH}" \ + govcms/require-dev:dev-"${SATIS_BRANCH}" \ + symfony/event-dispatcher:"v4.3.11 as v3.4.35" # @todo: remove once govcms/govcms no longer requires "symfony/event-dispatcher:v4.3.11 as v3.4.35" which only works at the root composer.json level. +else + # Get package versions required for testing stable release. + version_govcms="$(cat "./satis-config/govcms-stable.json" | jq -r '.require | .["govcms/govcms"]')" + version_tooling="$(cat "./satis-config/govcms-stable.json" | jq -r '.require | .["govcms/scaffold-tooling"]')" + version_require_dev="$(cat "./satis-config/govcms-stable.json" | jq -r '.require | .["govcms/require-dev"]')" + + echo "--> Expected stable versions, based on the Satis config:" + echo " - govcms/govcms: ${version_govcms}" + echo " - govcms/scaffold-tooling: ${version_tooling}" + echo " - govcms/require-dev: ${version_require_dev}" + php -d memory_limit=-1 "$(command -v composer)" --working-dir="${GOVCMS_SCAFFOLD_DIR}" require --no-update \ + govcms/govcms:"${version_govcms}" \ + govcms/scaffold-tooling:"${version_tooling}" \ + govcms/require-dev:"${version_require_dev}" \ + symfony/event-dispatcher:"v4.3.11 as v3.4.35" # @todo: remove once govcms/govcms no longer requires "symfony/event-dispatcher:v4.3.11 as v3.4.35" which only works at the root composer.json level. +fi + +echo "--> Contents of composer.json after dependency resolution." +cat "${GOVCMS_SCAFFOLD_DIR}"/composer.json | jq .require + +echo "--> Assert that Composer update works." +php -d memory_limit=-1 "$(command -v composer)" --working-dir="${GOVCMS_SCAFFOLD_DIR}" -n --quiet --no-suggest --no-scripts update + +echo "--> Assert that govcms* dependencies present." +composer --working-dir="${GOVCMS_SCAFFOLD_DIR}" info | grep ^govcms diff --git a/docs/MAINTENANCE.md b/docs/MAINTENANCE.md index ef825f2..eb3f841 100644 --- a/docs/MAINTENANCE.md +++ b/docs/MAINTENANCE.md @@ -2,25 +2,27 @@ ## What is it -There are four satis servers which are built from four config files (in ./satis-config). -These can be used in the "repositories" section of composer.json. +There are four satis servers which are built from four config files (in +`./satis-config`). +These can be used in the "repositories" section of `composer.json`. | | | | | --- | --- | --- | -| `STABLE` | govcms-stable.json | https://satis.govcms.gov.au/ | -| `MASTER` | govcms-master.json | https://satis.govcms.gov.au/master | -| `DEVELOP` | govcms-develop.json | https://satis.govcms.gov.au/develop | -| `WHITELIST` | govcms-whitelist.json | https://satis.govcms.gov.au/whitelist | +| `STABLE` | `govcms-stable.json` | https://satis.govcms.gov.au/ | +| `MASTER` | `govcms-master.json` | https://satis.govcms.gov.au/master | +| `DEVELOP` | `govcms-develop.json` | https://satis.govcms.gov.au/develop | +| `WHITELIST` | `govcms-whitelist.json` | https://satis.govcms.gov.au/whitelist | They are updated by running `ahoy build` which populates the `./app` directory with static files. -## When to update +## Updating content -Currently, updating is manual. The job to automate it was completed but it got stuck waiting -for deployment tokens. In the meantime it has been found that manual intervention is usually needed. +Updating of the Satis repository content is a manual process that includes +running a set of command locally and committing regenerated content directly +to the repository. -Updating `STABLE` should be done when there is a new point release of GovCMS. +Updating `STABLE` should be done when there is a new minor release of GovCMS. Updating `MASTER` and `DEVELOP` should be done when the `master` and `develop` branches are updated on @@ -31,10 +33,11 @@ branches are updated on ## Running the automated scripts -Each of the `ahoy build` `ahoy check-dupes` `ahoy verify` and `ahoy debug` commands can -accept any of the current versions (stable, master or develop) as a parameter. Providing no -parameter is equivalent to `stable`. There is also a `-all` variant to these commands that -will run the process for all branches. +Each of the `ahoy build`, `ahoy check-dupes`, `ahoy verify` and `ahoy debug` +commands can accept any of the current versions (`stable`, `master` or +`develop`) as a parameter. Providing no parameter is equivalent to `stable`. +There is also a `-all` variant to these commands that will run the process for +all branches. ## Update MASTER @@ -42,7 +45,8 @@ This will update the `./app/master` directory. 1. Clean `./satis-config/govcms-master.json` by removing: - * extra packages from `require` - only leave the first three `govcms/*` packages + * extra packages from `require` - only leave the first three `govcms/*` + packages * the `blacklist` section - remove completely 2. Run `ahoy build master` to update /app. @@ -51,16 +55,18 @@ This will update the `./app/master` directory. 4. Run `ahoy debug master` - follow instructions. -5. Re-run `ahoy build master` and `ahoy verify master` (ie. repeat the above steps as needed). +5. Re-run `ahoy build master` and `ahoy verify master` (ie. repeat the above + steps as needed). ## Update DEVELOP -This will update the `./app/develop` directory. Repeat the exact steps you followed to -update MASTER, just replace `master` with `develop`. +This will update the `./app/develop` directory. Repeat the exact steps you +followed to update MASTER, just replace `master` with `develop`. 1. Clean `./satis-config/govcms-develop.json` by removing: - * extra packages from `require` - only leave the first three `govcms/*` packages + * extra packages from `require` - only leave the first three `govcms/*` + packages * the `blacklist` section - remove completely 2. Run `ahoy build develop` to update /app. @@ -69,22 +75,25 @@ update MASTER, just replace `master` with `develop`. 4. Run `ahoy debug develop` - follow instructions. -5. Re-run `ahoy build develop` and `ahoy verify develop` (ie. repeat the above steps as needed). +5. Re-run `ahoy build develop` and `ahoy verify develop` (ie. repeat the above + steps as needed). ## Update STABLE -This will update the `./app` directory. You are repeating the steps you followed to -update MASTER, just replace `master` with `stable`. +This will update the `./app` directory. You are repeating the steps you followed +to update MASTER, just replace `master` with `stable`. There is only one extra step (step 2). 1. Clean `./satis-config/govcms-stable.json` by removing: - * extra packages from `require` - only leave the first three `govcms/*` packages + * extra packages from `require` - only leave the first three `govcms/*` + packages * the `blacklist` section - remove completely -2. *ONLY FOR STABLE* update the package versions for the `govcms/*` versions to the latest versions. +2. *ONLY FOR STABLE* update the package versions for the `govcms/*` versions + to the latest versions. 3. Run `ahoy build stable` to update /app. @@ -92,30 +101,32 @@ There is only one extra step (step 2). 5. Run `ahoy debug stable` - follow instructions. -6. Re-run `ahoy build stable` and `ahoy verify stable` (ie. repeat the above steps as needed). +6. Re-run `ahoy build stable` and `ahoy verify stable` (ie. repeat the above + steps as needed). ## Steps to update WHITELIST -This is a hassle free one because it doesn't calculate dependencies. Run `ahoy build-whitelist`. +This is a hassle free one because it doesn't calculate dependencies. +Run `ahoy build-whitelist`. ## Additional troubleshooting steps -To quickly see if there are any duplicates in your package.json files, `ahoy check-dupes { stable|master|develop }`. - +To quickly see if there are any duplicates in your package.json files, +`ahoy check-dupes { stable|master|develop }`. ## Push -Once you have updated all branches, create a PR to https://github.com/govcms/satis. Once this is merged it will trigger -quay.io to rebuild the an image (see docker-compose.yml). +Once you have updated all branches, create a PR to https://github.com/govcms/satis. + +Once this is merged it will trigger +quay.io to rebuild an image (see `docker-compose.yml`). ## Technical notes -This project is built on [composer/satis](https://github.com/composer/satis) using the -docker container method. The following settings are important because it forces Satis -to resolve an *ideal* set of package versions. +This project is built on top of [composer/satis](https://github.com/composer/satis) +using the Docker image. The following settings are important because it forces +Satis to resolve an *ideal* set of package versions. -``` "require-dependencies": true, "require-dev-dependencies": true, "only-best-candidates": true -``` diff --git a/docs/USAGE.md b/docs/USAGE.md index 8bec4c1..262393a 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -5,11 +5,12 @@ 1. The limited use of this packagist ensures that a project is GovCMS platform compliant - particularly SaaS projects. -2. Since Satis does the hard work of resolving module versions, your `composer update` -runs significantly faster. +2. Since Satis does the hard work of resolving module versions, your +`composer update` runs significantly faster. 3. By providing this packagist source, including an additional whitelist, we -may have a way to reduce the (significant) size of the GovCMS distribution itself. +may have a way to reduce the (significant) size of the GovCMS distribution +itself. 4. By using this as your only packagist source, you can ensure your Drupal site is compatible with GovCMS, without having to rely on the GovCMS scaffold. @@ -22,38 +23,39 @@ way to limit modules on their own projects. ## Usage -The de facto usage looks like this in composer.json. There is no requirement +The de facto usage looks like this in `composer.json`. There is no requirement to add additional sources. -``` -"repositories": { - "govcms": { - "type": "composer", - "url": "https://satis.govcms.gov.au/" + + "repositories": { + "govcms": { + "type": "composer", + "url": "https://satis.govcms.gov.au/" + }, + "govcms-whitelist": { + "type": "composer", + "url": "https://satis.govcms.gov.au/whitelist/" + }, + "packagist.org": false }, - "govcms-whitelist": { - "type": "composer", - "url": "https://satis.govcms.gov.au/whitelist/" - }, - "packagist.org": false -}, -"require": { - "govcms/govcms": "~1" - "govcms/scaffold-tooling": "~1" -} -"require-dev": { - "govcms/require-dev": "~1" -} - -``` + "require": { + "govcms/govcms": "~1" + "govcms/scaffold-tooling": "~1" + } + "require-dev": { + "govcms/require-dev": "~1" + } ### Breakdown of the above: -* `whitelist` is an additional whitelist of modules which are not in the GovCMS -distribution. (You may not be able to edit your composer.json to add these packages.) -* `"packagist.org": false` prevents adding modules and packages from https://packagist.org. -* `"~1"` allows us to keep the composer.json requirements very loose (best practice) as -Satis has done the job of constraining the versions already. +- `whitelist` is an additional whitelist of modules which are not in the GovCMS + distribution. You may not be able to edit your `composer.json` to add these + packages. +- `"packagist.org": false` prevents adding modules and packages from + https://packagist.org. +- `"~1"` allows us to keep the `composer.json` requirements very loose + (best practice) as Satis has done the job of constraining the versions + already. ## How to ... @@ -61,19 +63,19 @@ Satis has done the job of constraining the versions already. ### ... use the latest release of GovCMS If you are not hosting on the GovCMS platform you can still use the GovCMS -distribution by adding the code from the [Usage](#usage) section above. This also -requires setting up your settings.php appropriate to your hosting solution. +distribution by adding the code from the [Usage](#usage) section above. This +also requires setting up your settings.php appropriate to your hosting solution. ### ... leverage the GovCMS Drupal settings -By requiring `govcms/scaffold-tooling` in your composer.json you can +By requiring `govcms/scaffold-tooling` in your `composer.json` you can access GovCMS standard Drupal settings. The settings are available in `vendor/govcms/scaffold-tooling/drupal` after running composer update. See [settings.php](https://github.com/govCMS/govcms8-scaffold-paas/blob/develop/web/sites/default/settings.php) in the Drupal 8 scaffold for guidance on using these files. -### ... use GovCMS distribution, but with latest modules from Drupal.org +### ... use GovCMS distribution, but with the latest modules from Drupal.org There are a number of modules in GovCMS which at not the latest versions available. So is there a way to use the latest versions? @@ -88,11 +90,13 @@ could patch instead? Can you wait for GovCMS to update? If you are on PaaS, you can use an upcoming version of any module by manually placing the whole module in `web/sites/default/modules/`. This location -overrides locations like `web/modules/contrib`. This is not best practice +overrides locations like `web/modules/contrib`. This is not the best practice Drupal/Composer, but if this method helps PaaS customers remain close to a -"vanilla SaaS GovCMS" then it may be a valuable technique to add one or two modules. +"vanilla SaaS GovCMS" then it may be a valuable technique to add one or two +modules. -*In this case PaaS customers are wholly responsible for managing updates and regressions.* +*In this case PaaS customers are wholly responsible for managing updates and +regressions.* You can use the same method locally to test new module versions. We welcome these tests so please let us know via the issue queue if you @@ -106,34 +110,31 @@ in your repositories section, assuming they don't have version conflicts with GovCMS. Note that this shouldn't update the GovCMS module versions -because they are constrained by your govcms/govcms:~1 requirement. - -``` -"repositories": { - ... - "drupal ": { - "type": "composer", - "url": "https://packages.drupal.org/8" - } -``` +because they are constrained by your `govcms/govcms:~1` requirement. + + "repositories": { + ... + "drupal ": { + "type": "composer", + "url": "https://packages.drupal.org/8" + } -Add modules as desired. -``` -"require": { - "govcms/govcms": "~1", - "drupal/some-other-module": "~2", - "vanilla-ice/some-dependencies-for-my-module": "~2", - ..., -} -``` +Add modules as desired. + + "require": { + "govcms/govcms": "~1", + "drupal/some-other-module": "~2", + "vanilla-ice/some-dependencies-for-my-module": "~2", + ..., + } You can also add references to other Git repositories using [Composer vcs type] (https://getcomposer.org/doc/05-repositories.md#vcs). Finally, you can choose to remove`"packagist.org": false` to access all the many -packages on [packagist.org](https://packagist.org). Remember that this *may* cause your -composer to pick up sub-packages which are newer than what GovCMS distribution stipulates -(which is why it's not allowed on SaaS). If you are at this point you might as well -rearchitect with [drupal-composer/drupal-project](https://github.com/drupal-composer/drupal-project) +packages on [packagist.org](https://packagist.org). Remember that this *may* +cause your Composer to pick up sub-packages which are newer than what GovCMS +distribution stipulates (which is why it's not allowed on SaaS). If you are at +this point you might as well re-architect with [drupal-composer/drupal-project](https://github.com/drupal-composer/drupal-project) as your scaffold to get a more "custom vanilla" experience.