Use multistage build for master (docker#5369)

Dockerfile

@@ -1,35 +1,87 @@
-FROM docs/docker.github.io:docs-base
+# This Dockerfile builds the docs for https://docs.docker.com/
+# from the master branch of https://github.com/docker/docker.github.io
+#
+# Here is the sequence:
+# 1.  Set up the build
+# 2.  Fetch upstream resources
+# 3.  Build master
+# 4.  Copy static HTML from already-built archive images
+# 5.  Copy Nginx config
+#
+# When the image is run, it starts Nginx and serves the docs at port 4000
 
-# docs-base contains: GitHub Pages, nginx, wget, svn, and the docs archives,
-# running on Alpine. See the contents of docs-base at:
-# https://github.com/docker/docker.github.io/tree/docs-base
+# Get basic configs and Jekyll env
+FROM docs/docker.github.io:docs-builder AS builder
 
-# First, build non-edge (all of this is duplicated later -- that is on purpose)
+# Set the target again
+ENV TARGET=/usr/share/nginx/html
 
-# Copy master into target directory (skipping files / folders in .dockerignore)
-# These files represent the current docs
-COPY . md_source
+# Set the source directory to md_source
+ENV SOURCE=md_source
 
-# Move built html into md_source directory so we can reuse the target directory
-# to hold the static output.
-# Pull reference docs from upstream locations, then build the master docs
-# into static HTML in the "target" directory using Jekyll
-# then nuke the md_source directory.
+# Get the current docs from the checked out branch
+# ${SOURCE} will contain a directory for each archive
+COPY . ${SOURCE}
 
+####### START UPSTREAM RESOURCES ########
+# Set vars used by fetch-upstream-resources.sh script
 ## Branch to pull from, per ref doc
 ## To get master from svn the svn branch needs to be 'trunk'. To get a branch from svn it needs to be 'branches/branchname'
 
 # Engine
-ENV ENGINE_SVN_BRANCH="branches/17.06.x"
-ENV ENGINE_BRANCH="17.06.x"
+ENV ENGINE_SVN_BRANCH="branches/17.09.x"
+ENV ENGINE_BRANCH="17.09.x"
 
 # Distribution
 ENV DISTRIBUTION_SVN_BRANCH="branches/release/2.6"
 ENV DISTRIBUTION_BRANCH="release/2.6"
 
-RUN /sbin/apk --update add bash \
-  && bash ./md_source/_scripts/fetch-upstream-resources.sh \
-	&& jekyll build -s md_source -d target --config md_source/_config.yml \
-	&& rm -rf target/apidocs/layouts \
-	&& find target -type f -name '*.html' -print0 | xargs -0 sed -i 's#href="https://docs.docker.com/#href="/#g' \
-	&& rm -rf md_source
+# Fetch upstream resources
+RUN bash ./${SOURCE}/_scripts/fetch-upstream-resources.sh ${SOURCE}
+####### END UPSTREAM RESOURCES ########
+
+
+# Build the static HTML, now that everything is in place
+
+RUN jekyll build -s ${SOURCE} -d ${TARGET} --config ${SOURCE}/_config.yml
+
+# Fix up some links, don't touch the archives
+RUN find ${TARGET} -type f -name '*.html' | grep -vE "v[0-9]+\." | while read i; do sed -i 's#href="https://docs.docker.com/#href="/#g' "$i"; done
+
+# BUILD OF MASTER DOCS IS NOW DONE!
+# Reset to alpine so we don't get any docs source or extra apps
+FROM nginx:alpine
+
+# Set the target again
+ENV TARGET=/usr/share/nginx/html
+
+# Get the built docs output from the previous step
+COPY --from=builder ${TARGET} ${TARGET}
+
+# Get all the archive static HTML and put it into place
+# To add a new archive, add it here
+# AND ALSO edit _data/docsarchives/archives.yaml to add it to the drop-down
+COPY --from=docs/docker.github.io:v1.4 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.5 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.6 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.7 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.8 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.9 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.10 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.11 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.12 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v1.13 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v17.03 ${TARGET} ${TARGET}
+COPY --from=docs/docker.github.io:v17.06 ${TARGET} ${TARGET}
+
+# The archives are self-browseable and each come with an index.html. This creates
+# a conflict with the index.html and 404.html from the master build. The easiest
+# solution is to just overwrite them again here.
+COPY --from=builder ${TARGET}/index.html ${TARGET}/index.html
+COPY --from=builder ${TARGET}/404.html ${TARGET}/404.html
+
+# Get the nginx config from the nginx-onbuild image
+COPY --from=docs/docker.github.io:nginx-onbuild /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf
+
+# Serve the site (target), which is now all static HTML
+CMD echo -e "Docker docs are viewable at:\nhttp://0.0.0.0:4000"; exec nginx -g 'daemon off;'

Dockerfile.archive

@@ -0,0 +1,11 @@
+# Set to the version for this archive
+ARG VER=vXX
+
+# This image comes from the Dockerfile.onbuild file in the docs-builder branch
+# https://github.com/docker/docker.github.io/blob/docs-builder/Dockerfile.onbuild
+FROM docs/docker.github.io:docs-builder-onbuild AS builder
+
+# Reset the docs:onbuild image, which is based on nginx:alpine
+# This image comes from the Dockerfile in the nginx-onbuild branch
+# https://github.com/docker/docker.github.io/blob/nginx-onbuild/Dockerfile
+FROM docs/docker.github.io:nginx-onbuild

README.md

@@ -175,7 +175,7 @@ To read the docs offline, you can use either a standalone container or a swarm s
 To see all available tags, go to
 [Docker Cloud](https://cloud.docker.com/app/docs/repository/docker/docs/docker.github.io/tags).
 The following examples use the `latest` tag:
-    
+
 - Run a single container:
 
   ```bash
@@ -278,6 +278,83 @@ Bootstrap JS are loaded.
 
 > **Note**: In general, this is a bad idea.
 
+## Building archives and the live published docs
+
+All the images described below are automatically built using Docker Cloud. To
+build the site manually, from scratch, including all utility and archive
+images, see the [README in the publish-tools branch](https://github.com/docker/docker.github.io/blob/publish-tools/README.md).
+
+- Some utility images are built from Dockerfiles in the `publish-tools` branch.
+  See its [README](https://github.com/docker/docker.github.io/blob/publish-tools/README.md)
+  for details.
+- Each archive branch automatically builds an image tagged
+  `docs/docker.github.io:v<VERSION>` when a change is merged into that branch.
+- The `master` branch has a Dockerfile which uses the static HTML from each
+  archive image, in combination with the Markdown
+  files in `master` and some upstream resources which are fetched at build-time,
+  to create the full site at https://docs.docker.com/. All
+  of the long-running branches, such as `vnext-engine`, `vnext-compose`, etc,
+  use the same logic.
+
+## Creating a new archive
+
+When a new Docker CE Stable version is released, the previous state of `master`
+is archived into a version-specific branch like `v17.09`, by doing the following:
+
+1.  Create branch based off the commit hash before the new version was released.
+
+    ```bash
+    $ git checkout <HASH>
+    $ git checkout -b v17.09
+    ```
+
+2.  Run the `_scripts/fetch-upstream-resources.sh` script. This puts static
+    copies of the files in place that the `master`  build typically fetches
+    each build.
+
+    ```bash
+    $ _scripts/fetch-upstream/resources.sh
+    ```
+
+3.  Overwrite the `Dockerfile` with the `Dockerfile.archive` (use `cp` rather
+    than `mv` so you don't inadvertently remove either file). Edit the resulting
+    `Dockerfile` and set the `VER` build argument to the appropriate value, like
+    `v17.09`.
+
+    ```bash
+    $ mv Dockerfile.archive Dockerfile
+    $ vi Dockerfile
+
+      < edit the variable and save >
+    ```
+
+4.  Do `git status` and add all changes, being careful not to add anything extra
+    by accident. Commit your work.
+
+    ```bash
+    $ git status
+    $ git add <filename>
+    $ git add <filename> (etc etc etc)
+    $ git commit -m "Creating archive for 17.09 docs"
+    ```
+
+5.  Make sure the archive builds.
+
+    ```bash
+    $ docker build -t docker build -t docker.github.io/docs:v17.09 .
+    $ docker run --rm -it -p 4000:4000 docker.github.io/docs:v17.09
+    ```
+
+    After the `docker run` command, browse to `http://localhost:4000/` and
+    verify that the archive is self-browseable.
+
+6.  Push the branch to the upstream repository. Do not create a pull request
+    as there is no reference branch to compare against.
+
+    ```bash
+    $ git push upstream v17.09
+    ```
+
 ## Copyright and license
 
 Code and documentation copyright 2017 Docker, inc, released under the Apache 2.0 license.

_config.yml

@@ -11,13 +11,15 @@ permalink: pretty
 safe: false
 lsi: false
 url: https://docs.docker.com
+# This needs to have all the directories you expect to be in the archives (delivered by docs-base in the Dockerfile)
 keep_files: ["v1.4", "v1.5", "v1.6", "v1.7", "v1.8", "v1.9", "v1.10", "v1.11", "v1.12", "v1.13", "v17.03", "v17.06"]
+exclude: ["_scripts", "apidocs/layouts", "Gemfile", "hooks"]
 
 # Component versions -- address like site.docker_ce_stable_version
 # You can't have - characters in these for non-YAML reasons
 
 docker_ce_stable_version: "17.09"
-latest_stable_docker_engine_api_version: "1.32"
+latest_stable_docker_engine_api_version: "1.33"
 docker_ce_edge_version: "17.11"
 docker_ee_version: "17.06"
 compose_version: "1.18.0"