.gitlab-ci.yml

image: "debian:buster-slim"

cache: &global_cache
  key: global
  paths:
    - .yarn
    - node_modules

stages:
  - docker
  - prepare
  - build osm
  - build vicmap
  - conflate
  - candidate
  - upload
  - pages

docker prepare:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":prepare -f "docker/prepare.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":prepare
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

docker build-vicmap:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":build-vicmap -f "docker/build-vicmap.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":build-vicmap
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

docker build-osm:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":build-osm -f "docker/build-osm.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":build-osm
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

docker conflate:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":conflate -f "docker/conflate.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":conflate
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

docker candidate:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":candidate -f "docker/candidate.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":candidate
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

docker pages:
  image: docker:latest
  services:
    - docker:dind
  stage: docker
  before_script:
    - echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE":pages -f "docker/pages.Dockerfile" .
    - docker push "$CI_REGISTRY_IMAGE":pages
  after_script:
    - docker logout $CI_REGISTRY
  when: manual
  cache: {}

prepare:
  stage: prepare
  image: "registry.gitlab.com/alantgeo/vicmap2osm:prepare"
  before_script:
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - mkdir -p dist data
    - b2 sync --threads 4 --allowEmptySource --noProgress --delete dist b2://vicmap2osm/dist
    - b2 sync --threads 4 --allowEmptySource --noProgress --delete data b2://vicmap2osm/data
  cache:
    <<: *global_cache
    policy: push
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - mkdir -p dist
    - make data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
    - rm -rf data/vicmap.geojson data/VICMAP_ADDRESS.zip
    - make data/vicmap.geojson
  after_script:
    - b2 upload-file --noProgress vicmap2osm data/vicmap.geojson data/vicmap.geojson
  when: manual


build osm:
  stage: build osm
  image: "registry.gitlab.com/alantgeo/vicmap2osm:build-osm"
  before_script:
    - mkdir -p data dist
    - ls data dist
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
  cache:
    <<: *global_cache
    policy: pull-push
  variables:
    QT_QPA_PLATFORM: "offscreen"
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - mkdir -p dist data
    - make data/victoria-named-features.osm.geojson
    - make data/victoria-addr.osm.fgb
    - make data/victoria-addr.osm.centroids.fgb
    - make data/blocks.fgb
    - make dist/blocksByOSMAddr.geojson
    - make data/victoria-admin-level9.osm.geojson
    - make summariseBlocksByOSMAddr
  needs: ['prepare']
  dependencies:
    - prepare
  after_script:
    - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data
    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist

build vicmap:
  stage: build vicmap
  image: "registry.gitlab.com/alantgeo/vicmap2osm:build-vicmap"
  before_script:
    - mkdir -p data dist
    - ls data dist
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
  cache:
    <<: *global_cache
    policy: pull-push
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - yarn run test
    - mkdir -p dist data
    - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria
    - touch data/VICMAP_ADDRESS.zip
    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
    - touch --no-create data/vicmap.geojson
    - time make dist/vicmap-osm-with-suburb.geojson
    - time make dist/vicmap-osm-uniq.geojson
    - time make dist/vicmap-osm-uniq-flats.geojson
    - time make dist/vicmap-osm-uniq-flats-withinrange.geojson
    - time make dist/vicmap-osm-overlapping.geojson
    - time make convertGeoJSONResultsToFGB
    - wc -l dist/*.geojson
  needs: ['build osm']
  dependencies:
    - build osm
  after_script:
    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist
    - b2 sync --threads 4 --allowEmptySource --noProgress debug b2://vicmap2osm/debug

conflate:
  stage: conflate
  image: "registry.gitlab.com/alantgeo/vicmap2osm:conflate"
  before_script:
    - mkdir -p data dist
    - ls data dist
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
  cache:
    <<: *global_cache
    policy: pull
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - make dist/conflate
    - make dist/vicmap-complex-conflation
    - make dist/vicmap-building-conflation
    - make convertConflationResultsToFGB
    - wc -l dist/conflate/*.geojson
    - wc -l dist/vicmap-*-conflation/*.geojson
  needs: ['build vicmap', 'build osm']
  dependencies:
    - build vicmap
    - build osm
  after_script:
    - b2 sync --threads 4 --allowEmptySource --noProgress data b2://vicmap2osm/data
    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist

candidate:
  stage: candidate
  image: "registry.gitlab.com/alantgeo/vicmap2osm:candidate"
  before_script:
    - mkdir -p data dist
    - ls data dist
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
  cache:
    <<: *global_cache
    policy: pull
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - make dist/unitFromNumber.osc
    - make dist/candidates
  needs: ['conflate']
  dependencies:
    - conflate
  after_script:
    - b2 sync --threads 4 --allowEmptySource --noProgress dist b2://vicmap2osm/dist

upload dev:
  stage: upload
  image: "registry.gitlab.com/alantgeo/vicmap2osm:candidate"
  before_script:
    - mkdir -p data dist
    - ls data dist
  cache:
    <<: *global_cache
    policy: pull
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - ./bin/upload.sh
  when: manual
  needs: ['candidate']
  dependencies:
    - candidate
  artifacts:
    name: "upload"
    paths:
      - uploadLog

build compareSuburb:
  stage: build vicmap
  image: "registry.gitlab.com/alantgeo/vicmap2osm:build-osm"
  before_script:
    - mkdir -p data dist
    - ls data dist
  cache:
    <<: *global_cache
    policy: pull
  script:
    - yarn install
    - cp src/polygon-lookup-patch.js node_modules/polygon-lookup/index.js
    - mkdir -p dist data
    - mkdir -p data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria
    - touch data/VICMAP_ADDRESS.zip
    - touch data/vicmap/ll_gda2020/filegdb/whole_of_dataset/victoria/VMADD.gdb
    - touch --no-create data/vicmap.geojson
    - time make dist/vicmap-osm-with-suburb.geojson
    - time make dist/vicmapSuburbDiffersWithOSM.geojson
    - rm -f dist/vicmap-osm-with-suburb.geojson
    - make printDifferentSuburbs
  when: manual
  needs: ['prepare']
  dependencies:
    - prepare

pages:
  stage: pages
  image: "registry.gitlab.com/alantgeo/vicmap2osm:pages"
  before_script:
    - b2 authorize-account $B2_KEYID $B2_APPKEY
    - b2 sync --threads 4 --allowEmptySource --noProgress b2://vicmap2osm/ .
  script:
    - cp -r www public
    - cp dist/conflate/*.changes.json public/data/
    - b2 ls --recursive b2://vicmap2osm/ | sed -e 's/\(.*\)/ * [\1](\1)/g' > public/index.md
    - markdown public/index.md > public/index.html
    - b2 upload-file vicmap2osm public/index.html index.html
  artifacts:
    paths:
      - public
  needs: ['conflate']
  dependencies:
    - conflate