Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: divviup/janus
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 0.7.23
Choose a base ref
...
head repository: divviup/janus
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
Loading
Showing with 18,787 additions and 11,613 deletions.
  1. +93 −0 .github/dependabot.yml
  2. +8 −6 .github/workflows/ci-build.yml
  3. +1 −3 .github/workflows/make-release.yml
  4. +1 −1 .github/workflows/push-docker-images-release.yml
  5. +1,069 −1,017 Cargo.lock
  6. +63 −62 Cargo.toml
  7. +4 −4 Dockerfile
  8. +2 −2 Dockerfile.interop
  9. +2 −2 Dockerfile.interop_aggregator
  10. +4 −4 Dockerfile.sqlx
  11. +22 −18 README.md
  12. +5 −4 aggregator/Cargo.toml
  13. +614 −605 aggregator/src/aggregator.rs
  14. +69 −62 aggregator/src/aggregator/aggregate_init_tests.rs
  15. +3 −3 aggregator/src/aggregator/aggregate_share.rs
  16. +100 −105 aggregator/src/aggregator/aggregation_job_continue.rs
  17. +394 −691 aggregator/src/aggregator/aggregation_job_creator.rs
  18. +563 −233 aggregator/src/aggregator/aggregation_job_driver.rs
  19. +2,828 −550 aggregator/src/aggregator/aggregation_job_driver/tests.rs
  20. +272 −187 aggregator/src/aggregator/aggregation_job_writer.rs
  21. +28 −38 aggregator/src/aggregator/batch_creator.rs
  22. +34 −49 aggregator/src/aggregator/{query_type.rs → batch_mode.rs}
  23. +55 −50 aggregator/src/aggregator/collection_job_driver.rs
  24. +270 −318 aggregator/src/aggregator/collection_job_tests.rs
  25. +18 −13 aggregator/src/aggregator/error.rs
  26. +81 −62 aggregator/src/aggregator/garbage_collector.rs
  27. +203 −164 aggregator/src/aggregator/http_handlers.rs
  28. +34 −21 aggregator/src/aggregator/http_handlers/tests/aggregate_share.rs
  29. +343 −325 aggregator/src/aggregator/http_handlers/tests/aggregation_job_continue.rs
  30. +213 −154 aggregator/src/aggregator/http_handlers/tests/aggregation_job_init.rs
  31. +115 −122 aggregator/src/aggregator/http_handlers/tests/collection_job.rs
  32. +24 −17 aggregator/src/aggregator/http_handlers/tests/helper_e2e.rs
  33. +55 −215 aggregator/src/aggregator/http_handlers/tests/hpke_config.rs
  34. +61 −54 aggregator/src/aggregator/http_handlers/tests/report.rs
  35. +81 −81 aggregator/src/aggregator/key_rotator.rs
  36. +5 −6 aggregator/src/aggregator/problem_details.rs
  37. +821 −0 aggregator/src/aggregator/queue.rs
  38. +12 −11 aggregator/src/aggregator/report_writer.rs
  39. +178 −233 aggregator/src/aggregator/taskprov_tests.rs
  40. +53 −9 aggregator/src/aggregator/test_util.rs
  41. +91 −87 aggregator/src/aggregator/upload_tests.rs
  42. +17 −10 aggregator/src/binaries/aggregation_job_creator.rs
  43. +38 −21 aggregator/src/binaries/aggregation_job_driver.rs
  44. +82 −85 aggregator/src/binaries/aggregator.rs
  45. +43 −32 aggregator/src/binaries/collection_job_driver.rs
  46. +5 −2 aggregator/src/binaries/garbage_collector.rs
  47. +106 −110 aggregator/src/binaries/janus_cli.rs
  48. +4 −1 aggregator/src/binaries/key_rotator.rs
  49. +24 −25 aggregator/src/binary_utils.rs
  50. +3 −6 aggregator/src/binary_utils/job_driver.rs
  51. +88 −150 aggregator/src/cache.rs
  52. +90 −64 aggregator/src/config.rs
  53. +14 −26 aggregator/src/diagnostic.rs
  54. +0 −14 aggregator/src/lib.rs
  55. +36 −15 aggregator/src/main.rs
  56. +62 −41 aggregator/src/metrics.rs
  57. +9 −23 aggregator/src/metrics/tests/prometheus.rs
  58. +265 −171 aggregator/src/metrics/tokio_runtime.rs
  59. +14 −4 aggregator/src/trace.rs
  60. +37 −31 aggregator/tests/integration/graceful_shutdown.rs
  61. +3 −2 aggregator_api/Cargo.toml
  62. +20 −9 aggregator_api/src/lib.rs
  63. +41 −46 aggregator_api/src/models.rs
  64. +77 −61 aggregator_api/src/routes.rs
  65. +272 −207 aggregator_api/src/tests.rs
  66. +4 −4 aggregator_core/Cargo.toml
  67. +17 −0 aggregator_core/README.md
  68. +24 −29 aggregator_core/src/{query_type.rs → batch_mode.rs}
  69. +514 −467 aggregator_core/src/datastore.rs
  70. +398 −214 aggregator_core/src/datastore/models.rs
  71. +152 −24 aggregator_core/src/datastore/test_util.rs
  72. +744 −540 aggregator_core/src/datastore/tests.rs
  73. +5 −15 aggregator_core/src/lib.rs
  74. +217 −374 aggregator_core/src/task.rs
  75. +29 −28 aggregator_core/src/taskprov.rs
  76. +1 −1 client/Cargo.toml
  77. +20 −12 client/src/lib.rs
  78. +5 −11 client/src/tests/mod.rs
  79. +1 −1 client/src/tests/ohttp.rs
  80. +2 −3 collector/Cargo.toml
  81. +4 −3 collector/src/credential.rs
  82. +325 −250 collector/src/lib.rs
  83. +5 −3 core/Cargo.toml
  84. +20 −23 core/src/auth_tokens.rs
  85. +3 −19 core/src/dp.rs
  86. +17 −13 core/src/hpke.rs
  87. +2 −0 core/src/http.rs
  88. +4 −0 core/src/lib.rs
  89. +1 −1 core/src/report_id.rs
  90. +4 −0 core/src/retries.rs
  91. +12 −4 core/src/test_util/mod.rs
  92. +5 −5 core/src/test_util/runtime.rs
  93. +8 −1 core/src/time.rs
  94. +302 −82 core/src/vdaf.rs
  95. +4 −3 db/00000000000001_initial_schema.down.sql
  96. +76 −53 db/00000000000001_initial_schema.up.sql
  97. +0 −1 db/00000000000002_collection_job_step_attempts.down.sql
  98. +0 −3 db/00000000000002_collection_job_step_attempts.up.sql
  99. +0 −4 db/00000000000003_outstanding_batch_state.down.sql
  100. +0 −8 db/00000000000003_outstanding_batch_state.up.sql
  101. +0 −1 db/00000000000004_batch_aggregation_gc.down.sql
  102. +0 −2 db/00000000000004_batch_aggregation_gc.up.sql
  103. +0 −1 db/00000000000005_global_hpke_keys_last_state_change_at.down.sql
  104. +0 −11 db/00000000000005_global_hpke_keys_last_state_change_at.up.sql
  105. +0 −3 db/00000000000006_global_hpke_keys_last_state_change_at_drop_default.down.sql
  106. +0 −3 db/00000000000006_global_hpke_keys_last_state_change_at_drop_default.up.sql
  107. +9 −0 deny.toml
  108. +0 −99 docs/CONFIGURING_GLOBAL_HPKE_KEYS.md
  109. +153 −0 docs/CONFIGURING_HPKE_KEYS.md
  110. +24 −34 docs/CONFIGURING_TASKPROV.md
  111. +3 −0 docs/DEPLOYING.md
  112. +11 −0 docs/DEVELOPMENT.md
  113. +11 −12 docs/samples/advanced_config/aggregation_job_creator.yaml
  114. +25 −18 docs/samples/advanced_config/aggregation_job_driver.yaml
  115. +76 −55 docs/samples/advanced_config/aggregator.yaml
  116. +19 −20 docs/samples/advanced_config/collection_job_driver.yaml
  117. +9 −10 docs/samples/advanced_config/garbage_collector.yaml
  118. +9 −10 docs/samples/advanced_config/janus_cli.yaml
  119. +8 −9 docs/samples/advanced_config/key_rotator.yaml
  120. +2 −2 docs/samples/basic_config/aggregation_job_creator.yaml
  121. +4 −3 docs/samples/basic_config/aggregation_job_driver.yaml
  122. +3 −3 docs/samples/basic_config/collection_job_driver.yaml
  123. +1 −1 docs/samples/basic_config/key_rotator.yaml
  124. +7 −14 docs/samples/tasks.yaml
  125. +7 −0 integration_tests/Cargo.toml
  126. +6 −3 integration_tests/src/client.rs
  127. +4 −4 integration_tests/src/daphne.rs
  128. +26 −24 integration_tests/src/janus.rs
  129. +2 −2 integration_tests/src/lib.rs
  130. +70 −109 integration_tests/tests/integration/common.rs
  131. +5 −5 integration_tests/tests/integration/daphne.rs
  132. +19 −5 integration_tests/tests/integration/divviup_ts.rs
  133. +210 −52 integration_tests/tests/integration/in_cluster.rs
  134. +153 −75 integration_tests/tests/integration/janus.rs
  135. +1 −0 integration_tests/tests/integration/main.rs
  136. +750 −0 integration_tests/tests/integration/simulation/arbitrary.rs
  137. +509 −0 integration_tests/tests/integration/simulation/bad_client.rs
  138. +100 −0 integration_tests/tests/integration/simulation/mod.rs
  139. +118 −0 integration_tests/tests/integration/simulation/model.rs
  140. +183 −0 integration_tests/tests/integration/simulation/proxy.rs
  141. +121 −0 integration_tests/tests/integration/simulation/quicktest.rs
  142. +343 −0 integration_tests/tests/integration/simulation/reproduction.rs
  143. +584 −0 integration_tests/tests/integration/simulation/run.rs
  144. +321 −0 integration_tests/tests/integration/simulation/setup.rs
  145. +2 −6 interop_binaries/Cargo.toml
  146. +2 −2 interop_binaries/config/aggregation_job_creator.yaml
  147. +4 −3 interop_binaries/config/aggregation_job_driver.yaml
  148. +5 −5 interop_binaries/config/collection_job_driver.yaml
  149. +0 −1 interop_binaries/config/janus_interop_aggregator.yaml
  150. +36 −39 interop_binaries/src/commands/janus_interop_aggregator.rs
  151. +11 −7 interop_binaries/src/commands/janus_interop_client.rs
  152. +293 −297 interop_binaries/src/commands/janus_interop_collector.rs
  153. +35 −74 interop_binaries/src/lib.rs
  154. +89 −107 interop_binaries/tests/end_to_end.rs
  155. +2 −1 messages/Cargo.toml
  156. +20 −22 messages/src/{query_type.rs → batch_mode.rs}
  157. +279 −297 messages/src/lib.rs
  158. +7 −7 messages/src/problem_type.rs
  159. +517 −787 messages/src/taskprov.rs
  160. +151 −62 messages/src/tests/aggregation.rs
  161. +109 −81 messages/src/tests/collection.rs
  162. +19 −59 messages/src/tests/query.rs
  163. +56 −4 messages/src/tests/upload.rs
  164. +0 −1 tools/Cargo.toml
  165. +50 −187 tools/src/bin/collect.rs
  166. +20 −20 tools/src/bin/dap_decode.rs
  167. +5 −9 tools/tests/cmd/collect.trycmd
  168. +1 −1 tools/tests/cmd/collect_fpvec_bounded_l2.trycmd
  169. +1 −1 tools/tests/cmd/dap_decode.trycmd
93 changes: 93 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -12,6 +12,17 @@ updates:
- dependency-name: prio
update-types:
- version-update:semver-minor
# opentelemetry-rust has removed support for pull exporters, including
# opentelemetry-prometheus, and will add it back after the 1.0 release.
- dependency-name: opentelemetry
versions:
- ">= 0.25, < 1.0"
- dependency-name: opentelemetry_sdk
versions:
- ">= 0.25, < 1.0"
- dependency-name: opentelemetry-otlp
versions:
- ">= 0.25, < 1.0"
groups:
serde:
patterns:
@@ -46,6 +57,10 @@ updates:
patterns:
- thiserror
- anyhow
rstest:
patterns:
- rstest
- rstest_reuse
- package-ecosystem: "github-actions"
directory: "/"
schedule:
@@ -63,3 +78,81 @@ updates:
target-branch: "main"
ignore:
- dependency-name: "postgres"

# Manage dependencies on the release/0.7 branch
- package-ecosystem: "cargo"
directory: "/"
schedule:
interval: "weekly"
target-branch: "release/0.7"
open-pull-requests-limit: 20
ignore:
- dependency-name: prio
update-types:
- version-update:semver-minor
# opentelemetry-rust has removed support for pull exporters, including
# opentelemetry-prometheus, and will add it back after the 1.0 release.
- dependency-name: opentelemetry
versions:
- ">= 0.25, < 1.0"
- dependency-name: opentelemetry_sdk
versions:
- ">= 0.25, < 1.0"
- dependency-name: opentelemetry-otlp
versions:
- ">= 0.25, < 1.0"
groups:
serde:
patterns:
- serde
- serde_*
tracing:
patterns:
- tracing
- tracing-*
opentelemetry:
patterns:
- opentelemetry
- opentelemetry-*
- opentelemetry_*
trillium:
patterns:
- trillium
- trillium-*
kube:
patterns:
- kube
- k8s-openapi
postgres:
patterns:
- tokio-postgres
- postgres-*
deadpool:
patterns:
- deadpool
- deadpool-*
error-handling:
patterns:
- thiserror
- anyhow
rstest:
patterns:
- rstest
- rstest_reuse
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
target-branch: "release/0.7"
groups:
docker:
patterns:
- docker/*
- crazymax/ghaction-github-runtime
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "weekly"
target-branch: "release/0.7"
ignore:
- dependency-name: "postgres"
14 changes: 8 additions & 6 deletions .github/workflows/ci-build.yml
Original file line number Diff line number Diff line change
@@ -30,6 +30,8 @@ jobs:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-D warnings"
steps:
- name: 'Workaround for issue #3178'
run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
- name: Set default input values
id: default-input-values
run: |
@@ -69,14 +71,14 @@ jobs:
run: cargo test --package janus_client --features ohttp
# Note: keep Build & Test steps consecutive, and match flags other than `--no-run`.
- name: Build
run: cargo test --profile ci --locked --all-targets --no-run
run: cargo test --profile ci --locked --all-targets --features=prometheus --no-run
- name: Test
id: test
env:
RUST_LOG: info
JANUS_E2E_LOGS_PATH: ${{ github.workspace }}/test-logs
DIVVIUP_TS_INTEROP_CONTAINER: ${{ steps.default-input-values.outputs.divviup_ts_interop_container }}
run: cargo test --profile ci --locked --all-targets
run: cargo test --profile ci --locked --all-targets --features=prometheus
# Continue on error so we can upload logs
continue-on-error: true
- name: Build (Docker-specific tests)
@@ -136,7 +138,7 @@ jobs:
- name: Document
run: cargo doc --profile ci --workspace --all-features
- name: cargo-deny
uses: EmbarkStudios/cargo-deny-action@v1.6.3
uses: EmbarkStudios/cargo-deny-action@v2.0.4
with:
command: check bans licenses sources -A unmatched-organization

@@ -153,7 +155,7 @@ jobs:
driver: docker-container
use: true
- name: Build
uses: docker/bake-action@v5.3.0
uses: docker/bake-action@v5.11.0
with:
files: docker-bake.hcl
workdir: .
@@ -179,7 +181,7 @@ jobs:
driver: docker-container
use: true
- name: Build
uses: docker/bake-action@v5.3.0
uses: docker/bake-action@v5.11.0
with:
files: docker-bake.hcl
workdir: .
@@ -191,6 +193,6 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: cargo-deny
uses: EmbarkStudios/cargo-deny-action@v1.6.3
uses: EmbarkStudios/cargo-deny-action@v2.0.4
with:
command: check advisories
4 changes: 1 addition & 3 deletions .github/workflows/make-release.yml
Original file line number Diff line number Diff line change
@@ -9,9 +9,7 @@ on:
be marked as the latest release.
required: false
type: string
# This action may be run against the main branch so long as we are not
# in a prerelease window.
default: '["main"]'
default: '["release/0.7"]'

jobs:
bump-version:
2 changes: 1 addition & 1 deletion .github/workflows/push-docker-images-release.yml
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ jobs:
id: get_version
run: echo VERSION=${GITHUB_REF/refs\/tags\//} | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV
- name: Build
uses: docker/bake-action@v5.3.0
uses: docker/bake-action@v5.11.0
with:
files: docker-bake.hcl
workdir: .
Loading