Skip to content

Commit

Permalink
Merge pull request #260 from umccr/refactor/lambda-axum
Browse files Browse the repository at this point in the history 
refactor: htsget-lambda and misc deployment changes
  • Loading branch information
@mmalenic
mmalenic authored Sep 6, 2024
2 parents aa51483 + c45b092 commit c0e0926
Show file tree
Hide file tree
Showing 30 changed files with 565 additions and 1,262 deletions.
55 changes: 27 additions & 28 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions deploy/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM rust:1.76-slim AS builder
FROM rust:1.81-slim AS builder

LABEL org.opencontainers.image.source=https://github.com/umccr/htsget-rs
LABEL org.opencontainers.image.url=https://github.com/umccr/htsget-rs/pkgs/container/htsget-rs
Expand All @@ -17,9 +17,9 @@ RUN cargo build --all-features --release && \

FROM gcr.io/distroless/cc-debian12

COPY --from=builder /build/target/release/htsget-actix /usr/local/bin/htsget-actix
COPY --from=builder /build/target/release/htsget-axum /usr/local/bin/htsget-axum

ENV HTSGET_TICKET_SERVER_ADDR 0.0.0.0:8080
ENV HTSGET_DATA_SERVER_ADDR 0.0.0.0:8081

CMD [ "htsget-actix" ]
CMD [ "htsget-axum" ]
2 changes: 2 additions & 0 deletions deploy/Dockerfile.dockerignore
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
*

!/htsget-actix
!/htsget-axum
!/htsget-config
!/htsget-http
!/htsget-lambda
!/htsget-search
!/htsget-storage
!/htsget-test
!/Cargo.toml
!/Cargo.lock
Expand Down
37 changes: 26 additions & 11 deletions deploy/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,29 @@ The CDK code in this directory constructs a CDK app from [`HtsgetLambdaStack`][h
[`bin/settings.ts`][htsget-settings]:

#### HtsgetSettings

These are general settings for the CDK deployment.

| Name | Description | Type |
|----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|
| <span id="config">`config`</span> | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` |
| <span id="domain">`domain`</span> | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` |
| <span id="authorizer">`authorizer`</span> | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) |
| <span id="subDomain">`subDomain?`</span> | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` |
| <span id="s3BucketResources">`s3BucketResources?`</span> | The resources that are affected by the bucket policy with actions: `["s3:List*", "s3:Get*"]`. If this is not specified, it defaults to `["arn:aws:s3:::*"]`. This affects which buckets are allowed to be accessed with the policy. | `string[]` |
| <span id="lookupHostedZone">`lookupHostedZone?`</span> | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` |
| Name | Description | Type |
|--------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|
| <span id="config">`config`</span> | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` |
| <span id="domain">`domain`</span> | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` |
| <span id="authorizer">`authorizer`</span> | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) |
| <span id="subDomain">`subDomain?`</span> | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` |
| <span id="s3BucketResources">`s3BucketResources`</span> | The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. This option must be specified to allow `htsget-rs` to access data in buckets that are not created in this stack. | `string[]` |
| <span id="lookupHostedZone">`lookupHostedZone?`</span> | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` |
| <span id="createS3Bucket">`createS3Bucket?`</span> | Whether to create a test bucket. Defaults to true. Buckets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). The correct access permissions are automatically added. | `boolean` |
| <span id="bucketName">`bucketName?`</span> | The name of the bucket created using `createS3Bucket`. The name defaults to an automatically generated CDK name, use this option to override that. This option only has an affect is `createS3Buckets` is true. | `string` |
| <span id="copyTestData">`copyTestData?`</span> | Whether to copy test data into the bucket. Defaults to true. This copies the example data under the `data` directory to those buckets. This option only has an affect is `createS3Buckets` is true. | `boolean` |

#### HtsgetJwtAuthSettings

These settings are used to determine if the htsget API gateway endpoint is configured to have a JWT authorizer or not.

| Name | Description | Type |
|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|------------|
| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
| <span id="public">`public`</span> | Whether this deployment is public. If this is `true` then no authorizer is present on the API gateway and the options below have no effect. | `boolean` |
| <span id="jwtAudience">`jwtAudience?`</span> | A list of the intended recipients of the JWT. A valid JWT must provide an aud that matches at least one entry in this list. | `string[]` |
| <span id="jwtAudience">`jwtAudience?`</span> | A list of the intended recipients of the JWT. A valid JWT must provide an aud that matches at least one entry in this list. | `string[]` |
| <span id="cogUserPoolId?">`cogUserPoolId?`</span> | The cognito user pool id for the authorizer. If this is not set, then a new user pool is created. No user pool is created if [`public`](#public) is true. | `string` |

The [`HtsgetSettings`](#htsgetsettings) are passed into [`HtsgetLambdaStack`][htsget-lambda-stack] in order to change the deployment config. An example of a public instance deployment
Expand All @@ -49,7 +54,7 @@ After installing the basic dependencies, complete the following steps:

1. Login to AWS and define `CDK_DEFAULT_*` env variables (if not defined already). You must be authenticated with your AWS cloud to run this step.
2. Install [cargo-lambda], as it is used to compile artifacts that are uploaded to aws lambda.
3. Define which configuration to use for htsget-rs as stated in the configuration section.
3. Define which configuration to use for htsget-rs as stated in the configuration section.

Below is a summary of commands to run in this directory:

Expand All @@ -67,6 +72,11 @@ npm install

### Deploy to AWS

> [!IMPORTANT]
> The default deployment is designed to work out of the box. A bucket with a CDK-generated name is created with test
> data from the [`data`][data] directory. All deployment settings can be tweaked using the [`settings.ts`][htsget-settings].
> The only option that must be specified in the `domain`, which determines the domain name to serve htsget-rs at.
CDK should be bootstrapped once, if this hasn't been done before:

```sh
Expand All @@ -79,6 +89,10 @@ Then to deploy the stack, run:
npx cdk deploy
```

> [!WARNING]
> By default this deployment will create a public instance of htsget-rs. Anyone will be able to query the server
> without authorizing unless you modify the `HtsgetJwtAuthSettings` settings.
### Testing the endpoint

When the deployment is finished, the htsget endpoint can be tested by querying it. If a JWT authorizer is configured,
Expand Down Expand Up @@ -173,3 +187,4 @@ and a [MinIO][minio] deployment.
[rust]: https://www.rust-lang.org/tools/install
[zig]: https://ziglang.org/
[zig-getting-started]: https://ziglang.org/learn/getting-started/
[data]: ../data
16 changes: 9 additions & 7 deletions deploy/bin/settings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,18 @@ import { HtsgetSettings } from "../lib/htsget-lambda-stack";
* Settings to use for the htsget deployment.
*/
export const SETTINGS: HtsgetSettings = {
config: "config/dev_umccr.toml",
config: "config/example_deploy.toml",
// Specify the domain to serve htsget-rs under.
domain: "dev.umccr.org",
subDomain: "htsget",
s3BucketResources: [
"arn:aws:s3:::org.umccr.demo.sbeacon-data/*",
"arn:aws:s3:::org.umccr.demo.htsget-rs-data/*",
],
lookupHostedZone: true,
s3BucketResources: [],
lookupHostedZone: false,
createS3Bucket: true,
copyTestData: true,
// Override the bucket name.
// bucketName: "bucket",
jwtAuthorizer: {
// Set this to true if you want a public instance.
// Set this to false if you want a private instance.
public: false,
// jwtAudience: ["audience"],
// cogUserPoolId: "user-pool-id",
Expand Down
5 changes: 5 additions & 0 deletions deploy/config/dev_umccr.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ contact_url = "https://umccr.org/"
documentation_url = "https://github.com/umccr/htsget-rs"
environment = "dev"

[[resolvers]]
regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P<key>.*)$'
substitution_string = '$key'
storage = 'S3'

[[resolvers]]
regex = '^(umccr-10c-data-dev)/(?P<key>.*)$'
substitution_string = '$key'
Expand Down
20 changes: 20 additions & 0 deletions deploy/config/example_deploy.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
ticket_server_cors_allow_headers = "All"
ticket_server_cors_allow_origins = []
ticket_server_cors_allow_methods = "All"
ticket_server_cors_allow_credentials = true
ticket_server_cors_max_age = 300

data_server_enabled = false

name = "umccr-htsget-rs"
version = "0.1"
organization_name = "UMCCR"
organization_url = "https://umccr.org/"
contact_url = "https://umccr.org/"
documentation_url = "https://github.com/umccr/htsget-rs"
environment = "dev"

[[resolvers]]
regex = '^(?P<bucket>.*?)/(?P<key>.*)$'
substitution_string = '$key'
storage = 'S3'
Loading

0 comments on commit c0e0926

Please sign in to comment.