-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add post prod deploy smoke test and alert (#7057)
* frontend component and script * backend config * actions * rename * some other stuff * add slack alert back in * remove slack comment * move slack alert over * dan feedback * add okta call and update script config * lint * remove trailing slash * remove empty var * remove comment * move url to one place * use existing status check instead * string format and equality * move literal to left * lol it's friday alright * add comment to document workflow * better comment * use base domain env var instead * set env var * don't hard code node version
- Loading branch information
Showing
17 changed files
with
332 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| name: Smoke test post deploy | ||
| description: Invoke a script that visits a deploy smoke check page that displays whether the backend / db are healthy. | ||
| inputs: | ||
| base_domain_name: | ||
| description: The domain where the application is deployed (e.g. "simplereport.gov" or "test.simplereport.gov") | ||
| required: true | ||
| runs: | ||
| using: composite | ||
| steps: | ||
| - name: create env file | ||
| shell: bash | ||
| working-directory: frontend | ||
| run: | | ||
| touch .env | ||
| echo REACT_APP_BASE_URL=${{ inputs.base_domain_name }}>> .env.production.local | ||
| - name: Run smoke test script | ||
| shell: bash | ||
| working-directory: frontend | ||
| run: yarn smoke:deploy:ci |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| name: Smoke test deploy Prod | ||
| run-name: Smoke test the deploy for prod by @${{ github.actor }} | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: [ "Deploy Prod" ] | ||
| types: | ||
| - completed | ||
|
|
||
| env: | ||
| NODE_VERSION: 18 | ||
|
|
||
| jobs: | ||
| smoke-test-front-and-back-end: | ||
| runs-on: ubuntu-latest | ||
| environment: Production | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Use Node.js | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: ${{env.NODE_VERSION}} | ||
| - name: Cache yarn | ||
| uses: actions/[email protected] | ||
| with: | ||
| path: ~/.cache/yarn | ||
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | ||
| - name: Set up dependencies | ||
| working-directory: frontend | ||
| run: yarn install --prefer-offline | ||
| - name: Smoke test the env | ||
| uses: ./.github/actions/post-deploy-smoke-test | ||
| with: | ||
| base_domain_name: ${{ vars.BASE_DOMAIN_NAME }} | ||
| slack_alert: | ||
| runs-on: ubuntu-latest | ||
| if: failure() | ||
| needs: [ smoke-test-front-and-back-end ] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Send alert to Slack | ||
| uses: ./.github/actions/slack-message | ||
| with: | ||
| username: ${{ github.actor }} | ||
| description: | | ||
| :siren-gif: Post-deploy smoke test couldn't verify that the frontend is talking to the backend. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} :siren-gif: | ||
| webhook_url: ${{ secrets.SR_ALERTS_SLACK_WEBHOOK_URL }} | ||
| user_map: $${{ secrets.SR_ALERTS_GITHUB_SLACK_MAP }} |
41 changes: 41 additions & 0 deletions
41
...main/java/gov/cdc/usds/simplereport/api/heathcheck/BackendAndDatabaseHealthIndicator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package gov.cdc.usds.simplereport.api.heathcheck; | ||
|
|
||
| import com.okta.sdk.resource.client.ApiException; | ||
| import gov.cdc.usds.simplereport.db.repository.FeatureFlagRepository; | ||
| import gov.cdc.usds.simplereport.idp.repository.OktaRepository; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.hibernate.exception.JDBCConnectionException; | ||
| import org.springframework.boot.actuate.health.Health; | ||
| import org.springframework.boot.actuate.health.HealthIndicator; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| @Component("backend-and-db-smoke-test") | ||
| @Slf4j | ||
| @RequiredArgsConstructor | ||
| public class BackendAndDatabaseHealthIndicator implements HealthIndicator { | ||
| private final FeatureFlagRepository _ffRepo; | ||
| private final OktaRepository _oktaRepo; | ||
| public static final String ACTIVE_LITERAL = "ACTIVE"; | ||
|
|
||
| @Override | ||
| public Health health() { | ||
| try { | ||
| _ffRepo.findAll(); | ||
| String oktaStatus = _oktaRepo.getApplicationStatusForHealthCheck(); | ||
|
|
||
| if (!ACTIVE_LITERAL.equals(oktaStatus)) { | ||
| log.info("Okta status didn't return ACTIVE, instead returned " + oktaStatus); | ||
| return Health.down().build(); | ||
| } | ||
|
|
||
| return Health.up().build(); | ||
| } catch (JDBCConnectionException e) { | ||
| return Health.down().build(); | ||
| // Okta API call errored | ||
| } catch (ApiException e) { | ||
| log.info(e.getMessage()); | ||
| return Health.down().build(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
...java/gov/cdc/usds/simplereport/api/healthcheck/BackendAndDatabaseHealthIndicatorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package gov.cdc.usds.simplereport.api.healthcheck; | ||
|
|
||
| import static gov.cdc.usds.simplereport.api.heathcheck.BackendAndDatabaseHealthIndicator.ACTIVE_LITERAL; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| import gov.cdc.usds.simplereport.api.heathcheck.BackendAndDatabaseHealthIndicator; | ||
| import gov.cdc.usds.simplereport.db.repository.BaseRepositoryTest; | ||
| import gov.cdc.usds.simplereport.db.repository.FeatureFlagRepository; | ||
| import gov.cdc.usds.simplereport.idp.repository.OktaRepository; | ||
| import java.sql.SQLException; | ||
| import java.util.List; | ||
| import lombok.RequiredArgsConstructor; | ||
| import org.hibernate.exception.JDBCConnectionException; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.boot.actuate.health.Health; | ||
| import org.springframework.boot.context.properties.EnableConfigurationProperties; | ||
| import org.springframework.boot.test.mock.mockito.SpyBean; | ||
|
|
||
| @RequiredArgsConstructor | ||
| @EnableConfigurationProperties | ||
| class BackendAndDatabaseHealthIndicatorTest extends BaseRepositoryTest { | ||
|
|
||
| @SpyBean private FeatureFlagRepository mockFeatureFlagRepo; | ||
| @SpyBean private OktaRepository mockOktaRepo; | ||
|
|
||
| @Autowired private BackendAndDatabaseHealthIndicator indicator; | ||
|
|
||
| @Test | ||
| void health_succeedsWhenReposDoesntThrow() { | ||
| when(mockFeatureFlagRepo.findAll()).thenReturn(List.of()); | ||
| when(mockOktaRepo.getApplicationStatusForHealthCheck()).thenReturn(ACTIVE_LITERAL); | ||
|
|
||
| assertThat(indicator.health()).isEqualTo(Health.up().build()); | ||
| } | ||
|
|
||
| @Test | ||
| void health_failsWhenFeatureFlagRepoDoesntThrow() { | ||
| JDBCConnectionException dbConnectionException = | ||
| new JDBCConnectionException( | ||
| "connection issue", new SQLException("some reason", "some state")); | ||
| when(mockFeatureFlagRepo.findAll()).thenThrow(dbConnectionException); | ||
| assertThat(indicator.health()).isEqualTo(Health.down().build()); | ||
| } | ||
|
|
||
| @Test | ||
| void health_failsWhenOktaRepoDoesntReturnActive() { | ||
| when(mockOktaRepo.getApplicationStatusForHealthCheck()).thenReturn("INACTIVE"); | ||
| assertThat(indicator.health()).isEqualTo(Health.down().build()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Script that does a simple Selenium scrape of | ||
| // - A frontend page with a simple status message that hits a health check backend | ||
| // endpoint which does a simple ping to a non-sensitive DB table to verify | ||
| // all the connections are good. | ||
| // https://github.com/CDCgov/prime-simplereport/pull/7057 | ||
|
|
||
| require("dotenv").config(); | ||
| let { Builder } = require("selenium-webdriver"); | ||
| const Chrome = require("selenium-webdriver/chrome"); | ||
|
|
||
| const appUrl = process.env.REACT_APP_BASE_URL.includes("localhost") | ||
| ? `${process.env.REACT_APP_BASE_URL}/health/deploy-smoke-test` | ||
| : `${process.env.REACT_APP_BASE_URL}/app/health/deploy-smoke-test`; | ||
|
|
||
| console.log(`Running smoke test for ${appUrl}`); | ||
| const options = new Chrome.Options(); | ||
| const driver = new Builder() | ||
| .forBrowser("chrome") | ||
| .setChromeOptions(options.addArguments("--headless=new")) | ||
| .build(); | ||
| driver | ||
| .navigate() | ||
| .to(`${appUrl}`) | ||
| .then(() => { | ||
| let value = driver.findElement({ id: "root" }).getText(); | ||
| return value; | ||
| }) | ||
| .then((value) => { | ||
| driver.quit(); | ||
| return value; | ||
| }) | ||
| .then((value) => { | ||
| if (value.includes("success")) process.exit(0); | ||
| process.exit(1); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| import { render, screen, waitFor } from "@testing-library/react"; | ||
| import { FetchMock } from "jest-fetch-mock"; | ||
|
|
||
| import DeploySmokeTest from "./DeploySmokeTest"; | ||
|
|
||
| describe("DeploySmokeTest", () => { | ||
| beforeEach(() => { | ||
| (fetch as FetchMock).resetMocks(); | ||
| }); | ||
|
|
||
| it("renders success when returned from the API endpoint", async () => { | ||
| (fetch as FetchMock).mockResponseOnce(JSON.stringify({ status: "UP" })); | ||
|
|
||
| render(<DeploySmokeTest />); | ||
| await waitFor(() => | ||
| expect(screen.queryByText("Status loading...")).not.toBeInTheDocument() | ||
| ); | ||
| expect(screen.getByText("Status returned success :)")); | ||
| }); | ||
|
|
||
| it("renders failure when returned from the API endpoint", async () => { | ||
| (fetch as FetchMock).mockResponseOnce(JSON.stringify({ status: "DOWN" })); | ||
|
|
||
| render(<DeploySmokeTest />); | ||
| await waitFor(() => | ||
| expect(screen.queryByText("Status loading...")).not.toBeInTheDocument() | ||
| ); | ||
| expect(screen.getByText("Status returned failure :(")); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| import { useEffect, useState } from "react"; | ||
|
|
||
| import FetchClient from "./utils/api"; | ||
|
|
||
| const api = new FetchClient(undefined, { mode: "cors" }); | ||
| const DeploySmokeTest = (): JSX.Element => { | ||
| const [success, setSuccess] = useState<boolean>(); | ||
| useEffect(() => { | ||
| api | ||
| .getRequest("/actuator/health/backend-and-db-smoke-test") | ||
| .then((response) => { | ||
| const status = JSON.parse(response); | ||
| if (status.status === "UP") return setSuccess(true); | ||
| setSuccess(false); | ||
| }) | ||
| .catch((e) => { | ||
| console.error(e); | ||
| setSuccess(false); | ||
| }); | ||
| }, []); | ||
|
|
||
| if (success === undefined) return <>Status loading...</>; | ||
| if (success) return <> Status returned success :) </>; | ||
| return <> Status returned failure :( </>; | ||
| }; | ||
|
|
||
| export default DeploySmokeTest; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.