From 25624b9cf851411b6935e2f526389e25e767606d Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 20 Nov 2024 20:43:27 +0000 Subject: [PATCH 01/24] B-21669 - add UPLOAD_DOC_STATUS constant w/ setTimeout UI. --- .../DocumentViewer/DocumentViewer.jsx | 34 +++++++++++++++++-- src/shared/constants.js | 7 ++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index 73daff482d3..78236620f86 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -16,6 +16,8 @@ import { bulkDownloadPaymentRequest, updateUpload } from 'services/ghcApi'; import { formatDate } from 'shared/dates'; import { filenameFromPath } from 'utils/formatters'; import AsyncPacketDownloadLink from 'shared/AsyncPacketDownloadLink/AsyncPacketDownloadLink'; +import { UPLOAD_DOC_STATUS } from 'shared/constants'; +import Alert from 'shared/Alert'; /** * TODO @@ -26,6 +28,7 @@ import AsyncPacketDownloadLink from 'shared/AsyncPacketDownloadLink/AsyncPacketD const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { const [selectedFileIndex, selectFile] = useState(0); + const [fileStatus, setFileStatus] = useState(null); const [disableSaveButton, setDisableSaveButton] = useState(false); const [menuIsOpen, setMenuOpen] = useState(false); const sortedFiles = files.sort((a, b) => moment(b.createdAt) - moment(a.createdAt)); @@ -37,6 +40,19 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { const queryClient = useQueryClient(); + const handleFileProcessingStatus = async () => { + setFileStatus('UPLOADING'); + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus('SCANNING')); + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus('ESTABLISHING')); + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus('LOADED')); + }; + const { mutate: mutateUploads } = useMutation(updateUpload, { onSuccess: async (data, variables) => { if (mountedRef.current) { @@ -60,7 +76,7 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { } else { setDisableSaveButton(true); } - }, [rotationValue, selectedFile, selectFile]); + }, [rotationValue, selectedFile]); useEffect(() => { return () => { @@ -74,12 +90,13 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { useEffect(() => { setRotationValue(selectedFile?.rotation || 0); + handleFileProcessingStatus(); }, [selectedFile]); const fileType = useRef(selectedFile?.contentType); - if (!selectedFile) { - return

File Not Found

; + if (!selectedFile || !fileStatus) { + return ; } const openMenu = () => { @@ -91,9 +108,20 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { const handleSelectFile = (index) => { selectFile(index); + closeMenu(); }; + if (fileStatus && fileStatus !== 'LOADED') { + return ( + + {fileStatus === UPLOAD_DOC_STATUS.UPLOADING && 'Uploading'} + {fileStatus === UPLOAD_DOC_STATUS.SCANNING && 'Scanning'} + {fileStatus === UPLOAD_DOC_STATUS.ESTABLISHING && 'Establishing Document for View'} + + ); + } + const fileTypeMap = { 'application/pdf': 'pdf', 'image/png': 'png', diff --git a/src/shared/constants.js b/src/shared/constants.js index 94903bd65c4..c8b714c1864 100644 --- a/src/shared/constants.js +++ b/src/shared/constants.js @@ -69,6 +69,13 @@ export const UPLOAD_SCAN_STATUS = { PROCESSING: 'PROCESSING', }; +export const UPLOAD_DOC_STATUS = { + UPLOADING: 'UPLOADING', + SCANNING: 'SCANNING', + ESTABLISHING: 'ESTABLISHING', + LOADED: 'LOADED', +}; + export const CONUS_STATUS = { CONUS: 'CONUS', OCONUS: 'OCONUS', From 8db737d8291c77f2253333bad77ca693f3e0894a Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 21 Nov 2024 23:01:40 +0000 Subject: [PATCH 02/24] B-21669 - transitory steps. using broken status based on aws tags --- .../DocumentViewer/DocumentViewer.jsx | 38 +++++++++++-------- .../DocumentViewerFileManager.jsx | 4 ++ .../MoveDocumentWrapper.jsx | 10 ++++- src/pages/Office/Orders/Orders.jsx | 4 +- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index 78236620f86..8a94763049e 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -16,7 +16,7 @@ import { bulkDownloadPaymentRequest, updateUpload } from 'services/ghcApi'; import { formatDate } from 'shared/dates'; import { filenameFromPath } from 'utils/formatters'; import AsyncPacketDownloadLink from 'shared/AsyncPacketDownloadLink/AsyncPacketDownloadLink'; -import { UPLOAD_DOC_STATUS } from 'shared/constants'; +import { UPLOAD_DOC_STATUS, UPLOAD_SCAN_STATUS } from 'shared/constants'; import Alert from 'shared/Alert'; /** @@ -26,7 +26,7 @@ import Alert from 'shared/Alert'; * - handle fetch doc errors */ -const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { +const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestId }) => { const [selectedFileIndex, selectFile] = useState(0); const [fileStatus, setFileStatus] = useState(null); const [disableSaveButton, setDisableSaveButton] = useState(false); @@ -40,18 +40,11 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { const queryClient = useQueryClient(); - const handleFileProcessingStatus = async () => { - setFileStatus('UPLOADING'); - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus('SCANNING')); - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus('ESTABLISHING')); - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus('LOADED')); - }; + useEffect(() => { + if (isFileUploading) { + setFileStatus(UPLOAD_DOC_STATUS.UPLOADING); + } + }, [isFileUploading]); const { mutate: mutateUploads } = useMutation(updateUpload, { onSuccess: async (data, variables) => { @@ -90,12 +83,27 @@ const DocumentViewer = ({ files, allowDownload, paymentRequestId }) => { useEffect(() => { setRotationValue(selectedFile?.rotation || 0); + const handleFileProcessingStatus = async () => { + if (selectedFile.status === UPLOAD_SCAN_STATUS.PROCESSING) { + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus(UPLOAD_DOC_STATUS.SCANNING)); + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING)); + await new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus('LOADED')); + } else if (selectedFile.status === UPLOAD_SCAN_STATUS.CLEAN) { + setFileStatus('LOADED'); + } + }; handleFileProcessingStatus(); }, [selectedFile]); const fileType = useRef(selectedFile?.contentType); - if (!selectedFile || !fileStatus) { + if (!selectedFile || !fileStatus || selectedFile?.status === UPLOAD_SCAN_STATUS.INFECTED) { return ; } diff --git a/src/components/DocumentViewerFileManager/DocumentViewerFileManager.jsx b/src/components/DocumentViewerFileManager/DocumentViewerFileManager.jsx index 8cb2557dc77..981627f1083 100644 --- a/src/components/DocumentViewerFileManager/DocumentViewerFileManager.jsx +++ b/src/components/DocumentViewerFileManager/DocumentViewerFileManager.jsx @@ -28,6 +28,8 @@ const DocumentViewerFileManager = ({ files, documentType, updateAmendedDocument, + onUploadStarted, + onUploadEnded, }) => { const queryClient = useQueryClient(); const filePondEl = useRef(); @@ -102,6 +104,7 @@ const DocumentViewerFileManager = ({ .finally(() => { queryClient.invalidateQueries([ORDERS_DOCUMENTS, documentId]); setIsFileProcessing(false); + onUploadEnded(); }); }; @@ -236,6 +239,7 @@ const DocumentViewerFileManager = ({ ref={filePondEl} createUpload={handleUpload} onChange={handleChange} + onAddFile={onUploadStarted} labelIdle={'Drag files here or click to upload'} /> PDF, JPG, or PNG only. Maximum file size 25MB. Each page must be clear and legible diff --git a/src/pages/Office/MoveDocumentWrapper/MoveDocumentWrapper.jsx b/src/pages/Office/MoveDocumentWrapper/MoveDocumentWrapper.jsx index f7d97fde5a9..3b4d3dbaa06 100644 --- a/src/pages/Office/MoveDocumentWrapper/MoveDocumentWrapper.jsx +++ b/src/pages/Office/MoveDocumentWrapper/MoveDocumentWrapper.jsx @@ -15,6 +15,8 @@ const MoveDocumentWrapper = () => { const { moveCode } = useParams(); const { pathname } = useLocation(); + const [isFileUploading, setFileUploading] = useState(false); + const { upload, amendedOrderDocumentId, isLoading, isError } = useOrdersDocumentQueries(moveCode); // some moves do not have amendedOrderDocumentId created and is null. // this is to update the id when it is created to store amendedUpload data. @@ -63,7 +65,7 @@ const MoveDocumentWrapper = () => {
{documentsForViewer && (
- +
)} {showOrders ? ( @@ -72,6 +74,12 @@ const MoveDocumentWrapper = () => { files={documentsByTypes} amendedDocumentId={amendedDocumentId} updateAmendedDocument={updateAmendedDocument} + onUploadStarted={() => { + setFileUploading(true); + }} + onUploadEnded={() => { + setFileUploading(false); + }} /> ) : ( diff --git a/src/pages/Office/Orders/Orders.jsx b/src/pages/Office/Orders/Orders.jsx index 035f63b3a0b..79d219bd340 100644 --- a/src/pages/Office/Orders/Orders.jsx +++ b/src/pages/Office/Orders/Orders.jsx @@ -32,7 +32,7 @@ const ordersTypeDropdownOptions = dropdownInputOptions(ORDERS_TYPE_OPTIONS); const ordersTypeDetailsDropdownOptions = dropdownInputOptions(ORDERS_TYPE_DETAILS_OPTIONS); const payGradeDropdownOptions = dropdownInputOptions(ORDERS_PAY_GRADE_OPTIONS); -const Orders = ({ files, amendedDocumentId, updateAmendedDocument }) => { +const Orders = ({ files, amendedDocumentId, updateAmendedDocument, onUploadStarted, onUploadEnded }) => { const navigate = useNavigate(); const { moveCode } = useParams(); const [tacValidationState, tacValidationDispatch] = useReducer(tacReducer, null, initialTacState); @@ -373,6 +373,8 @@ const Orders = ({ files, amendedDocumentId, updateAmendedDocument }) => { documentId={documentId} files={ordersDocuments} documentType={MOVE_DOCUMENT_TYPE.ORDERS} + onUploadStarted={onUploadStarted} + onUploadEnded={onUploadEnded} /> Date: Fri, 22 Nov 2024 21:16:31 +0000 Subject: [PATCH 03/24] B-21669 - add first api endpoint for getUploadStatus --- pkg/gen/internalapi/configure_mymove.go | 5 + pkg/gen/internalapi/embedded_spec.go | 98 ++++++++++ .../internaloperations/mymove_api.go | 12 ++ .../uploads/get_upload_status.go | 58 ++++++ .../uploads/get_upload_status_parameters.go | 91 +++++++++ .../uploads/get_upload_status_responses.go | 177 ++++++++++++++++++ .../uploads/get_upload_status_urlbuilder.go | 101 ++++++++++ pkg/handlers/authentication/auth.go | 1 + pkg/handlers/internalapi/api.go | 1 + pkg/handlers/internalapi/uploads.go | 19 ++ pkg/handlers/internalapi/uploads_test.go | 44 +++++ .../DocumentViewer/DocumentViewer.jsx | 9 +- src/services/internalApi.js | 12 ++ swagger-def/internal.yaml | 35 ++++ swagger/internal.yaml | 34 ++++ 15 files changed, 695 insertions(+), 2 deletions(-) create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go create mode 100644 pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go diff --git a/pkg/gen/internalapi/configure_mymove.go b/pkg/gen/internalapi/configure_mymove.go index a53746ea640..60cf2568da3 100644 --- a/pkg/gen/internalapi/configure_mymove.go +++ b/pkg/gen/internalapi/configure_mymove.go @@ -200,6 +200,11 @@ func configureAPI(api *internaloperations.MymoveAPI) http.Handler { return middleware.NotImplemented("operation transportation_offices.GetTransportationOffices has not yet been implemented") }) } + if api.UploadsGetUploadStatusHandler == nil { + api.UploadsGetUploadStatusHandler = uploads.GetUploadStatusHandlerFunc(func(params uploads.GetUploadStatusParams) middleware.Responder { + return middleware.NotImplemented("operation uploads.GetUploadStatus has not yet been implemented") + }) + } if api.EntitlementsIndexEntitlementsHandler == nil { api.EntitlementsIndexEntitlementsHandler = entitlements.IndexEntitlementsHandlerFunc(func(params entitlements.IndexEntitlementsParams) middleware.Responder { return middleware.NotImplemented("operation entitlements.IndexEntitlements has not yet been implemented") diff --git a/pkg/gen/internalapi/embedded_spec.go b/pkg/gen/internalapi/embedded_spec.go index 44bfbf10988..5bb44a88071 100644 --- a/pkg/gen/internalapi/embedded_spec.go +++ b/pkg/gen/internalapi/embedded_spec.go @@ -3234,6 +3234,55 @@ func init() { } } }, + "/uploads/{uploadId}/status": { + "get": { + "description": "Returns status of an upload based on antivirus run", + "tags": [ + "uploads" + ], + "summary": "Returns status of an upload", + "operationId": "getUploadStatus", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "UUID of the upload to return status of", + "name": "uploadId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "the requested upload status", + "schema": { + "type": "string", + "enum": [ + "INFECTED", + "CLEAN", + "PROCESSING" + ], + "readOnly": true + } + }, + "400": { + "description": "invalid request", + "schema": { + "$ref": "#/definitions/InvalidRequestResponsePayload" + } + }, + "403": { + "description": "not authorized" + }, + "404": { + "description": "not found" + }, + "500": { + "description": "server error" + } + } + } + }, "/users/is_logged_in": { "get": { "description": "Returns boolean as to whether the user is logged in", @@ -12062,6 +12111,55 @@ func init() { } } }, + "/uploads/{uploadId}/status": { + "get": { + "description": "Returns status of an upload based on antivirus run", + "tags": [ + "uploads" + ], + "summary": "Returns status of an upload", + "operationId": "getUploadStatus", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "UUID of the upload to return status of", + "name": "uploadId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "the requested upload status", + "schema": { + "type": "string", + "enum": [ + "INFECTED", + "CLEAN", + "PROCESSING" + ], + "readOnly": true + } + }, + "400": { + "description": "invalid request", + "schema": { + "$ref": "#/definitions/InvalidRequestResponsePayload" + } + }, + "403": { + "description": "not authorized" + }, + "404": { + "description": "not found" + }, + "500": { + "description": "server error" + } + } + } + }, "/users/is_logged_in": { "get": { "description": "Returns boolean as to whether the user is logged in", diff --git a/pkg/gen/internalapi/internaloperations/mymove_api.go b/pkg/gen/internalapi/internaloperations/mymove_api.go index e43938616a6..6fe72081ad1 100644 --- a/pkg/gen/internalapi/internaloperations/mymove_api.go +++ b/pkg/gen/internalapi/internaloperations/mymove_api.go @@ -145,6 +145,9 @@ func NewMymoveAPI(spec *loads.Document) *MymoveAPI { TransportationOfficesGetTransportationOfficesHandler: transportation_offices.GetTransportationOfficesHandlerFunc(func(params transportation_offices.GetTransportationOfficesParams) middleware.Responder { return middleware.NotImplemented("operation transportation_offices.GetTransportationOffices has not yet been implemented") }), + UploadsGetUploadStatusHandler: uploads.GetUploadStatusHandlerFunc(func(params uploads.GetUploadStatusParams) middleware.Responder { + return middleware.NotImplemented("operation uploads.GetUploadStatus has not yet been implemented") + }), EntitlementsIndexEntitlementsHandler: entitlements.IndexEntitlementsHandlerFunc(func(params entitlements.IndexEntitlementsParams) middleware.Responder { return middleware.NotImplemented("operation entitlements.IndexEntitlements has not yet been implemented") }), @@ -373,6 +376,8 @@ type MymoveAPI struct { MovesGetAllMovesHandler moves.GetAllMovesHandler // TransportationOfficesGetTransportationOfficesHandler sets the operation handler for the get transportation offices operation TransportationOfficesGetTransportationOfficesHandler transportation_offices.GetTransportationOfficesHandler + // UploadsGetUploadStatusHandler sets the operation handler for the get upload status operation + UploadsGetUploadStatusHandler uploads.GetUploadStatusHandler // EntitlementsIndexEntitlementsHandler sets the operation handler for the index entitlements operation EntitlementsIndexEntitlementsHandler entitlements.IndexEntitlementsHandler // MoveDocsIndexMoveDocumentsHandler sets the operation handler for the index move documents operation @@ -620,6 +625,9 @@ func (o *MymoveAPI) Validate() error { if o.TransportationOfficesGetTransportationOfficesHandler == nil { unregistered = append(unregistered, "transportation_offices.GetTransportationOfficesHandler") } + if o.UploadsGetUploadStatusHandler == nil { + unregistered = append(unregistered, "uploads.GetUploadStatusHandler") + } if o.EntitlementsIndexEntitlementsHandler == nil { unregistered = append(unregistered, "entitlements.IndexEntitlementsHandler") } @@ -948,6 +956,10 @@ func (o *MymoveAPI) initHandlerCache() { if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) } + o.handlers["GET"]["/uploads/{uploadId}/status"] = uploads.NewGetUploadStatus(o.context, o.UploadsGetUploadStatusHandler) + if o.handlers["GET"] == nil { + o.handlers["GET"] = make(map[string]http.Handler) + } o.handlers["GET"]["/entitlements"] = entitlements.NewIndexEntitlements(o.context, o.EntitlementsIndexEntitlementsHandler) if o.handlers["GET"] == nil { o.handlers["GET"] = make(map[string]http.Handler) diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go new file mode 100644 index 00000000000..dc2c021f021 --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status.go @@ -0,0 +1,58 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" +) + +// GetUploadStatusHandlerFunc turns a function with the right signature into a get upload status handler +type GetUploadStatusHandlerFunc func(GetUploadStatusParams) middleware.Responder + +// Handle executing the request and returning a response +func (fn GetUploadStatusHandlerFunc) Handle(params GetUploadStatusParams) middleware.Responder { + return fn(params) +} + +// GetUploadStatusHandler interface for that can handle valid get upload status params +type GetUploadStatusHandler interface { + Handle(GetUploadStatusParams) middleware.Responder +} + +// NewGetUploadStatus creates a new http.Handler for the get upload status operation +func NewGetUploadStatus(ctx *middleware.Context, handler GetUploadStatusHandler) *GetUploadStatus { + return &GetUploadStatus{Context: ctx, Handler: handler} +} + +/* + GetUploadStatus swagger:route GET /uploads/{uploadId}/status uploads getUploadStatus + +# Returns status of an upload + +Returns status of an upload based on antivirus run +*/ +type GetUploadStatus struct { + Context *middleware.Context + Handler GetUploadStatusHandler +} + +func (o *GetUploadStatus) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewGetUploadStatusParams() + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go new file mode 100644 index 00000000000..1770aa8ca6b --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_parameters.go @@ -0,0 +1,91 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/validate" +) + +// NewGetUploadStatusParams creates a new GetUploadStatusParams object +// +// There are no default values defined in the spec. +func NewGetUploadStatusParams() GetUploadStatusParams { + + return GetUploadStatusParams{} +} + +// GetUploadStatusParams contains all the bound params for the get upload status operation +// typically these are obtained from a http.Request +// +// swagger:parameters getUploadStatus +type GetUploadStatusParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` + + /*UUID of the upload to return status of + Required: true + In: path + */ + UploadID strfmt.UUID +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewGetUploadStatusParams() beforehand. +func (o *GetUploadStatusParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + rUploadID, rhkUploadID, _ := route.Params.GetOK("uploadId") + if err := o.bindUploadID(rUploadID, rhkUploadID, route.Formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// bindUploadID binds and validates parameter UploadID from path. +func (o *GetUploadStatusParams) bindUploadID(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + + // Format: uuid + value, err := formats.Parse("uuid", raw) + if err != nil { + return errors.InvalidType("uploadId", "path", "strfmt.UUID", raw) + } + o.UploadID = *(value.(*strfmt.UUID)) + + if err := o.validateUploadID(formats); err != nil { + return err + } + + return nil +} + +// validateUploadID carries on validations for parameter UploadID +func (o *GetUploadStatusParams) validateUploadID(formats strfmt.Registry) error { + + if err := validate.FormatOf("uploadId", "path", "uuid", o.UploadID.String(), formats); err != nil { + return err + } + return nil +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go new file mode 100644 index 00000000000..7b6b4b15b7d --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_responses.go @@ -0,0 +1,177 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/transcom/mymove/pkg/gen/internalmessages" +) + +// GetUploadStatusOKCode is the HTTP code returned for type GetUploadStatusOK +const GetUploadStatusOKCode int = 200 + +/* +GetUploadStatusOK the requested upload status + +swagger:response getUploadStatusOK +*/ +type GetUploadStatusOK struct { + + /* + In: Body + */ + Payload string `json:"body,omitempty"` +} + +// NewGetUploadStatusOK creates GetUploadStatusOK with default headers values +func NewGetUploadStatusOK() *GetUploadStatusOK { + + return &GetUploadStatusOK{} +} + +// WithPayload adds the payload to the get upload status o k response +func (o *GetUploadStatusOK) WithPayload(payload string) *GetUploadStatusOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get upload status o k response +func (o *GetUploadStatusOK) SetPayload(payload string) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(200) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + +// GetUploadStatusBadRequestCode is the HTTP code returned for type GetUploadStatusBadRequest +const GetUploadStatusBadRequestCode int = 400 + +/* +GetUploadStatusBadRequest invalid request + +swagger:response getUploadStatusBadRequest +*/ +type GetUploadStatusBadRequest struct { + + /* + In: Body + */ + Payload *internalmessages.InvalidRequestResponsePayload `json:"body,omitempty"` +} + +// NewGetUploadStatusBadRequest creates GetUploadStatusBadRequest with default headers values +func NewGetUploadStatusBadRequest() *GetUploadStatusBadRequest { + + return &GetUploadStatusBadRequest{} +} + +// WithPayload adds the payload to the get upload status bad request response +func (o *GetUploadStatusBadRequest) WithPayload(payload *internalmessages.InvalidRequestResponsePayload) *GetUploadStatusBadRequest { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the get upload status bad request response +func (o *GetUploadStatusBadRequest) SetPayload(payload *internalmessages.InvalidRequestResponsePayload) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *GetUploadStatusBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(400) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} + +// GetUploadStatusForbiddenCode is the HTTP code returned for type GetUploadStatusForbidden +const GetUploadStatusForbiddenCode int = 403 + +/* +GetUploadStatusForbidden not authorized + +swagger:response getUploadStatusForbidden +*/ +type GetUploadStatusForbidden struct { +} + +// NewGetUploadStatusForbidden creates GetUploadStatusForbidden with default headers values +func NewGetUploadStatusForbidden() *GetUploadStatusForbidden { + + return &GetUploadStatusForbidden{} +} + +// WriteResponse to the client +func (o *GetUploadStatusForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(403) +} + +// GetUploadStatusNotFoundCode is the HTTP code returned for type GetUploadStatusNotFound +const GetUploadStatusNotFoundCode int = 404 + +/* +GetUploadStatusNotFound not found + +swagger:response getUploadStatusNotFound +*/ +type GetUploadStatusNotFound struct { +} + +// NewGetUploadStatusNotFound creates GetUploadStatusNotFound with default headers values +func NewGetUploadStatusNotFound() *GetUploadStatusNotFound { + + return &GetUploadStatusNotFound{} +} + +// WriteResponse to the client +func (o *GetUploadStatusNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(404) +} + +// GetUploadStatusInternalServerErrorCode is the HTTP code returned for type GetUploadStatusInternalServerError +const GetUploadStatusInternalServerErrorCode int = 500 + +/* +GetUploadStatusInternalServerError server error + +swagger:response getUploadStatusInternalServerError +*/ +type GetUploadStatusInternalServerError struct { +} + +// NewGetUploadStatusInternalServerError creates GetUploadStatusInternalServerError with default headers values +func NewGetUploadStatusInternalServerError() *GetUploadStatusInternalServerError { + + return &GetUploadStatusInternalServerError{} +} + +// WriteResponse to the client +func (o *GetUploadStatusInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses + + rw.WriteHeader(500) +} diff --git a/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go new file mode 100644 index 00000000000..276a011d780 --- /dev/null +++ b/pkg/gen/internalapi/internaloperations/uploads/get_upload_status_urlbuilder.go @@ -0,0 +1,101 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package uploads + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" + "strings" + + "github.com/go-openapi/strfmt" +) + +// GetUploadStatusURL generates an URL for the get upload status operation +type GetUploadStatusURL struct { + UploadID strfmt.UUID + + _basePath string + // avoid unkeyed usage + _ struct{} +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetUploadStatusURL) WithBasePath(bp string) *GetUploadStatusURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *GetUploadStatusURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *GetUploadStatusURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/uploads/{uploadId}/status" + + uploadID := o.UploadID.String() + if uploadID != "" { + _path = strings.Replace(_path, "{uploadId}", uploadID, -1) + } else { + return nil, errors.New("uploadId is required on GetUploadStatusURL") + } + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/internal" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *GetUploadStatusURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *GetUploadStatusURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *GetUploadStatusURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on GetUploadStatusURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on GetUploadStatusURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *GetUploadStatusURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/pkg/handlers/authentication/auth.go b/pkg/handlers/authentication/auth.go index 226cbe88e1a..2d69ea0ec8f 100644 --- a/pkg/handlers/authentication/auth.go +++ b/pkg/handlers/authentication/auth.go @@ -220,6 +220,7 @@ var allowedRoutes = map[string]bool{ "uploads.deleteUpload": true, "users.showLoggedInUser": true, "okta_profile.showOktaInfo": true, + "uploads.getUploadStatus": true, } // checkIfRouteIsAllowed checks to see if the route is one of the ones that should be allowed through without stricter diff --git a/pkg/handlers/internalapi/api.go b/pkg/handlers/internalapi/api.go index 91673f1f838..4f9cc937add 100644 --- a/pkg/handlers/internalapi/api.go +++ b/pkg/handlers/internalapi/api.go @@ -169,6 +169,7 @@ func NewInternalAPI(handlerConfig handlers.HandlerConfig) *internalops.MymoveAPI internalAPI.UploadsCreateUploadHandler = CreateUploadHandler{handlerConfig} internalAPI.UploadsDeleteUploadHandler = DeleteUploadHandler{handlerConfig, upload.NewUploadInformationFetcher()} internalAPI.UploadsDeleteUploadsHandler = DeleteUploadsHandler{handlerConfig} + internalAPI.UploadsGetUploadStatusHandler = GetUploadStatusHandler{handlerConfig, upload.NewUploadInformationFetcher()} internalAPI.QueuesShowQueueHandler = ShowQueueHandler{handlerConfig} internalAPI.OfficeApproveMoveHandler = ApproveMoveHandler{handlerConfig, moveRouter} diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 4167d7ed2b8..359f510fc3e 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -246,6 +246,25 @@ func (h DeleteUploadsHandler) Handle(params uploadop.DeleteUploadsParams) middle }) } +// UploadStatusHandler returns status of an upload +type GetUploadStatusHandler struct { + handlers.HandlerConfig + services.UploadInformationFetcher +} + +// Handle returns status of an upload +func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { + return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, + func(appCtx appcontext.AppContext) (middleware.Responder, error) { + // uploadID, _ := uuid.FromString(params.UploadID.String()) + // _, err := models.FetchUserUploadFromUploadID(appCtx.DB(), appCtx.Session(), uploadID) + // if err != nil { + // return handlers.ResponseForError(appCtx.Logger(), err), err + // } + return uploadop.NewGetUploadStatusOK().WithPayload("CLEAN"), nil + }) +} + func (h CreatePPMUploadHandler) Handle(params ppmop.CreatePPMUploadParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index 36119617912..773caf8cea1 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -447,6 +447,50 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { suite.NotNil(queriedUpload.DeletedAt) } +func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { + fakeS3 := storageTest.NewFakeS3Storage(true) + + move := factory.BuildMove(suite.DB(), nil, nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: move.Orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + + file := suite.Fixture(FixturePDF) + fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + + params := uploadop.NewGetUploadStatusParams() + params.UploadID = strfmt.UUID(uploadUser1.ID.String()) + + req := &http.Request{} + req = suite.AuthenticateRequest(req, uploadUser1.Document.ServiceMember) + params.HTTPRequest = req + + handlerConfig := suite.HandlerConfig() + handlerConfig.SetFileStorer(fakeS3) + uploadInformationFetcher := upload.NewUploadInformationFetcher() + handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} + + response := handler.Handle(params) + + res, ok := response.(*uploadop.GetUploadStatusOK) + suite.True(ok) + + queriedUpload := models.Upload{} + err := suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) + suite.Nil(err) + suite.Equal("CLEAN", res.Payload) +} + func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { suite.Run("uploads .xls file", func() { fakeS3 := storageTest.NewFakeS3Storage(true) diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index 8a94763049e..619648415ee 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -18,6 +18,7 @@ import { filenameFromPath } from 'utils/formatters'; import AsyncPacketDownloadLink from 'shared/AsyncPacketDownloadLink/AsyncPacketDownloadLink'; import { UPLOAD_DOC_STATUS, UPLOAD_SCAN_STATUS } from 'shared/constants'; import Alert from 'shared/Alert'; +import { getUploadStatus } from 'services/internalApi'; /** * TODO @@ -43,6 +44,8 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI useEffect(() => { if (isFileUploading) { setFileStatus(UPLOAD_DOC_STATUS.UPLOADING); + } else { + setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING); } }, [isFileUploading]); @@ -84,7 +87,9 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI useEffect(() => { setRotationValue(selectedFile?.rotation || 0); const handleFileProcessingStatus = async () => { - if (selectedFile.status === UPLOAD_SCAN_STATUS.PROCESSING) { + const scanStatus = await getUploadStatus(selectedFile.id); + setFileStatus(scanStatus); + if (scanStatus === UPLOAD_SCAN_STATUS.PROCESSING) { await new Promise((resolve) => { setTimeout(resolve, 3000); }).then(() => setFileStatus(UPLOAD_DOC_STATUS.SCANNING)); @@ -94,7 +99,7 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI await new Promise((resolve) => { setTimeout(resolve, 3000); }).then(() => setFileStatus('LOADED')); - } else if (selectedFile.status === UPLOAD_SCAN_STATUS.CLEAN) { + } else if (scanStatus === UPLOAD_SCAN_STATUS.CLEAN) { setFileStatus('LOADED'); } }; diff --git a/src/services/internalApi.js b/src/services/internalApi.js index aec11ed66d1..ba470b2f271 100644 --- a/src/services/internalApi.js +++ b/src/services/internalApi.js @@ -207,6 +207,18 @@ export async function createUpload(file) { ); } +export async function getUploadStatus(uploadId) { + return makeInternalRequest( + 'uploads.getUploadStatus', + { + uploadId, + }, + { + normalize: false, + }, + ); +} + export async function createUploadForAmendedOrdersDocument(file, ordersId) { return makeInternalRequest( 'orders.uploadAmendedOrders', diff --git a/swagger-def/internal.yaml b/swagger-def/internal.yaml index c26a53eb8ed..dd29aee12b1 100644 --- a/swagger-def/internal.yaml +++ b/swagger-def/internal.yaml @@ -3393,6 +3393,41 @@ paths: description: not found '500': description: server error + + /uploads/{uploadId}/status: + get: + summary: Returns status of an upload + description: Returns status of an upload based on antivirus run + operationId: getUploadStatus + tags: + - uploads + parameters: + - in: path + name: uploadId + type: string + format: uuid + required: true + description: UUID of the upload to return status of + responses: + '200': + description: the requested upload status + schema: + type: string + enum: + - INFECTED + - CLEAN + - PROCESSING + readOnly: true + '400': + description: invalid request + schema: + $ref: '#/definitions/InvalidRequestResponsePayload' + '403': + description: not authorized + '404': + description: not found + '500': + description: server error /service_members: post: summary: Creates service member for a logged-in user diff --git a/swagger/internal.yaml b/swagger/internal.yaml index 419ede47b9d..8beffbdb284 100644 --- a/swagger/internal.yaml +++ b/swagger/internal.yaml @@ -5106,6 +5106,40 @@ paths: description: not found '500': description: server error + /uploads/{uploadId}/status: + get: + summary: Returns status of an upload + description: Returns status of an upload based on antivirus run + operationId: getUploadStatus + tags: + - uploads + parameters: + - in: path + name: uploadId + type: string + format: uuid + required: true + description: UUID of the upload to return status of + responses: + '200': + description: the requested upload status + schema: + type: string + enum: + - INFECTED + - CLEAN + - PROCESSING + readOnly: true + '400': + description: invalid request + schema: + $ref: '#/definitions/InvalidRequestResponsePayload' + '403': + description: not authorized + '404': + description: not found + '500': + description: server error /service_members: post: summary: Creates service member for a logged-in user From 3201e2301ef986a78ab833ba0cff5201e72c7459 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 29 Nov 2024 19:47:38 +0000 Subject: [PATCH 04/24] B-21669 - mock with custom endpoint using eventsource and text/event-stream --- pkg/gen/internalapi/configure_mymove.go | 1 + pkg/gen/internalapi/doc.go | 1 + pkg/gen/internalapi/embedded_spec.go | 6 +++ .../internaloperations/mymove_api.go | 9 ++++ pkg/handlers/internalapi/api.go | 1 + pkg/handlers/internalapi/uploads.go | 30 ++++++++++- .../DocumentViewer/DocumentViewer.jsx | 54 ++++++++++++------- src/services/internalApi.js | 12 ----- swagger-def/internal.yaml | 2 + swagger/internal.yaml | 2 + 10 files changed, 87 insertions(+), 31 deletions(-) diff --git a/pkg/gen/internalapi/configure_mymove.go b/pkg/gen/internalapi/configure_mymove.go index 60cf2568da3..be1f6579db9 100644 --- a/pkg/gen/internalapi/configure_mymove.go +++ b/pkg/gen/internalapi/configure_mymove.go @@ -60,6 +60,7 @@ func configureAPI(api *internaloperations.MymoveAPI) http.Handler { api.BinProducer = runtime.ByteStreamProducer() api.JSONProducer = runtime.JSONProducer() + api.TextEventStreamProducer = runtime.ByteStreamProducer() // You may change here the memory limit for this multipart form parser. Below is the default (32 MB). // ppm.CreatePPMUploadMaxParseMemory = 32 << 20 diff --git a/pkg/gen/internalapi/doc.go b/pkg/gen/internalapi/doc.go index 463e7be3e81..f8040028e22 100644 --- a/pkg/gen/internalapi/doc.go +++ b/pkg/gen/internalapi/doc.go @@ -22,6 +22,7 @@ // Produces: // - application/pdf // - application/json +// - text/event-stream // // swagger:meta package internalapi diff --git a/pkg/gen/internalapi/embedded_spec.go b/pkg/gen/internalapi/embedded_spec.go index 5bb44a88071..df13bc2ec42 100644 --- a/pkg/gen/internalapi/embedded_spec.go +++ b/pkg/gen/internalapi/embedded_spec.go @@ -3237,6 +3237,9 @@ func init() { "/uploads/{uploadId}/status": { "get": { "description": "Returns status of an upload based on antivirus run", + "produces": [ + "text/event-stream" + ], "tags": [ "uploads" ], @@ -12114,6 +12117,9 @@ func init() { "/uploads/{uploadId}/status": { "get": { "description": "Returns status of an upload based on antivirus run", + "produces": [ + "text/event-stream" + ], "tags": [ "uploads" ], diff --git a/pkg/gen/internalapi/internaloperations/mymove_api.go b/pkg/gen/internalapi/internaloperations/mymove_api.go index 6fe72081ad1..848626068e6 100644 --- a/pkg/gen/internalapi/internaloperations/mymove_api.go +++ b/pkg/gen/internalapi/internaloperations/mymove_api.go @@ -66,6 +66,7 @@ func NewMymoveAPI(spec *loads.Document) *MymoveAPI { BinProducer: runtime.ByteStreamProducer(), JSONProducer: runtime.JSONProducer(), + TextEventStreamProducer: runtime.ByteStreamProducer(), OfficeApproveMoveHandler: office.ApproveMoveHandlerFunc(func(params office.ApproveMoveParams) middleware.Responder { return middleware.NotImplemented("operation office.ApproveMove has not yet been implemented") @@ -323,6 +324,9 @@ type MymoveAPI struct { // JSONProducer registers a producer for the following mime types: // - application/json JSONProducer runtime.Producer + // TextEventStreamProducer registers a producer for the following mime types: + // - text/event-stream + TextEventStreamProducer runtime.Producer // OfficeApproveMoveHandler sets the operation handler for the approve move operation OfficeApproveMoveHandler office.ApproveMoveHandler @@ -546,6 +550,9 @@ func (o *MymoveAPI) Validate() error { if o.JSONProducer == nil { unregistered = append(unregistered, "JSONProducer") } + if o.TextEventStreamProducer == nil { + unregistered = append(unregistered, "TextEventStreamProducer") + } if o.OfficeApproveMoveHandler == nil { unregistered = append(unregistered, "office.ApproveMoveHandler") @@ -809,6 +816,8 @@ func (o *MymoveAPI) ProducersFor(mediaTypes []string) map[string]runtime.Produce result["application/pdf"] = o.BinProducer case "application/json": result["application/json"] = o.JSONProducer + case "text/event-stream": + result["text/event-stream"] = o.TextEventStreamProducer } if p, ok := o.customProducers[mt]; ok { diff --git a/pkg/handlers/internalapi/api.go b/pkg/handlers/internalapi/api.go index 4f9cc937add..718d8fcd922 100644 --- a/pkg/handlers/internalapi/api.go +++ b/pkg/handlers/internalapi/api.go @@ -183,6 +183,7 @@ func NewInternalAPI(handlerConfig handlers.HandlerConfig) *internalops.MymoveAPI internalAPI.PpmShowAOAPacketHandler = showAOAPacketHandler{handlerConfig, SSWPPMComputer, SSWPPMGenerator, AOAPacketCreator} internalAPI.RegisterProducer(uploader.FileTypePDF, PDFProducer()) + internalAPI.TextEventStreamProducer = runtime.ByteStreamProducer() internalAPI.PostalCodesValidatePostalCodeWithRateDataHandler = ValidatePostalCodeWithRateDataHandler{ handlerConfig, diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 359f510fc3e..6a701304383 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -3,9 +3,12 @@ package internalapi import ( "fmt" "io" + "net/http" "path/filepath" "regexp" + "strconv" "strings" + "time" "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" @@ -252,6 +255,30 @@ type GetUploadStatusHandler struct { services.UploadInformationFetcher } +type CustomNewUploadStatusOK struct{} + +func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + id_counter := 0 + + for range 2 { + resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: PROCESSING\n\n") + if err := producer.Produce(rw, resProcess); err != nil { + panic(err) // let the recovery middleware deal with this + } + if f, ok := rw.(http.Flusher); ok { + f.Flush() + } + + time.Sleep(4 * time.Second) + id_counter++ + } + + resClean := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: CLEAN\n\n") + if err := producer.Produce(rw, resClean); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + // Handle returns status of an upload func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, @@ -261,7 +288,8 @@ func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) mi // if err != nil { // return handlers.ResponseForError(appCtx.Logger(), err), err // } - return uploadop.NewGetUploadStatusOK().WithPayload("CLEAN"), nil + + return &CustomNewUploadStatusOK{}, nil }) } diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index 619648415ee..ef3b3e5f3c2 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -18,7 +18,6 @@ import { filenameFromPath } from 'utils/formatters'; import AsyncPacketDownloadLink from 'shared/AsyncPacketDownloadLink/AsyncPacketDownloadLink'; import { UPLOAD_DOC_STATUS, UPLOAD_SCAN_STATUS } from 'shared/constants'; import Alert from 'shared/Alert'; -import { getUploadStatus } from 'services/internalApi'; /** * TODO @@ -86,25 +85,44 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI useEffect(() => { setRotationValue(selectedFile?.rotation || 0); - const handleFileProcessingStatus = async () => { - const scanStatus = await getUploadStatus(selectedFile.id); - setFileStatus(scanStatus); - if (scanStatus === UPLOAD_SCAN_STATUS.PROCESSING) { - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus(UPLOAD_DOC_STATUS.SCANNING)); - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING)); - await new Promise((resolve) => { - setTimeout(resolve, 3000); - }).then(() => setFileStatus('LOADED')); - } else if (scanStatus === UPLOAD_SCAN_STATUS.CLEAN) { - setFileStatus('LOADED'); + + if (isFileUploading) return undefined; + + const handleFileProcessing = async (newStatus) => { + if (newStatus === UPLOAD_SCAN_STATUS.PROCESSING) { + setFileStatus(UPLOAD_DOC_STATUS.SCANNING); + } else if (newStatus === UPLOAD_SCAN_STATUS.CLEAN) { + setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING); + } else if (newStatus === UPLOAD_SCAN_STATUS.INFECTED) { + setFileStatus(UPLOAD_DOC_STATUS.INFECTED); + } else { + setFileStatus(null); + } + }; + + const sse = new EventSource(`/internal/uploads/${selectedFile.id}/status`, { withCredentials: true }); + sse.onmessage = (event) => { + if (event.data === UPLOAD_SCAN_STATUS.CLEAN || event.data === UPLOAD_SCAN_STATUS.INFECTED) { + sse.close(); } + handleFileProcessing(event.data); + }; + sse.onerror = () => { + setFileStatus(null); }; - handleFileProcessingStatus(); - }, [selectedFile]); + + return () => { + sse.close(); + }; + }, [selectedFile, isFileUploading]); + + useEffect(() => { + if (fileStatus === 'ESTABLISHING') { + new Promise((resolve) => { + setTimeout(resolve, 3000); + }).then(() => setFileStatus(UPLOAD_DOC_STATUS.LOADED)); + } + }, [fileStatus]); const fileType = useRef(selectedFile?.contentType); diff --git a/src/services/internalApi.js b/src/services/internalApi.js index ba470b2f271..aec11ed66d1 100644 --- a/src/services/internalApi.js +++ b/src/services/internalApi.js @@ -207,18 +207,6 @@ export async function createUpload(file) { ); } -export async function getUploadStatus(uploadId) { - return makeInternalRequest( - 'uploads.getUploadStatus', - { - uploadId, - }, - { - normalize: false, - }, - ); -} - export async function createUploadForAmendedOrdersDocument(file, ordersId) { return makeInternalRequest( 'orders.uploadAmendedOrders', diff --git a/swagger-def/internal.yaml b/swagger-def/internal.yaml index dd29aee12b1..7e5afa5f16d 100644 --- a/swagger-def/internal.yaml +++ b/swagger-def/internal.yaml @@ -3399,6 +3399,8 @@ paths: summary: Returns status of an upload description: Returns status of an upload based on antivirus run operationId: getUploadStatus + produces: + - text/event-stream tags: - uploads parameters: diff --git a/swagger/internal.yaml b/swagger/internal.yaml index 8beffbdb284..015cea44b40 100644 --- a/swagger/internal.yaml +++ b/swagger/internal.yaml @@ -5111,6 +5111,8 @@ paths: summary: Returns status of an upload description: Returns status of an upload based on antivirus run operationId: getUploadStatus + produces: + - text/event-stream tags: - uploads parameters: From ebdbb93ae3b6b0fdfb20b01bdb16f678e5f6066a Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 3 Dec 2024 21:53:09 +0000 Subject: [PATCH 05/24] B-21669 - fix race on useEffect. --- pkg/gen/internalapi/configure_mymove.go | 5 ++- .../internaloperations/mymove_api.go | 5 ++- .../DocumentViewer/DocumentViewer.jsx | 36 ++++++++++--------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/pkg/gen/internalapi/configure_mymove.go b/pkg/gen/internalapi/configure_mymove.go index be1f6579db9..e05689357e7 100644 --- a/pkg/gen/internalapi/configure_mymove.go +++ b/pkg/gen/internalapi/configure_mymove.go @@ -4,6 +4,7 @@ package internalapi import ( "crypto/tls" + "io" "net/http" "github.com/go-openapi/errors" @@ -60,7 +61,9 @@ func configureAPI(api *internaloperations.MymoveAPI) http.Handler { api.BinProducer = runtime.ByteStreamProducer() api.JSONProducer = runtime.JSONProducer() - api.TextEventStreamProducer = runtime.ByteStreamProducer() + api.TextEventStreamProducer = runtime.ProducerFunc(func(w io.Writer, data interface{}) error { + return errors.NotImplemented("textEventStream producer has not yet been implemented") + }) // You may change here the memory limit for this multipart form parser. Below is the default (32 MB). // ppm.CreatePPMUploadMaxParseMemory = 32 << 20 diff --git a/pkg/gen/internalapi/internaloperations/mymove_api.go b/pkg/gen/internalapi/internaloperations/mymove_api.go index 848626068e6..44b0acdf964 100644 --- a/pkg/gen/internalapi/internaloperations/mymove_api.go +++ b/pkg/gen/internalapi/internaloperations/mymove_api.go @@ -7,6 +7,7 @@ package internaloperations import ( "fmt" + "io" "net/http" "strings" @@ -66,7 +67,9 @@ func NewMymoveAPI(spec *loads.Document) *MymoveAPI { BinProducer: runtime.ByteStreamProducer(), JSONProducer: runtime.JSONProducer(), - TextEventStreamProducer: runtime.ByteStreamProducer(), + TextEventStreamProducer: runtime.ProducerFunc(func(w io.Writer, data interface{}) error { + return errors.NotImplemented("textEventStream producer has not yet been implemented") + }), OfficeApproveMoveHandler: office.ApproveMoveHandlerFunc(func(params office.ApproveMoveParams) middleware.Responder { return middleware.NotImplemented("operation office.ApproveMove has not yet been implemented") diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index ef3b3e5f3c2..db856eb49d0 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -43,10 +43,10 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI useEffect(() => { if (isFileUploading) { setFileStatus(UPLOAD_DOC_STATUS.UPLOADING); - } else { - setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING); + } else if (selectedFile) { + setFileStatus(null); } - }, [isFileUploading]); + }, [isFileUploading, selectedFile]); const { mutate: mutateUploads } = useMutation(updateUpload, { onSuccess: async (data, variables) => { @@ -95,31 +95,33 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI setFileStatus(UPLOAD_DOC_STATUS.ESTABLISHING); } else if (newStatus === UPLOAD_SCAN_STATUS.INFECTED) { setFileStatus(UPLOAD_DOC_STATUS.INFECTED); - } else { - setFileStatus(null); } }; - const sse = new EventSource(`/internal/uploads/${selectedFile.id}/status`, { withCredentials: true }); - sse.onmessage = (event) => { - if (event.data === UPLOAD_SCAN_STATUS.CLEAN || event.data === UPLOAD_SCAN_STATUS.INFECTED) { + let sse; + if (selectedFile) { + sse = new EventSource(`/internal/uploads/${selectedFile.id}/status`, { withCredentials: true }); + sse.onmessage = (event) => { + handleFileProcessing(event.data); + if (event.data === UPLOAD_SCAN_STATUS.CLEAN || event.data === UPLOAD_SCAN_STATUS.INFECTED) { + sse.close(); + } + }; + sse.onerror = () => { sse.close(); - } - handleFileProcessing(event.data); - }; - sse.onerror = () => { - setFileStatus(null); - }; + setFileStatus(null); + }; + } return () => { - sse.close(); + sse?.close(); }; }, [selectedFile, isFileUploading]); useEffect(() => { if (fileStatus === 'ESTABLISHING') { new Promise((resolve) => { - setTimeout(resolve, 3000); + setTimeout(resolve, 5000); }).then(() => setFileStatus(UPLOAD_DOC_STATUS.LOADED)); } }, [fileStatus]); @@ -143,7 +145,7 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI closeMenu(); }; - if (fileStatus && fileStatus !== 'LOADED') { + if (fileStatus !== 'LOADED') { return ( {fileStatus === UPLOAD_DOC_STATUS.UPLOADING && 'Uploading'} From 07e55ebfb06c01f632fb9f7fb922d498c15697f5 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 3 Dec 2024 22:20:28 +0000 Subject: [PATCH 06/24] B-21669 - fix uploading status. --- src/components/DocumentViewer/DocumentViewer.jsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/components/DocumentViewer/DocumentViewer.jsx b/src/components/DocumentViewer/DocumentViewer.jsx index db856eb49d0..cbb1af0fb74 100644 --- a/src/components/DocumentViewer/DocumentViewer.jsx +++ b/src/components/DocumentViewer/DocumentViewer.jsx @@ -43,10 +43,8 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI useEffect(() => { if (isFileUploading) { setFileStatus(UPLOAD_DOC_STATUS.UPLOADING); - } else if (selectedFile) { - setFileStatus(null); } - }, [isFileUploading, selectedFile]); + }, [isFileUploading]); const { mutate: mutateUploads } = useMutation(updateUpload, { onSuccess: async (data, variables) => { @@ -128,7 +126,11 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI const fileType = useRef(selectedFile?.contentType); - if (!selectedFile || !fileStatus || selectedFile?.status === UPLOAD_SCAN_STATUS.INFECTED) { + if ( + (!selectedFile && fileStatus !== UPLOAD_DOC_STATUS.UPLOADING) || + !fileStatus || + selectedFile?.status === UPLOAD_SCAN_STATUS.INFECTED + ) { return ; } @@ -141,7 +143,6 @@ const DocumentViewer = ({ files, isFileUploading, allowDownload, paymentRequestI const handleSelectFile = (index) => { selectFile(index); - closeMenu(); }; From 29569156afb0aef638020ade1f22b1fffa79fcf5 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Mon, 9 Dec 2024 15:44:25 +0000 Subject: [PATCH 07/24] B-21669 - minimal test for draft. --- pkg/gen/internalapi/embedded_spec.go | 14 ++++++++++++++ pkg/handlers/internalapi/uploads_test.go | 4 ++-- swagger/internal.yaml | 9 +++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/pkg/gen/internalapi/embedded_spec.go b/pkg/gen/internalapi/embedded_spec.go index df13bc2ec42..b3ad7a2e71b 100644 --- a/pkg/gen/internalapi/embedded_spec.go +++ b/pkg/gen/internalapi/embedded_spec.go @@ -6428,6 +6428,13 @@ func init() { "x-omitempty": false, "example": false }, + "maxIncentive": { + "description": "The max amount the government will pay the service member to move their belongings based on the moving date, locations, and shipment weight.", + "type": "integer", + "format": "cents", + "x-nullable": true, + "x-omitempty": false + }, "movingExpenses": { "description": "All expense documentation receipt records of this PPM shipment.", "type": "array", @@ -15312,6 +15319,13 @@ func init() { "x-omitempty": false, "example": false }, + "maxIncentive": { + "description": "The max amount the government will pay the service member to move their belongings based on the moving date, locations, and shipment weight.", + "type": "integer", + "format": "cents", + "x-nullable": true, + "x-omitempty": false + }, "movingExpenses": { "description": "All expense documentation receipt records of this PPM shipment.", "type": "array", diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index 773caf8cea1..ebc6eb0373c 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -447,6 +447,7 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { suite.NotNil(queriedUpload.DeletedAt) } +// TODO: functioning test func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { fakeS3 := storageTest.NewFakeS3Storage(true) @@ -482,13 +483,12 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { response := handler.Handle(params) - res, ok := response.(*uploadop.GetUploadStatusOK) + _, ok := response.(*CustomNewUploadStatusOK) suite.True(ok) queriedUpload := models.Upload{} err := suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) suite.Nil(err) - suite.Equal("CLEAN", res.Payload) } func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { diff --git a/swagger/internal.yaml b/swagger/internal.yaml index 015cea44b40..fe88b2b541c 100644 --- a/swagger/internal.yaml +++ b/swagger/internal.yaml @@ -3702,6 +3702,15 @@ definitions: format: cents x-nullable: true x-omitempty: false + maxIncentive: + description: >- + The max amount the government will pay the service member to move + their belongings based on the moving date, locations, and shipment + weight. + type: integer + format: cents + x-nullable: true + x-omitempty: false finalIncentive: description: > The final calculated incentive for the PPM shipment. This does not From 110fd1288a896c7fa00272c3c04bcf232c7b4ed1 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Mon, 16 Dec 2024 21:34:33 +0000 Subject: [PATCH 08/24] B-21669 - switch to reading av_status value from db - intial work. --- migrations/app/migrations_manifest.txt | 1 + ...6184550_upload_doc_av_status_column.up.sql | 12 ++++++ .../internal/payloads/model_to_payload.go | 15 ++++++-- pkg/handlers/internalapi/uploads.go | 37 +++++++++++++++---- pkg/models/upload.go | 35 ++++++++++++------ 5 files changed, 77 insertions(+), 23 deletions(-) create mode 100644 migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql diff --git a/migrations/app/migrations_manifest.txt b/migrations/app/migrations_manifest.txt index d016b8adfe5..48eb46212df 100644 --- a/migrations/app/migrations_manifest.txt +++ b/migrations/app/migrations_manifest.txt @@ -1036,3 +1036,4 @@ 20241202163059_create_test_sequence_dev_env.up.sql 20241203024453_add_ppm_max_incentive_column.up.sql 20241204210208_retroactive_update_of_ppm_max_and_estimated_incentives_prd.up.sql +20241216184550_upload_doc_av_status_column.up.sql diff --git a/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql b/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql new file mode 100644 index 00000000000..a50693603e3 --- /dev/null +++ b/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql @@ -0,0 +1,12 @@ +-- Add enum and column to track the anti-virus processing and availability of an upload + +CREATE TYPE av_status_type AS ENUM ( + 'PROCESSING', + 'CLEAN', + 'INFECTED' +); + +ALTER TABLE uploads ADD COLUMN IF NOT EXISTS av_status av_status_type; -- default null, will update to match s3 on first access for column + +COMMENT ON TYPE av_status_type IS 'The matching type for the anti-virus status.'; +COMMENT ON COLUMN uploads.av_status IS 'Column to track the anti-virus status for s3.'; \ No newline at end of file diff --git a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go index 38beccf4ef9..73a75f6f417 100644 --- a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go +++ b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go @@ -418,12 +418,19 @@ func PayloadForUploadModel( CreatedAt: strfmt.DateTime(upload.CreatedAt), UpdatedAt: strfmt.DateTime(upload.UpdatedAt), } - tags, err := storer.Tags(upload.StorageKey) - if err != nil || len(tags) == 0 { - uploadPayload.Status = "PROCESSING" + + if upload.AVStatus == nil { + tags, err := storer.Tags(upload.StorageKey) + if err != nil || len(tags) == 0 { + uploadPayload.Status = "PROCESSING" + } else { + uploadPayload.Status = tags["av-status"] + // TODO: update db with the tags + } } else { - uploadPayload.Status = tags["av-status"] + uploadPayload.Status = string(*upload.AVStatus) } + return uploadPayload } diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 6a701304383..52809b8946e 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -255,11 +255,35 @@ type GetUploadStatusHandler struct { services.UploadInformationFetcher } -type CustomNewUploadStatusOK struct{} +type CustomNewUploadStatusOK struct { + params uploadop.GetUploadStatusParams + appCtx appcontext.AppContext +} func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { id_counter := 0 + // TODO: add check for permissions to view upload + + err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { + uploadId, err := uuid.FromString(o.params.UploadID.String()) + if err != nil { + panic(err) + } + uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) + if err != nil { + txnAppCtx.Logger().Error(err.Error()) + } + + txnAppCtx.Logger().Info("HELLOW: " + uploaded.UploadID.String()) + + return err + }) + + if err != nil { + o.appCtx.Logger().Error(err.Error()) + } + for range 2 { resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: PROCESSING\n\n") if err := producer.Produce(rw, resProcess); err != nil { @@ -283,13 +307,10 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { - // uploadID, _ := uuid.FromString(params.UploadID.String()) - // _, err := models.FetchUserUploadFromUploadID(appCtx.DB(), appCtx.Session(), uploadID) - // if err != nil { - // return handlers.ResponseForError(appCtx.Logger(), err), err - // } - - return &CustomNewUploadStatusOK{}, nil + return &CustomNewUploadStatusOK{ + params: params, + appCtx: h.AppContextFromRequest(params.HTTPRequest), + }, nil }) } diff --git a/pkg/models/upload.go b/pkg/models/upload.go index d6afc2d0d4a..0703dff29ca 100644 --- a/pkg/models/upload.go +++ b/pkg/models/upload.go @@ -25,19 +25,32 @@ const ( UploadTypeOFFICE UploadType = "OFFICE" ) +// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected +type AVStatusType string + +const ( + // AVStatusTypePROCESSING string PROCESSING + AVStatusTypePROCESSING AVStatusType = "PROCESSING" + // AVStatusTypeCLEAN string CLEAN + AVStatusTypeCLEAN AVStatusType = "CLEAN" + // AVStatusTypeINFECTED string INFECTED + AVStatusTypeINFECTED AVStatusType = "INFECTED" +) + // An Upload represents an uploaded file, such as an image or PDF. type Upload struct { - ID uuid.UUID `db:"id"` - Filename string `db:"filename"` - Bytes int64 `db:"bytes"` - Rotation *int64 `db:"rotation"` - ContentType string `db:"content_type"` - Checksum string `db:"checksum"` - StorageKey string `db:"storage_key"` - UploadType UploadType `db:"upload_type"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - DeletedAt *time.Time `db:"deleted_at"` + ID uuid.UUID `db:"id"` + Filename string `db:"filename"` + Bytes int64 `db:"bytes"` + Rotation *int64 `db:"rotation"` + ContentType string `db:"content_type"` + Checksum string `db:"checksum"` + StorageKey string `db:"storage_key"` + AVStatus *AVStatusType `db:"av_status"` + UploadType UploadType `db:"upload_type"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeletedAt *time.Time `db:"deleted_at"` } // TableName overrides the table name used by Pop. From 048fd7612e410b685ffd77188532a7d1835ec0ce Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 17 Dec 2024 01:40:27 +0000 Subject: [PATCH 09/24] B-21669 - continuously reading from the db. --- pkg/handlers/internalapi/uploads.go | 43 ++++++++++++++++------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 52809b8946e..2a0a7218a9c 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -265,30 +265,35 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer // TODO: add check for permissions to view upload - err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - uploadId, err := uuid.FromString(o.params.UploadID.String()) - if err != nil { - panic(err) - } - uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) - if err != nil { - txnAppCtx.Logger().Error(err.Error()) - } + for range 2 { - txnAppCtx.Logger().Info("HELLOW: " + uploaded.UploadID.String()) + err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { + uploadId, err := uuid.FromString(o.params.UploadID.String()) + if err != nil { + panic(err) + } + uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) + if err != nil { + txnAppCtx.Logger().Error(err.Error()) + } - return err - }) + uploadStatus := models.AVStatusTypePROCESSING + if uploaded.Upload.AVStatus != nil { + uploadStatus = *uploaded.Upload.AVStatus + } - if err != nil { - o.appCtx.Logger().Error(err.Error()) - } + resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) // let the recovery middleware deal with this + } - for range 2 { - resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: PROCESSING\n\n") - if err := producer.Produce(rw, resProcess); err != nil { - panic(err) // let the recovery middleware deal with this + return nil + }) + + if err != nil { + o.appCtx.Logger().Error(err.Error()) } + if f, ok := rw.(http.Flusher); ok { f.Flush() } From de88145844a888c092c9798b3a846bce92961910 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 24 Dec 2024 20:14:07 +0000 Subject: [PATCH 10/24] B-21669 - sqs initial setup. --- cmd/milmove/serve.go | 4 + go.mod | 1 + go.sum | 2 + pkg/handlers/config.go | 14 +++ pkg/handlers/config_test.go | 2 +- pkg/handlers/internalapi/uploads.go | 38 ++++--- pkg/notifications/notification_receiver.go | 112 +++++++++++++++++++++ 7 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 pkg/notifications/notification_receiver.go diff --git a/cmd/milmove/serve.go b/cmd/milmove/serve.go index 505936d3868..8e9d8878d82 100644 --- a/cmd/milmove/serve.go +++ b/cmd/milmove/serve.go @@ -478,6 +478,9 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool appCtx.Logger().Fatal("notification sender sending not enabled", zap.Error(err)) } + // Email + notificationReceiver, _ := notifications.InitReceiver(v, appCtx.Logger()) + routingConfig.BuildRoot = v.GetString(cli.BuildRootFlag) sendProductionInvoice := v.GetBool(cli.GEXSendProdInvoiceFlag) @@ -567,6 +570,7 @@ func buildRoutingConfig(appCtx appcontext.AppContext, v *viper.Viper, redisPool dtodRoutePlanner, fileStorer, notificationSender, + notificationReceiver, iwsPersonLookup, sendProductionInvoice, gexSender, diff --git a/go.mod b/go.mod index e528f684f9d..818321b5a34 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.78.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 + github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 github.com/aws/smithy-go v1.20.4 diff --git a/go.sum b/go.sum index edfdd1c49f0..441af02374b 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 h1:Cso4Ev/XauMVsbwdhYEoxg8rxZWw4 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 h1:wcfUsE2nqsXhEj68gxr7MnGXNPcBPKx0RW2DzBVgVlM= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3/go.mod h1:6Ul/Ir8oOCsI3dFN0prULK9fvpxP+WTYmlHDkFzaAVA= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 h1:DbjODDHumQBdJ3T+EO7AXVoFUeUhAsJYOdjStH5Ws4A= +github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6/go.mod h1:7idt3XszF6sE9WPS1GqZRiDJOxw4oPtlRBXodWnCGjU= github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 h1:7cjN4Wp3U3cud17TsnUxSomTwKzKQGUWdq/N1aWqgMk= github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8/go.mod h1:nUSNPaG8mv5rIu7EclHnFqZOjhreEUwRKENtKTtJ9aw= github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= diff --git a/pkg/handlers/config.go b/pkg/handlers/config.go index b4bb2026915..50d45ee1978 100644 --- a/pkg/handlers/config.go +++ b/pkg/handlers/config.go @@ -39,6 +39,7 @@ type HandlerConfig interface { ) http.Handler FileStorer() storage.FileStorer NotificationSender() notifications.NotificationSender + NotificationReceiver() notifications.NotificationReceiver HHGPlanner() route.Planner DTODPlanner() route.Planner CookieSecret() string @@ -66,6 +67,7 @@ type Config struct { dtodPlanner route.Planner storage storage.FileStorer notificationSender notifications.NotificationSender + notificationReceiver notifications.NotificationReceiver iwsPersonLookup iws.PersonLookup sendProductionInvoice bool senderToGex services.GexSender @@ -86,6 +88,7 @@ func NewHandlerConfig( dtodPlanner route.Planner, storage storage.FileStorer, notificationSender notifications.NotificationSender, + notificationReceiver notifications.NotificationReceiver, iwsPersonLookup iws.PersonLookup, sendProductionInvoice bool, senderToGex services.GexSender, @@ -103,6 +106,7 @@ func NewHandlerConfig( dtodPlanner: dtodPlanner, storage: storage, notificationSender: notificationSender, + notificationReceiver: notificationReceiver, iwsPersonLookup: iwsPersonLookup, sendProductionInvoice: sendProductionInvoice, senderToGex: senderToGex, @@ -247,6 +251,16 @@ func (c *Config) SetNotificationSender(sender notifications.NotificationSender) c.notificationSender = sender } +// NotificationReceiver returns the sender to use in the current context +func (c *Config) NotificationReceiver() notifications.NotificationReceiver { + return c.notificationReceiver +} + +// SetNotificationSender is a simple setter for AWS SQS private field +func (c *Config) SetNotificationReceiver(receiver notifications.NotificationReceiver) { + c.notificationReceiver = receiver +} + // SetPlanner is a simple setter for the route.Planner private field func (c *Config) SetPlanner(planner route.Planner) { c.planner = planner diff --git a/pkg/handlers/config_test.go b/pkg/handlers/config_test.go index 26595daea29..85c9ccbff7c 100644 --- a/pkg/handlers/config_test.go +++ b/pkg/handlers/config_test.go @@ -30,7 +30,7 @@ func (suite *ConfigSuite) TestConfigHandler() { appCtx := suite.AppContextForTest() sessionManagers := auth.SetupSessionManagers(nil, false, time.Duration(180*time.Second), time.Duration(180*time.Second)) - handler := NewHandlerConfig(appCtx.DB(), nil, "", nil, nil, nil, nil, nil, false, nil, nil, false, ApplicationTestServername(), sessionManagers, nil) + handler := NewHandlerConfig(appCtx.DB(), nil, "", nil, nil, nil, nil, nil, nil, false, nil, nil, false, ApplicationTestServername(), sessionManagers, nil) req, err := http.NewRequest("GET", "/", nil) suite.NoError(err) myMethodCalled := false diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 2a0a7218a9c..6962ba4aa8a 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -22,6 +22,7 @@ import ( "github.com/transcom/mymove/pkg/handlers" "github.com/transcom/mymove/pkg/handlers/internalapi/internal/payloads" "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/notifications" "github.com/transcom/mymove/pkg/services" "github.com/transcom/mymove/pkg/services/ppmshipment" weightticketparser "github.com/transcom/mymove/pkg/services/weight_ticket_parser" @@ -256,8 +257,9 @@ type GetUploadStatusHandler struct { } type CustomNewUploadStatusOK struct { - params uploadop.GetUploadStatusParams - appCtx appcontext.AppContext + params uploadop.GetUploadStatusParams + appCtx appcontext.AppContext + receiver notifications.NotificationReceiver } func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { @@ -265,22 +267,27 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer // TODO: add check for permissions to view upload + err := o.receiver.SubscribeToTopic(o.appCtx, notifications.NotificationFilter{}) + if err != nil { + o.appCtx.Logger().Error(err.Error()) + } + for range 2 { err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - uploadId, err := uuid.FromString(o.params.UploadID.String()) - if err != nil { - panic(err) - } - uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) - if err != nil { - txnAppCtx.Logger().Error(err.Error()) - } + // uploadId, err := uuid.FromString(o.params.UploadID.String()) + // if err != nil { + // panic(err) + // } + // uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) + // if err != nil { + // txnAppCtx.Logger().Error(err.Error()) + // } uploadStatus := models.AVStatusTypePROCESSING - if uploaded.Upload.AVStatus != nil { - uploadStatus = *uploaded.Upload.AVStatus - } + // if uploaded.Upload.AVStatus != nil { + // uploadStatus = *uploaded.Upload.AVStatus + // } resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") if produceErr := producer.Produce(rw, resProcess); produceErr != nil { @@ -313,8 +320,9 @@ func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) mi return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { return &CustomNewUploadStatusOK{ - params: params, - appCtx: h.AppContextFromRequest(params.HTTPRequest), + params: params, + appCtx: h.AppContextFromRequest(params.HTTPRequest), + receiver: h.NotificationReceiver(), }, nil }) } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go new file mode 100644 index 00000000000..79c81329962 --- /dev/null +++ b/pkg/notifications/notification_receiver.go @@ -0,0 +1,112 @@ +package notifications + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/spf13/viper" + "go.uber.org/zap" + + "github.com/transcom/mymove/pkg/appcontext" + "github.com/transcom/mymove/pkg/cli" +) + +// Notification is an interface for creating emails +type NotificationFilter struct { +} + +// NotificationSender is an interface for sending notifications +// +//go:generate mockery --name NotificationSender +type NotificationReceiver interface { + SubscribeToTopic(appCtx appcontext.AppContext, filter NotificationFilter) error +} + +// NotificationSendingContext provides context to a notification sender +type NotificationReceiverContext struct { + svc *sqs.Client +} + +// NewNotificationSender returns a new NotificationSendingContext +func NewNotificationReceiver(svc *sqs.Client) NotificationReceiverContext { + return NotificationReceiverContext{ + svc: svc, + } +} + +// SendNotification sends a one or more notifications for all supported mediums +func (n NotificationReceiverContext) SubscribeToTopic(appCtx appcontext.AppContext, filter NotificationFilter) error { + queueRaw := "testQueue" + queue := &queueRaw + + urlResult, _ := n.svc.GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{ + QueueName: queue, + }) + + result, err := n.svc.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ + QueueUrl: urlResult.QueueUrl, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }) + if err != nil { + appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + } else { + for _, val := range result.Messages { + appCtx.Logger().Info(*val.MessageId) + } + } + return err +} + +// InitEmail initializes the email backend +func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { + // if v.GetString(cli.EmailBackendFlag) == "ses" { + // // Setup Amazon SES (email) service TODO: This might be able + // // to be combined with the AWS Session that we're using for S3 + // // down below. + + // awsSESRegion := v.GetString(cli.AWSSESRegionFlag) + // awsSESDomain := v.GetString(cli.AWSSESDomainFlag) + // sysAdminEmail := v.GetString(cli.SysAdminEmail) + // logger.Info("Using ses email backend", + // zap.String("region", awsSESRegion), + // zap.String("domain", awsSESDomain)) + // cfg, err := config.LoadDefaultConfig(context.Background(), + // config.WithRegion(awsSESRegion), + // ) + // if err != nil { + // logger.Fatal("error loading ses aws config", zap.Error(err)) + // } + + // sesService := ses.NewFromConfig(cfg) + // input := &ses.GetAccountSendingEnabledInput{} + // result, err := sesService.GetAccountSendingEnabled(context.Background(), input) + // if err != nil || result == nil || !result.Enabled { + // logger.Error("email sending not enabled", zap.Error(err)) + // return NewNotificationSender(nil, awsSESDomain, sysAdminEmail), err + // } + // return NewNotificationSender(sesService, awsSESDomain, sysAdminEmail), nil + // } + + // domain := "milmovelocal" + // logger.Info("Using local email backend", zap.String("domain", domain)) + // return NewStubNotificationSender(domain), nil + + // Setup Amazon SES (email) service TODO: This might be able + // to be combined with the AWS Session that we're using for S3 + // down below. + + awsSESRegion := v.GetString(cli.AWSSESRegionFlag) + + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(awsSESRegion), + ) + if err != nil { + logger.Fatal("error loading ses aws config", zap.Error(err)) + } + + sqsService := sqs.NewFromConfig(cfg) + + return NewNotificationReceiver(sqsService), nil +} From b273296aee5c8d57b0ca88c6eb6b86c5b018b54b Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 27 Dec 2024 16:31:16 +0000 Subject: [PATCH 11/24] B-21669 - sqs working --- go.mod | 1 + go.sum | 2 + pkg/handlers/internalapi/uploads.go | 17 ++- pkg/notifications/notification_receiver.go | 130 ++++++++++++++++++--- 4 files changed, 132 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 818321b5a34..8681234fe7c 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.78.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 + github.com/aws/aws-sdk-go-v2/service/sns v1.31.8 github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 diff --git a/go.sum b/go.sum index 441af02374b..c272e10eca1 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0 h1:Cso4Ev/XauMVsbwdhYEoxg8rxZWw4 github.com/aws/aws-sdk-go-v2/service/s3 v1.59.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3 h1:wcfUsE2nqsXhEj68gxr7MnGXNPcBPKx0RW2DzBVgVlM= github.com/aws/aws-sdk-go-v2/service/ses v1.25.3/go.mod h1:6Ul/Ir8oOCsI3dFN0prULK9fvpxP+WTYmlHDkFzaAVA= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.8 h1:vRSk062d1SmaEVbiqFePkvYuhCTnW2JnPkUdt19nqeY= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.8/go.mod h1:wjhxA9hlVu75dCL/5Wcx8Cwmszvu6t0i8WEDypcB4+s= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6 h1:DbjODDHumQBdJ3T+EO7AXVoFUeUhAsJYOdjStH5Ws4A= github.com/aws/aws-sdk-go-v2/service/sqs v1.34.6/go.mod h1:7idt3XszF6sE9WPS1GqZRiDJOxw4oPtlRBXodWnCGjU= github.com/aws/aws-sdk-go-v2/service/ssm v1.52.8 h1:7cjN4Wp3U3cud17TsnUxSomTwKzKQGUWdq/N1aWqgMk= diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 6962ba4aa8a..5d7da99e99a 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -267,12 +267,19 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer // TODO: add check for permissions to view upload - err := o.receiver.SubscribeToTopic(o.appCtx, notifications.NotificationFilter{}) + notificationParams := notifications.NotificationQueueParams{ + Action: "ObjectTagsUpdated", + ObjectId: o.params.UploadID.String(), + } + + topicArn := "arn:aws-us-gov:sns:us-gov-west-1:021081706899:app_s3_tag_events" + + queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicArn, notificationParams) if err != nil { o.appCtx.Logger().Error(err.Error()) } - for range 2 { + for range 5 { err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { // uploadId, err := uuid.FromString(o.params.UploadID.String()) @@ -297,6 +304,12 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer return nil }) + o.appCtx.Logger().Info("Receiving...") + errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) + if errs != nil { + o.appCtx.Logger().Error(errs.Error()) + } + if err != nil { o.appCtx.Logger().Error(err.Error()) } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 79c81329962..248cc98709f 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -2,9 +2,14 @@ package notifications import ( "context" + "fmt" + "log" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/spf13/viper" "go.uber.org/zap" @@ -13,39 +18,131 @@ import ( ) // Notification is an interface for creating emails -type NotificationFilter struct { +type NotificationQueueParams struct { + // TODO: change to enum + Action string + ObjectId string } // NotificationSender is an interface for sending notifications // //go:generate mockery --name NotificationSender type NotificationReceiver interface { - SubscribeToTopic(appCtx appcontext.AppContext, filter NotificationFilter) error + CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) error } // NotificationSendingContext provides context to a notification sender type NotificationReceiverContext struct { - svc *sqs.Client + snsService *sns.Client + sqsService *sqs.Client } // NewNotificationSender returns a new NotificationSendingContext -func NewNotificationReceiver(svc *sqs.Client) NotificationReceiverContext { +func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client) NotificationReceiverContext { return NotificationReceiverContext{ - svc: svc, + snsService: snsService, + sqsService: sqsService, } } -// SendNotification sends a one or more notifications for all supported mediums -func (n NotificationReceiverContext) SubscribeToTopic(appCtx appcontext.AppContext, filter NotificationFilter) error { - queueRaw := "testQueue" - queue := &queueRaw +func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) { - urlResult, _ := n.svc.GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{ - QueueName: queue, - }) + queueName := fmt.Sprintf("%s_%s", params.Action, params.ObjectId) + + input := &sqs.CreateQueueInput{ + QueueName: &queueName, + Attributes: map[string]string{ + "MessageRetentionPeriod": "120", + }, + } + + // Create the SQS queue + result, err := n.sqsService.CreateQueue(context.Background(), input) + if err != nil { + log.Fatalf("Failed to create SQS queue, %v", err) + } + + // Get queue attributes to retrieve the ARN + attrInput := &sqs.GetQueueAttributesInput{ + QueueUrl: result.QueueUrl, + AttributeNames: []types.QueueAttributeName{ + types.QueueAttributeNameQueueArn, + }, + } + + attrResult, err := n.sqsService.GetQueueAttributes(context.Background(), attrInput) + if err != nil { + log.Fatalf("Failed to get queue attributes, %v", err) + } - result, err := n.svc.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ - QueueUrl: urlResult.QueueUrl, + queueArn := attrResult.Attributes[string(types.QueueAttributeNameQueueArn)] + + // Define the access policy + accessPolicy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowSNSPublish", + "Effect": "Allow", + "Principal": { + "Service": "sns.amazonaws.com" + }, + "Action": ["sqs:SendMessage"], + "Resource": "%s", + "Condition": { + "ArnEquals": { + "aws:SourceArn": "%s" + } + } + }] + }`, queueArn, topicArn) + + newAttributes := &sqs.SetQueueAttributesInput{ + QueueUrl: result.QueueUrl, + Attributes: map[string]string{ + "Policy": accessPolicy, + }, + } + + // TODO: need to figure this out on creation, the queue attributes can take up to 60 seconds to propogate + _, err = n.sqsService.SetQueueAttributes(context.Background(), newAttributes) + if err != nil { + log.Fatalf("Failed to set access policy on queue, %v", err) + } + + // Define the filter policy + filterPolicy := fmt.Sprintf(`{ + "detail": { + "object": { + "key": [ + {"suffix": "%s"} + ] + } + } + }`, params.ObjectId) + + // Create a subscription (replace with your actual endpoint) + subscribeInput := &sns.SubscribeInput{ + TopicArn: &topicArn, + Protocol: aws.String("sqs"), + Endpoint: &queueArn, + Attributes: map[string]string{ + "FilterPolicy": filterPolicy, + "FilterPolicyScope": "MessageBody", + }, + } + _, err = n.snsService.Subscribe(context.Background(), subscribeInput) + if err != nil { + log.Fatalf("Failed to create subscription, %v", err) + } + + return *result.QueueUrl, err +} + +// SendNotification sends a one or more notifications for all supported mediums +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) error { + result, err := n.sqsService.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ + QueueUrl: &queueUrl, MaxNumberOfMessages: 1, WaitTimeSeconds: 5, }) @@ -53,7 +150,7 @@ func (n NotificationReceiverContext) SubscribeToTopic(appCtx appcontext.AppConte appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) } else { for _, val := range result.Messages { - appCtx.Logger().Info(*val.MessageId) + appCtx.Logger().Info(*val.Body) } } return err @@ -106,7 +203,8 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err logger.Fatal("error loading ses aws config", zap.Error(err)) } + snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - return NewNotificationReceiver(sqsService), nil + return NewNotificationReceiver(snsService, sqsService), nil } From c94541365f77bfdf2125ed4575784cb4a1c87237 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 27 Dec 2024 18:29:05 +0000 Subject: [PATCH 12/24] B-21669 - sqs working w/ frontend. --- pkg/handlers/internalapi/uploads.go | 80 ++++++++++++---------- pkg/notifications/notification_receiver.go | 12 ++-- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 5d7da99e99a..7898c9ed934 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -8,7 +8,6 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" @@ -26,6 +25,7 @@ import ( "github.com/transcom/mymove/pkg/services" "github.com/transcom/mymove/pkg/services/ppmshipment" weightticketparser "github.com/transcom/mymove/pkg/services/weight_ticket_parser" + "github.com/transcom/mymove/pkg/storage" "github.com/transcom/mymove/pkg/uploader" uploaderpkg "github.com/transcom/mymove/pkg/uploader" ) @@ -260,16 +260,18 @@ type CustomNewUploadStatusOK struct { params uploadop.GetUploadStatusParams appCtx appcontext.AppContext receiver notifications.NotificationReceiver + storer storage.FileStorer } func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - id_counter := 0 // TODO: add check for permissions to view upload + uploadId := o.params.UploadID.String() + notificationParams := notifications.NotificationQueueParams{ Action: "ObjectTagsUpdated", - ObjectId: o.params.UploadID.String(), + ObjectId: uploadId, } topicArn := "arn:aws-us-gov:sns:us-gov-west-1:021081706899:app_s3_tag_events" @@ -279,53 +281,54 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer o.appCtx.Logger().Error(err.Error()) } - for range 5 { - - err := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - // uploadId, err := uuid.FromString(o.params.UploadID.String()) - // if err != nil { - // panic(err) - // } - // uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadId) - // if err != nil { - // txnAppCtx.Logger().Error(err.Error()) - // } - - uploadStatus := models.AVStatusTypePROCESSING - // if uploaded.Upload.AVStatus != nil { - // uploadStatus = *uploaded.Upload.AVStatus - // } - - resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") - if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) // let the recovery middleware deal with this - } - - return nil - }) - + id_counter := 0 + // Run for 120 seconds, 20 second long polling 6 times + for range 6 { o.appCtx.Logger().Info("Receiving...") - errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) + messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) if errs != nil { o.appCtx.Logger().Error(errs.Error()) } - if err != nil { - o.appCtx.Logger().Error(err.Error()) + if len(messages) != 0 { + errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { + + uploadUUID, err := uuid.FromString(uploadId) + if err != nil { + panic(err) + } + uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadUUID) + if err != nil { + txnAppCtx.Logger().Error(err.Error()) + } + + tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + + var uploadStatus models.AVStatusType + if err != nil || len(tags) == 0 { + uploadStatus = models.AVStatusTypePROCESSING + } else { + uploadStatus = models.AVStatusType(tags["av-status"]) + } + + resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) // let the recovery middleware deal with this + } + + return nil + }) + + if errTransaction != nil { + o.appCtx.Logger().Error(err.Error()) + } } if f, ok := rw.(http.Flusher); ok { f.Flush() } - - time.Sleep(4 * time.Second) id_counter++ } - - resClean := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: CLEAN\n\n") - if err := producer.Produce(rw, resClean); err != nil { - panic(err) // let the recovery middleware deal with this - } } // Handle returns status of an upload @@ -336,6 +339,7 @@ func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) mi params: params, appCtx: h.AppContextFromRequest(params.HTTPRequest), receiver: h.NotificationReceiver(), + storer: h.FileStorer(), }, nil }) } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 248cc98709f..fce4a9b700b 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -29,7 +29,7 @@ type NotificationQueueParams struct { //go:generate mockery --name NotificationSender type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) - ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) error + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) } // NotificationSendingContext provides context to a notification sender @@ -140,20 +140,16 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte } // SendNotification sends a one or more notifications for all supported mediums -func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) error { +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { result, err := n.sqsService.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ QueueUrl: &queueUrl, MaxNumberOfMessages: 1, - WaitTimeSeconds: 5, + WaitTimeSeconds: 20, }) if err != nil { appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) - } else { - for _, val := range result.Messages { - appCtx.Logger().Info(*val.Body) - } } - return err + return result.Messages, err } // InitEmail initializes the email backend From b692a9013e407aeb205034e4e8d4ae8a7fc998e2 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 27 Dec 2024 21:02:11 +0000 Subject: [PATCH 13/24] B-21669 - arn constructed from env vars. --- pkg/handlers/internalapi/uploads.go | 54 +++++++++++----- pkg/notifications/notification_receiver.go | 71 +++++++++------------- 2 files changed, 68 insertions(+), 57 deletions(-) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 7898c9ed934..d541a550073 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -269,14 +269,48 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer uploadId := o.params.UploadID.String() + uploadUUID, err := uuid.FromString(uploadId) + if err != nil { + panic(err) + } + + // Check current tag before event-driven wait for anti-virus + + uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) + if err != nil { + o.appCtx.Logger().Error(err.Error()) + } + + tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + var uploadStatus models.AVStatusType + if err != nil || len(tags) == 0 { + uploadStatus = models.AVStatusTypePROCESSING + } else { + uploadStatus = models.AVStatusType(tags["av-status"]) + } + + resProcess := []byte("id: 0\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) + } + + if f, ok := rw.(http.Flusher); ok { + f.Flush() + } + + if uploadStatus == models.AVStatusTypeCLEAN || uploadStatus == models.AVStatusTypeINFECTED { + return + } + + // Start waiting for tag updates + + topicName := "app_s3_tag_events" notificationParams := notifications.NotificationQueueParams{ - Action: "ObjectTagsUpdated", + Action: "ObjectTagsAdded", ObjectId: uploadId, } - topicArn := "arn:aws-us-gov:sns:us-gov-west-1:021081706899:app_s3_tag_events" - - queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicArn, notificationParams) + queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicName, notificationParams) if err != nil { o.appCtx.Logger().Error(err.Error()) } @@ -293,18 +327,8 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer if len(messages) != 0 { errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - uploadUUID, err := uuid.FromString(uploadId) - if err != nil { - panic(err) - } - uploaded, err := models.FetchUserUploadFromUploadID(txnAppCtx.DB(), txnAppCtx.Session(), uploadUUID) - if err != nil { - txnAppCtx.Logger().Error(err.Error()) - } - tags, err := o.storer.Tags(uploaded.Upload.StorageKey) - var uploadStatus models.AVStatusType if err != nil || len(tags) == 0 { uploadStatus = models.AVStatusTypePROCESSING } else { @@ -329,6 +353,8 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } id_counter++ } + + // TODO: add a close here after ends } // Handle returns status of an upload diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index fce4a9b700b..469fff74aab 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -34,51 +34,30 @@ type NotificationReceiver interface { // NotificationSendingContext provides context to a notification sender type NotificationReceiverContext struct { - snsService *sns.Client - sqsService *sqs.Client + snsService *sns.Client + sqsService *sqs.Client + awsRegion string + awsAccountId string } // NewNotificationSender returns a new NotificationSendingContext -func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client) NotificationReceiverContext { +func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ - snsService: snsService, - sqsService: sqsService, + snsService: snsService, + sqsService: sqsService, + awsRegion: awsRegion, + awsAccountId: awsAccountId, } } -func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) { +func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicName string, params NotificationQueueParams) (string, error) { queueName := fmt.Sprintf("%s_%s", params.Action, params.ObjectId) + queueArn := n.constructArn("sqs", queueName) + topicArn := n.constructArn("sns", topicName) - input := &sqs.CreateQueueInput{ - QueueName: &queueName, - Attributes: map[string]string{ - "MessageRetentionPeriod": "120", - }, - } - - // Create the SQS queue - result, err := n.sqsService.CreateQueue(context.Background(), input) - if err != nil { - log.Fatalf("Failed to create SQS queue, %v", err) - } - - // Get queue attributes to retrieve the ARN - attrInput := &sqs.GetQueueAttributesInput{ - QueueUrl: result.QueueUrl, - AttributeNames: []types.QueueAttributeName{ - types.QueueAttributeNameQueueArn, - }, - } - - attrResult, err := n.sqsService.GetQueueAttributes(context.Background(), attrInput) - if err != nil { - log.Fatalf("Failed to get queue attributes, %v", err) - } + // Create queue - queueArn := attrResult.Attributes[string(types.QueueAttributeNameQueueArn)] - - // Define the access policy accessPolicy := fmt.Sprintf(`{ "Version": "2012-10-17", "Statement": [{ @@ -97,20 +76,21 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte }] }`, queueArn, topicArn) - newAttributes := &sqs.SetQueueAttributesInput{ - QueueUrl: result.QueueUrl, + input := &sqs.CreateQueueInput{ + QueueName: &queueName, Attributes: map[string]string{ - "Policy": accessPolicy, + "MessageRetentionPeriod": "120", + "Policy": accessPolicy, }, } - // TODO: need to figure this out on creation, the queue attributes can take up to 60 seconds to propogate - _, err = n.sqsService.SetQueueAttributes(context.Background(), newAttributes) + result, err := n.sqsService.CreateQueue(context.Background(), input) if err != nil { - log.Fatalf("Failed to set access policy on queue, %v", err) + log.Fatalf("Failed to create SQS queue, %v", err) } - // Define the filter policy + // Create subscription + filterPolicy := fmt.Sprintf(`{ "detail": { "object": { @@ -121,7 +101,6 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte } }`, params.ObjectId) - // Create a subscription (replace with your actual endpoint) subscribeInput := &sns.SubscribeInput{ TopicArn: &topicArn, Protocol: aws.String("sqs"), @@ -190,7 +169,9 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err // to be combined with the AWS Session that we're using for S3 // down below. + // TODO: verify if we should change this param name to awsNotificationRegion awsSESRegion := v.GetString(cli.AWSSESRegionFlag) + awsAccountId := v.GetString("aws-account-id") cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(awsSESRegion), @@ -202,5 +183,9 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - return NewNotificationReceiver(snsService, sqsService), nil + return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil +} + +func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { + return fmt.Sprintf("arn:aws-us-gov:%s:%s:%s:%s", awsService, n.awsRegion, n.awsAccountId, endpointName) } From 60237c2571cb3c9e51dc82903cf88b6a9ce3f1f4 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 31 Dec 2024 20:06:42 +0000 Subject: [PATCH 14/24] B-21669 - remove unused avstatus on db; add cancel context for request close. --- migrations/app/migrations_manifest.txt | 1 - ...6184550_upload_doc_av_status_column.up.sql | 12 ---- .../internal/payloads/model_to_payload.go | 14 ++--- pkg/handlers/internalapi/uploads.go | 37 +++++++++--- pkg/models/upload.go | 35 ++++-------- pkg/notifications/notification_receiver.go | 56 ++++++++++++++----- 6 files changed, 88 insertions(+), 67 deletions(-) delete mode 100644 migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql diff --git a/migrations/app/migrations_manifest.txt b/migrations/app/migrations_manifest.txt index 065d83debd8..eb24094411c 100644 --- a/migrations/app/migrations_manifest.txt +++ b/migrations/app/migrations_manifest.txt @@ -1051,5 +1051,4 @@ 20241203024453_add_ppm_max_incentive_column.up.sql 20241204155919_update_ordering_proc.up.sql 20241204210208_retroactive_update_of_ppm_max_and_estimated_incentives_prd.up.sql -20241216184550_upload_doc_av_status_column.up.sql 20241227153723_remove_empty_string_emplid_values.up.sql diff --git a/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql b/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql deleted file mode 100644 index a50693603e3..00000000000 --- a/migrations/app/schema/20241216184550_upload_doc_av_status_column.up.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Add enum and column to track the anti-virus processing and availability of an upload - -CREATE TYPE av_status_type AS ENUM ( - 'PROCESSING', - 'CLEAN', - 'INFECTED' -); - -ALTER TABLE uploads ADD COLUMN IF NOT EXISTS av_status av_status_type; -- default null, will update to match s3 on first access for column - -COMMENT ON TYPE av_status_type IS 'The matching type for the anti-virus status.'; -COMMENT ON COLUMN uploads.av_status IS 'Column to track the anti-virus status for s3.'; \ No newline at end of file diff --git a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go index f24d3ea21fd..9550b4a11f9 100644 --- a/pkg/handlers/internalapi/internal/payloads/model_to_payload.go +++ b/pkg/handlers/internalapi/internal/payloads/model_to_payload.go @@ -454,16 +454,12 @@ func PayloadForUploadModel( UpdatedAt: strfmt.DateTime(upload.UpdatedAt), } - if upload.AVStatus == nil { - tags, err := storer.Tags(upload.StorageKey) - if err != nil || len(tags) == 0 { - uploadPayload.Status = "PROCESSING" - } else { - uploadPayload.Status = tags["av-status"] - // TODO: update db with the tags - } + tags, err := storer.Tags(upload.StorageKey) + if err != nil || len(tags) == 0 { + uploadPayload.Status = "PROCESSING" } else { - uploadPayload.Status = string(*upload.AVStatus) + uploadPayload.Status = tags["av-status"] + // TODO: update db with the tags } return uploadPayload diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index d541a550073..c8b8a4b13ab 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -1,6 +1,7 @@ package internalapi import ( + "context" "fmt" "io" "net/http" @@ -263,6 +264,18 @@ type CustomNewUploadStatusOK struct { storer storage.FileStorer } +// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected +type AVStatusType string + +const ( + // AVStatusTypePROCESSING string PROCESSING + AVStatusTypePROCESSING AVStatusType = "PROCESSING" + // AVStatusTypeCLEAN string CLEAN + AVStatusTypeCLEAN AVStatusType = "CLEAN" + // AVStatusTypeINFECTED string INFECTED + AVStatusTypeINFECTED AVStatusType = "INFECTED" +) + func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { // TODO: add check for permissions to view upload @@ -282,11 +295,11 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } tags, err := o.storer.Tags(uploaded.Upload.StorageKey) - var uploadStatus models.AVStatusType + var uploadStatus AVStatusType if err != nil || len(tags) == 0 { - uploadStatus = models.AVStatusTypePROCESSING + uploadStatus = AVStatusTypePROCESSING } else { - uploadStatus = models.AVStatusType(tags["av-status"]) + uploadStatus = AVStatusType(tags["av-status"]) } resProcess := []byte("id: 0\nevent: message\ndata: " + string(uploadStatus) + "\n\n") @@ -298,7 +311,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer f.Flush() } - if uploadStatus == models.AVStatusTypeCLEAN || uploadStatus == models.AVStatusTypeINFECTED { + if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { return } @@ -315,24 +328,34 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer o.appCtx.Logger().Error(err.Error()) } + // Cleanup + go func() { + <-o.params.HTTPRequest.Context().Done() + _ = o.receiver.CloseoutQueue(o.appCtx, queueUrl) + }() + id_counter := 0 // Run for 120 seconds, 20 second long polling 6 times for range 6 { o.appCtx.Logger().Info("Receiving...") messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) - if errs != nil { + if errs != nil && errs != context.Canceled { o.appCtx.Logger().Error(errs.Error()) } + if errs == context.Canceled { + break + } + if len(messages) != 0 { errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { tags, err := o.storer.Tags(uploaded.Upload.StorageKey) if err != nil || len(tags) == 0 { - uploadStatus = models.AVStatusTypePROCESSING + uploadStatus = AVStatusTypePROCESSING } else { - uploadStatus = models.AVStatusType(tags["av-status"]) + uploadStatus = AVStatusType(tags["av-status"]) } resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") diff --git a/pkg/models/upload.go b/pkg/models/upload.go index 0703dff29ca..d6afc2d0d4a 100644 --- a/pkg/models/upload.go +++ b/pkg/models/upload.go @@ -25,32 +25,19 @@ const ( UploadTypeOFFICE UploadType = "OFFICE" ) -// AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected -type AVStatusType string - -const ( - // AVStatusTypePROCESSING string PROCESSING - AVStatusTypePROCESSING AVStatusType = "PROCESSING" - // AVStatusTypeCLEAN string CLEAN - AVStatusTypeCLEAN AVStatusType = "CLEAN" - // AVStatusTypeINFECTED string INFECTED - AVStatusTypeINFECTED AVStatusType = "INFECTED" -) - // An Upload represents an uploaded file, such as an image or PDF. type Upload struct { - ID uuid.UUID `db:"id"` - Filename string `db:"filename"` - Bytes int64 `db:"bytes"` - Rotation *int64 `db:"rotation"` - ContentType string `db:"content_type"` - Checksum string `db:"checksum"` - StorageKey string `db:"storage_key"` - AVStatus *AVStatusType `db:"av_status"` - UploadType UploadType `db:"upload_type"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - DeletedAt *time.Time `db:"deleted_at"` + ID uuid.UUID `db:"id"` + Filename string `db:"filename"` + Bytes int64 `db:"bytes"` + Rotation *int64 `db:"rotation"` + ContentType string `db:"content_type"` + Checksum string `db:"checksum"` + StorageKey string `db:"storage_key"` + UploadType UploadType `db:"upload_type"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + DeletedAt *time.Time `db:"deleted_at"` } // TableName overrides the table name used by Pop. diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 469fff74aab..4c737549c35 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -30,23 +31,26 @@ type NotificationQueueParams struct { type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) + CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error } // NotificationSendingContext provides context to a notification sender type NotificationReceiverContext struct { - snsService *sns.Client - sqsService *sqs.Client - awsRegion string - awsAccountId string + snsService *sns.Client + sqsService *sqs.Client + awsRegion string + awsAccountId string + receiverContextMap map[string]context.CancelFunc } // NewNotificationSender returns a new NotificationSendingContext -func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { +func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string, receiverContextMap map[string]context.CancelFunc) NotificationReceiverContext { return NotificationReceiverContext{ - snsService: snsService, - sqsService: sqsService, - awsRegion: awsRegion, - awsAccountId: awsAccountId, + snsService: snsService, + sqsService: sqsService, + awsRegion: awsRegion, + awsAccountId: awsAccountId, + receiverContextMap: receiverContextMap, } } @@ -120,15 +124,37 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte // SendNotification sends a one or more notifications for all supported mediums func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { - result, err := n.sqsService.ReceiveMessage(context.Background(), &sqs.ReceiveMessageInput{ + recCtx, cancelRecCtx := context.WithTimeout(context.Background(), 120*time.Second) + defer cancelRecCtx() + n.receiverContextMap[queueUrl] = cancelRecCtx + + result, err := n.sqsService.ReceiveMessage(recCtx, &sqs.ReceiveMessageInput{ QueueUrl: &queueUrl, MaxNumberOfMessages: 1, WaitTimeSeconds: 20, }) - if err != nil { - appCtx.Logger().Fatal("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + if err != nil && recCtx.Err() != context.Canceled { + appCtx.Logger().Info("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + return nil, err } - return result.Messages, err + + if recCtx.Err() == context.Canceled { + return nil, recCtx.Err() + } + + return result.Messages, recCtx.Err() +} + +// map of queueUrl to context + +// CloseoutQueue stops receiving messages and cleans up the queue and its subscriptions +func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + n.receiverContextMap[queueUrl]() + // n.snsService.Unsubscribe(...) + // n.sqsService.DeleteQueue(...) + appCtx.Logger().Error("CLOSING OUT CONTEXT") + n.receiverContextMap[queueUrl] = nil + return nil } // InitEmail initializes the email backend @@ -183,7 +209,9 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil + receiverContextMap := make(map[string]context.CancelFunc) + + return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId, receiverContextMap), nil } func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { From dbc9a18096db4a735aa52c428d4a2c1fa8c438c9 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Thu, 2 Jan 2025 21:19:41 +0000 Subject: [PATCH 15/24] B-21669 - unique queues, working on destuction of queue and subscription. --- pkg/notifications/notification_receiver.go | 69 ++++++++++++++-------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 4c737549c35..a5dee3840b6 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/gofrs/uuid" "github.com/spf13/viper" "go.uber.org/zap" @@ -34,29 +35,33 @@ type NotificationReceiver interface { CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error } -// NotificationSendingContext provides context to a notification sender +// NotificationSendingContext provides context to a notification sender. Maps use queueUrl type NotificationReceiverContext struct { - snsService *sns.Client - sqsService *sqs.Client - awsRegion string - awsAccountId string - receiverContextMap map[string]context.CancelFunc + snsService *sns.Client + sqsService *sqs.Client + awsRegion string + awsAccountId string + queueSubscriptionMap map[string]string + receiverCancelMap map[string]context.CancelFunc } // NewNotificationSender returns a new NotificationSendingContext -func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string, receiverContextMap map[string]context.CancelFunc) NotificationReceiverContext { +func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ - snsService: snsService, - sqsService: sqsService, - awsRegion: awsRegion, - awsAccountId: awsAccountId, - receiverContextMap: receiverContextMap, + snsService: snsService, + sqsService: sqsService, + awsRegion: awsRegion, + awsAccountId: awsAccountId, + queueSubscriptionMap: make(map[string]string), + receiverCancelMap: make(map[string]context.CancelFunc), } } func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicName string, params NotificationQueueParams) (string, error) { - queueName := fmt.Sprintf("%s_%s", params.Action, params.ObjectId) + queueUUID := uuid.Must(uuid.NewV4()) + + queueName := fmt.Sprintf("%s_%s", params.Action, queueUUID) queueArn := n.constructArn("sqs", queueName) topicArn := n.constructArn("sns", topicName) @@ -114,11 +119,13 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte "FilterPolicyScope": "MessageBody", }, } - _, err = n.snsService.Subscribe(context.Background(), subscribeInput) + subscribeOutput, err := n.snsService.Subscribe(context.Background(), subscribeInput) if err != nil { log.Fatalf("Failed to create subscription, %v", err) } + n.queueSubscriptionMap[*result.QueueUrl] = *subscribeOutput.SubscriptionArn + return *result.QueueUrl, err } @@ -126,7 +133,7 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { recCtx, cancelRecCtx := context.WithTimeout(context.Background(), 120*time.Second) defer cancelRecCtx() - n.receiverContextMap[queueUrl] = cancelRecCtx + n.receiverCancelMap[queueUrl] = cancelRecCtx result, err := n.sqsService.ReceiveMessage(recCtx, &sqs.ReceiveMessageInput{ QueueUrl: &queueUrl, @@ -149,11 +156,29 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex // CloseoutQueue stops receiving messages and cleans up the queue and its subscriptions func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { - n.receiverContextMap[queueUrl]() - // n.snsService.Unsubscribe(...) - // n.sqsService.DeleteQueue(...) - appCtx.Logger().Error("CLOSING OUT CONTEXT") - n.receiverContextMap[queueUrl] = nil + appCtx.Logger().Info("CLOSING OUT QUEUE CONTEXT") + + if cancelFunc, exists := n.receiverCancelMap[queueUrl]; exists { + cancelFunc() + delete(n.receiverCancelMap, queueUrl) + } + if subscriptionArn, exists := n.queueSubscriptionMap[queueUrl]; exists { + _, err := n.snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ + SubscriptionArn: &subscriptionArn, + }) + if err != nil { + return err + } + delete(n.queueSubscriptionMap, queueUrl) + } + + _, err := n.sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ + QueueUrl: &queueUrl, + }) + if err != nil { + return err + } + return nil } @@ -209,9 +234,7 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - receiverContextMap := make(map[string]context.CancelFunc) - - return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId, receiverContextMap), nil + return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil } func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { From 762ff6f5b18361a6ff196d54f74ac14ad76a8c9a Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 3 Jan 2025 20:12:12 +0000 Subject: [PATCH 16/24] B-21669 - environment vars and receiver cleanup. --- .envrc | 12 ++- pkg/cli/receiver.go | 53 ++++++++++++ pkg/cli/receiver_test.go | 6 ++ pkg/handlers/internalapi/uploads.go | 32 +++++-- pkg/notifications/notification_receiver.go | 97 ++++++++-------------- 5 files changed, 130 insertions(+), 70 deletions(-) create mode 100644 pkg/cli/receiver.go create mode 100644 pkg/cli/receiver_test.go diff --git a/.envrc b/.envrc index 3891c5b8d85..5a7f1a3682d 100644 --- a/.envrc +++ b/.envrc @@ -232,16 +232,19 @@ export TZ="UTC" # To use S3/SES for local builds, you'll need to uncomment the following. # Do not commit the change: # -# export STORAGE_BACKEND=s3 + export STORAGE_BACKEND=s3 # export EMAIL_BACKEND=ses + export RECEIVER_BACKEND="sns&sqs" # # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally +# Instructions for using SNS&SQS backend here: ... # # The default and equivalent to not being set is: # # export STORAGE_BACKEND=local # export EMAIL_BACKEND=local +# export RECEIVER_BACKEND=local # # Setting region and profile conditionally while we migrate from com to govcloud. if [ "$STORAGE_BACKEND" == "s3" ]; then @@ -255,6 +258,11 @@ export AWS_S3_KEY_NAMESPACE=$USER export AWS_SES_DOMAIN="devlocal.dp3.us" export AWS_SES_REGION="us-gov-west-1" +if [ "$RECEIVER_BACKEND" == "sns&sqs" ]; then + export AWS_SNS_OBJECT_TAGS_ADDED_TOPIC="app_s3_tag_events" + export AWS_SNS_REGION="us-gov-west-1" +fi + # To use s3 links aws-bucketname/xx/user/ for local builds, # you'll need to add the following to your .envrc.local: # @@ -441,4 +449,4 @@ then fi # Check that all required environment variables are set -check_required_variables \ No newline at end of file +check_required_variables diff --git a/pkg/cli/receiver.go b/pkg/cli/receiver.go new file mode 100644 index 00000000000..91f6f30f872 --- /dev/null +++ b/pkg/cli/receiver.go @@ -0,0 +1,53 @@ +package cli + +import ( + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + // ReceiverBackend is the Receiver Backend Flag + ReceiverBackendFlag string = "receiver-backend" + // AWSSNSObjectTagsAddedTopic is the AWS SNS Object Tags Added Topic Flag + AWSSNSObjectTagsAddedTopicFlag string = "aws-sns-object-tags-added-topic" + // AWSS3RegionFlag is the AWS SNS Region Flag + AWSSNSRegionFlag string = "aws-sns-region" + // AWSSNSAccountId is the application's AWS account id + AWSSNSAccountId string = "aws-account-id" +) + +// InitReceiverFlags initializes Storage command line flags +func InitReceiverFlags(flag *pflag.FlagSet) { + flag.String(ReceiverBackendFlag, "local", "Receiver backend to use, either local or sns&sqs.") + flag.String(AWSSNSObjectTagsAddedTopicFlag, "", "SNS Topic for receiving event messages") + flag.String(AWSSNSRegionFlag, "", "AWS region used for SNS and SQS") + flag.String(AWSSNSAccountId, "", "AWS account Id") +} + +// CheckReceiver validates Storage command line flags +func CheckReceiver(v *viper.Viper) error { + + receiverBackend := v.GetString(ReceiverBackendFlag) + if !stringSliceContains([]string{"local", "sns&sqs"}, receiverBackend) { + return fmt.Errorf("invalid receiver-backend %s, expecting local or sns&sqs", receiverBackend) + } + + if receiverBackend == "sns&sqs" { + r := v.GetString(AWSSNSRegionFlag) + if r == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSRegionFlag, r) + } + topic := v.GetString(AWSSNSObjectTagsAddedTopicFlag) + if topic == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSObjectTagsAddedTopicFlag, topic) + } + accountId := v.GetString(AWSSNSAccountId) + if topic == "" { + return fmt.Errorf("invalid value for %s: %s", AWSSNSAccountId, accountId) + } + } + + return nil +} diff --git a/pkg/cli/receiver_test.go b/pkg/cli/receiver_test.go new file mode 100644 index 00000000000..7095a672f5f --- /dev/null +++ b/pkg/cli/receiver_test.go @@ -0,0 +1,6 @@ +package cli + +func (suite *cliTestSuite) TestConfigReceiver() { + suite.Setup(InitReceiverFlags, []string{}) + suite.NoError(CheckReceiver(suite.viper)) +} diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index c8b8a4b13ab..a32a21f5736 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -14,9 +14,11 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/gobuffalo/validate/v3" "github.com/gofrs/uuid" + "github.com/spf13/viper" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" + "github.com/transcom/mymove/pkg/cli" ppmop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/ppm" uploadop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/uploads" "github.com/transcom/mymove/pkg/handlers" @@ -291,7 +293,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) if err != nil { - o.appCtx.Logger().Error(err.Error()) + panic(err) } tags, err := o.storer.Tags(uploaded.Upload.StorageKey) @@ -316,14 +318,32 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } // Start waiting for tag updates + v := viper.New() + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + v.AutomaticEnv() + topicName := v.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) + if topicName == "" { + o.appCtx.Logger().Error("aws_sns_object_tags_added_topic key not available.") + return + } + + filterPolicy := fmt.Sprintf(`{ + "detail": { + "object": { + "key": [ + {"suffix": "%s"} + ] + } + } + }`, uploadId) - topicName := "app_s3_tag_events" notificationParams := notifications.NotificationQueueParams{ - Action: "ObjectTagsAdded", - ObjectId: uploadId, + SubscriptionTopicName: topicName, + NamePrefix: "ObjectTagsAdded", + FilterPolicy: filterPolicy, } - queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, topicName, notificationParams) + queueUrl, err := o.receiver.CreateQueueWithSubscription(o.appCtx, notificationParams) if err != nil { o.appCtx.Logger().Error(err.Error()) } @@ -335,7 +355,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer }() id_counter := 0 - // Run for 120 seconds, 20 second long polling 6 times + // Run for 120 seconds, 20 second long polling for receiver, 6 times for range 6 { o.appCtx.Logger().Info("Receiving...") messages, errs := o.receiver.ReceiveMessages(o.appCtx, queueUrl) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index a5dee3840b6..67d6fc7c8ed 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -19,23 +18,23 @@ import ( "github.com/transcom/mymove/pkg/cli" ) -// Notification is an interface for creating emails +// NotificationQueueParams stores the params for queue creation type NotificationQueueParams struct { - // TODO: change to enum - Action string - ObjectId string + SubscriptionTopicName string + NamePrefix string + FilterPolicy string } -// NotificationSender is an interface for sending notifications +// NotificationReceiver is an interface for receiving notifications // -//go:generate mockery --name NotificationSender +//go:generate mockery --name NotificationReceiver type NotificationReceiver interface { - CreateQueueWithSubscription(appCtx appcontext.AppContext, topicArn string, params NotificationQueueParams) (string, error) + CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error } -// NotificationSendingContext provides context to a notification sender. Maps use queueUrl +// NotificationReceiverConext provides context to a notification Receiver. Maps use queueUrl for key type NotificationReceiverContext struct { snsService *sns.Client sqsService *sqs.Client @@ -45,7 +44,7 @@ type NotificationReceiverContext struct { receiverCancelMap map[string]context.CancelFunc } -// NewNotificationSender returns a new NotificationSendingContext +// NewNotificationReceiver returns a new NotificationReceiverContext func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ snsService: snsService, @@ -57,13 +56,14 @@ func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, aws } } -func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, topicName string, params NotificationQueueParams) (string, error) { +// CreateQueueWithSubscription first creates a new queue, then subscribes an AWS topic to it +func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) { queueUUID := uuid.Must(uuid.NewV4()) - queueName := fmt.Sprintf("%s_%s", params.Action, queueUUID) + queueName := fmt.Sprintf("%s_%s", params.NamePrefix, queueUUID) queueArn := n.constructArn("sqs", queueName) - topicArn := n.constructArn("sns", topicName) + topicArn := n.constructArn("sns", params.SubscriptionTopicName) // Create queue @@ -98,24 +98,12 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte log.Fatalf("Failed to create SQS queue, %v", err) } - // Create subscription - - filterPolicy := fmt.Sprintf(`{ - "detail": { - "object": { - "key": [ - {"suffix": "%s"} - ] - } - } - }`, params.ObjectId) - subscribeInput := &sns.SubscribeInput{ TopicArn: &topicArn, Protocol: aws.String("sqs"), Endpoint: &queueArn, Attributes: map[string]string{ - "FilterPolicy": filterPolicy, + "FilterPolicy": params.FilterPolicy, "FilterPolicyScope": "MessageBody", }, } @@ -129,9 +117,9 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte return *result.QueueUrl, err } -// SendNotification sends a one or more notifications for all supported mediums +// ReceiveMessages polls given queue continuously for messages for up to 20 seconds func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { - recCtx, cancelRecCtx := context.WithTimeout(context.Background(), 120*time.Second) + recCtx, cancelRecCtx := context.WithCancel(context.Background()) defer cancelRecCtx() n.receiverCancelMap[queueUrl] = cancelRecCtx @@ -162,6 +150,7 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, cancelFunc() delete(n.receiverCancelMap, queueUrl) } + if subscriptionArn, exists := n.queueSubscriptionMap[queueUrl]; exists { _, err := n.snsService.Unsubscribe(context.Background(), &sns.UnsubscribeInput{ SubscriptionArn: &subscriptionArn, @@ -184,26 +173,7 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, // InitEmail initializes the email backend func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { - // if v.GetString(cli.EmailBackendFlag) == "ses" { - // // Setup Amazon SES (email) service TODO: This might be able - // // to be combined with the AWS Session that we're using for S3 - // // down below. - - // awsSESRegion := v.GetString(cli.AWSSESRegionFlag) - // awsSESDomain := v.GetString(cli.AWSSESDomainFlag) - // sysAdminEmail := v.GetString(cli.SysAdminEmail) - // logger.Info("Using ses email backend", - // zap.String("region", awsSESRegion), - // zap.String("domain", awsSESDomain)) - // cfg, err := config.LoadDefaultConfig(context.Background(), - // config.WithRegion(awsSESRegion), - // ) - // if err != nil { - // logger.Fatal("error loading ses aws config", zap.Error(err)) - // } - // sesService := ses.NewFromConfig(cfg) - // input := &ses.GetAccountSendingEnabledInput{} // result, err := sesService.GetAccountSendingEnabled(context.Background(), input) // if err != nil || result == nil || !result.Enabled { // logger.Error("email sending not enabled", zap.Error(err)) @@ -216,25 +186,28 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err // logger.Info("Using local email backend", zap.String("domain", domain)) // return NewStubNotificationSender(domain), nil - // Setup Amazon SES (email) service TODO: This might be able - // to be combined with the AWS Session that we're using for S3 - // down below. + if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { + // Setup notification receiver service with SNS & SQS backend dependencies + awsSNSRegion := v.GetString(cli.AWSSNSRegionFlag) + awsAccountId := v.GetString(cli.AWSSNSAccountId) - // TODO: verify if we should change this param name to awsNotificationRegion - awsSESRegion := v.GetString(cli.AWSSESRegionFlag) - awsAccountId := v.GetString("aws-account-id") + logger.Info("Using aws sns&sqs receiver backend", zap.String("region", awsSNSRegion)) - cfg, err := config.LoadDefaultConfig(context.Background(), - config.WithRegion(awsSESRegion), - ) - if err != nil { - logger.Fatal("error loading ses aws config", zap.Error(err)) - } + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(awsSNSRegion), + ) + if err != nil { + logger.Fatal("error loading sns aws config", zap.Error(err)) + } - snsService := sns.NewFromConfig(cfg) - sqsService := sqs.NewFromConfig(cfg) + snsService := sns.NewFromConfig(cfg) + sqsService := sqs.NewFromConfig(cfg) + + return NewNotificationReceiver(snsService, sqsService, awsSNSRegion, awsAccountId), nil + } - return NewNotificationReceiver(snsService, sqsService, awsSESRegion, awsAccountId), nil + // TODO: add local notification receiver initializer here + return nil, nil } func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { From 08087ce82ea5bad5b80a211618a7fde0c68d9b3b Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Fri, 3 Jan 2025 23:45:16 +0000 Subject: [PATCH 17/24] B-21669 - stub notification receiver. --- pkg/handlers/internalapi/uploads.go | 2 +- pkg/notifications/notification_receiver.go | 3 +- .../notification_receiver_stub.go | 47 +++++++++++++++++++ ...on_stub.go => notification_sender_stub.go} | 0 ...on_test.go => notification_sender_test.go} | 0 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 pkg/notifications/notification_receiver_stub.go rename pkg/notifications/{notification_stub.go => notification_sender_stub.go} (100%) rename pkg/notifications/{notification_test.go => notification_sender_test.go} (100%) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index a32a21f5736..38688c8219d 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -286,11 +286,11 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer uploadUUID, err := uuid.FromString(uploadId) if err != nil { + uploadop.NewGetUploadStatusInternalServerError().WriteResponse(rw, producer) panic(err) } // Check current tag before event-driven wait for anti-virus - uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) if err != nil { panic(err) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 67d6fc7c8ed..e55a15183eb 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -206,8 +206,7 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err return NewNotificationReceiver(snsService, sqsService, awsSNSRegion, awsAccountId), nil } - // TODO: add local notification receiver initializer here - return nil, nil + return NewStubNotificationReceiver(), nil } func (n NotificationReceiverContext) constructArn(awsService string, endpointName string) string { diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go new file mode 100644 index 00000000000..25c3bd901b7 --- /dev/null +++ b/pkg/notifications/notification_receiver_stub.go @@ -0,0 +1,47 @@ +package notifications + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "go.uber.org/zap" + + "github.com/transcom/mymove/pkg/appcontext" +) + +// StubNotificationReceiver mocks an SNS & SQS client for local usage +type StubNotificationReceiver NotificationReceiverContext + +// NewStubNotificationReceiver returns a new StubNotificationReceiver +func NewStubNotificationReceiver() StubNotificationReceiver { + return StubNotificationReceiver{ + snsService: nil, + sqsService: nil, + awsRegion: "", + awsAccountId: "", + queueSubscriptionMap: make(map[string]string), + receiverCancelMap: make(map[string]context.CancelFunc), + } +} + +func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) { + return "fakeQueueName", nil +} + +func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { + // TODO: sleep func here & swap types for return to avoid aws type + messageId := "fakeMessageId" + body := queueUrl + ":fakeMessageBody" + mockMessages := make([]types.Message, 1) + mockMessages = append(mockMessages, types.Message{ + MessageId: &messageId, + Body: &body, + }) + appCtx.Logger().Debug("Receiving a fake message for queue: %v", zap.String("queueUrl", queueUrl)) + return mockMessages, nil +} + +func (n StubNotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + appCtx.Logger().Debug("Closing out the fake queue.") + return nil +} diff --git a/pkg/notifications/notification_stub.go b/pkg/notifications/notification_sender_stub.go similarity index 100% rename from pkg/notifications/notification_stub.go rename to pkg/notifications/notification_sender_stub.go diff --git a/pkg/notifications/notification_test.go b/pkg/notifications/notification_sender_test.go similarity index 100% rename from pkg/notifications/notification_test.go rename to pkg/notifications/notification_sender_test.go From 65b0b3aee4d8a1f3eb44f1c0e9770a5cd69cd7f5 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Mon, 6 Jan 2025 17:06:57 +0000 Subject: [PATCH 18/24] B-21669 - faked notification receiver working for local. --- .envrc | 4 ++-- pkg/handlers/internalapi/uploads.go | 9 ++------- pkg/notifications/notification_receiver.go | 16 ++++++++++++++++ pkg/notifications/notification_receiver_stub.go | 4 ++++ pkg/storage/filesystem.go | 2 ++ pkg/storage/memory.go | 2 ++ 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.envrc b/.envrc index 5a7f1a3682d..54c1ce004ba 100644 --- a/.envrc +++ b/.envrc @@ -232,9 +232,9 @@ export TZ="UTC" # To use S3/SES for local builds, you'll need to uncomment the following. # Do not commit the change: # - export STORAGE_BACKEND=s3 +# export STORAGE_BACKEND=s3 # export EMAIL_BACKEND=ses - export RECEIVER_BACKEND="sns&sqs" +# export RECEIVER_BACKEND="sns&sqs" # # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 38688c8219d..50f40fa6920 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -14,11 +14,9 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/gobuffalo/validate/v3" "github.com/gofrs/uuid" - "github.com/spf13/viper" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" - "github.com/transcom/mymove/pkg/cli" ppmop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/ppm" uploadop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/uploads" "github.com/transcom/mymove/pkg/handlers" @@ -318,11 +316,8 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } // Start waiting for tag updates - v := viper.New() - v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - v.AutomaticEnv() - topicName := v.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) - if topicName == "" { + topicName, err := o.receiver.GetDefaultTopic() + if err != nil { o.appCtx.Logger().Error("aws_sns_object_tags_added_topic key not available.") return } diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index e55a15183eb..576fdc71bcc 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -2,8 +2,10 @@ package notifications import ( "context" + "errors" "fmt" "log" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -32,6 +34,7 @@ type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error + GetDefaultTopic() (string, error) } // NotificationReceiverConext provides context to a notification Receiver. Maps use queueUrl for key @@ -171,6 +174,19 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, return nil } +func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { + // Start waiting for tag updates + v := viper.New() + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + v.AutomaticEnv() + topicName := v.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) + receiverBackend := v.GetString(cli.ReceiverBackendFlag) + if topicName == "" && receiverBackend == "sns&sqs" { + return "", errors.New("aws_sns_object_tags_added_topic key not available.") + } + return topicName, nil +} + // InitEmail initializes the email backend func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index 25c3bd901b7..8ea3ccd12cb 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -45,3 +45,7 @@ func (n StubNotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, qu appCtx.Logger().Debug("Closing out the fake queue.") return nil } + +func (n StubNotificationReceiver) GetDefaultTopic() (string, error) { + return "", nil +} diff --git a/pkg/storage/filesystem.go b/pkg/storage/filesystem.go index 259fd4ee8ab..702a7d372a4 100644 --- a/pkg/storage/filesystem.go +++ b/pkg/storage/filesystem.go @@ -116,6 +116,8 @@ func (fs *Filesystem) Fetch(key string) (io.ReadCloser, error) { // Tags returns the tags for a specified key func (fs *Filesystem) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) + // Assume anti-virus complete + tags["av_status"] = "CLEAN" return tags, nil } diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index 2f06ed6b96e..dcdd27ca200 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -116,6 +116,8 @@ func (fs *Memory) Fetch(key string) (io.ReadCloser, error) { // Tags returns the tags for a specified key func (fs *Memory) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) + // Assume anti-virus complete + tags["av_status"] = "CLEAN" return tags, nil } From 74545d0d48d5d13db0af301d8e6c5342a03d4f5d Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Mon, 6 Jan 2025 23:00:28 +0000 Subject: [PATCH 19/24] B-21669 - working on unit tests. --- pkg/handlers/apitests.go | 12 +- pkg/handlers/internalapi/uploads.go | 10 +- pkg/handlers/internalapi/uploads_test.go | 5 +- pkg/handlers/routing/base_routing_suite.go | 1 + .../routing/internalapi_test/uploads_test.go | 41 ++++++ .../mocks/NotificationReceiver.go | 133 ++++++++++++++++++ pkg/notifications/notification_receiver.go | 52 +++---- .../notification_receiver_stub.go | 22 ++- .../notification_receiver_test.go | 46 ++++++ pkg/storage/filesystem.go | 2 +- pkg/storage/memory.go | 2 +- pkg/storage/test/s3.go | 3 +- 12 files changed, 279 insertions(+), 50 deletions(-) create mode 100644 pkg/handlers/routing/internalapi_test/uploads_test.go create mode 100644 pkg/notifications/mocks/NotificationReceiver.go create mode 100644 pkg/notifications/notification_receiver_test.go diff --git a/pkg/handlers/apitests.go b/pkg/handlers/apitests.go index a84a6627f2c..a540d37e1f3 100644 --- a/pkg/handlers/apitests.go +++ b/pkg/handlers/apitests.go @@ -9,6 +9,7 @@ import ( "path" "path/filepath" "runtime/debug" + "strings" "time" "github.com/go-openapi/runtime" @@ -148,6 +149,11 @@ func (suite *BaseHandlerTestSuite) TestNotificationSender() notifications.Notifi return suite.notificationSender } +// TestNotificationReceiver returns the notification sender to use in the suite +func (suite *BaseHandlerTestSuite) TestNotificationReceiver() notifications.NotificationReceiver { + return notifications.NewStubNotificationReceiver() +} + // HasWebhookNotification checks that there's a record on the WebhookNotifications table for the object and trace IDs func (suite *BaseHandlerTestSuite) HasWebhookNotification(objectID uuid.UUID, traceID uuid.UUID) { notification := &models.WebhookNotification{} @@ -277,8 +283,12 @@ func (suite *BaseHandlerTestSuite) Fixture(name string) *runtime.File { if err != nil { suite.T().Error(err) } + cdRouting := "" + if strings.Contains(cwd, "routing") { + cdRouting = ".." + } - fixturePath := path.Join(cwd, "..", "..", fixtureDir, name) + fixturePath := path.Join(cwd, "..", "..", cdRouting, fixtureDir, name) file, err := os.Open(filepath.Clean(fixturePath)) if err != nil { diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 50f40fa6920..7901cae3d19 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -276,6 +276,10 @@ const ( AVStatusTypeINFECTED AVStatusType = "INFECTED" ) +func constructEventStreamMessage(id int, data string) []byte { + return []byte(fmt.Sprintf("id: %s\nevent: message\ndata: %s\n\n", strconv.Itoa(id), data)) +} + func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { // TODO: add check for permissions to view upload @@ -301,8 +305,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } else { uploadStatus = AVStatusType(tags["av-status"]) } - - resProcess := []byte("id: 0\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + resProcess := constructEventStreamMessage(0, string(uploadStatus)) if produceErr := producer.Produce(rw, resProcess); produceErr != nil { panic(produceErr) } @@ -373,11 +376,10 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer uploadStatus = AVStatusType(tags["av-status"]) } - resProcess := []byte("id: " + strconv.Itoa(id_counter) + "\nevent: message\ndata: " + string(uploadStatus) + "\n\n") + resProcess := constructEventStreamMessage(id_counter, string(uploadStatus)) if produceErr := producer.Produce(rw, resProcess); produceErr != nil { panic(produceErr) // let the recovery middleware deal with this } - return nil }) diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index ebc6eb0373c..bc07c4a5619 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -24,6 +24,7 @@ import ( uploadop "github.com/transcom/mymove/pkg/gen/internalapi/internaloperations/uploads" "github.com/transcom/mymove/pkg/handlers" "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/notifications" paperworkgenerator "github.com/transcom/mymove/pkg/paperwork" "github.com/transcom/mymove/pkg/services/upload" weightticketparser "github.com/transcom/mymove/pkg/services/weight_ticket_parser" @@ -109,6 +110,7 @@ func createPPMExpensePrereqs(suite *HandlerSuite, fixtureFile string) (models.Do func makeRequest(suite *HandlerSuite, params uploadop.CreateUploadParams, serviceMember models.ServiceMember, fakeS3 *storageTest.FakeS3Storage) middleware.Responder { req := &http.Request{} + req = suite.AuthenticateRequest(req, serviceMember) params.HTTPRequest = req @@ -450,6 +452,7 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { // TODO: functioning test func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { fakeS3 := storageTest.NewFakeS3Storage(true) + localReceiver := notifications.StubNotificationReceiver{} move := factory.BuildMove(suite.DB(), nil, nil) uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ @@ -478,11 +481,11 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { handlerConfig := suite.HandlerConfig() handlerConfig.SetFileStorer(fakeS3) + handlerConfig.SetNotificationReceiver(localReceiver) uploadInformationFetcher := upload.NewUploadInformationFetcher() handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} response := handler.Handle(params) - _, ok := response.(*CustomNewUploadStatusOK) suite.True(ok) diff --git a/pkg/handlers/routing/base_routing_suite.go b/pkg/handlers/routing/base_routing_suite.go index 23e538792b7..77049e33664 100644 --- a/pkg/handlers/routing/base_routing_suite.go +++ b/pkg/handlers/routing/base_routing_suite.go @@ -85,6 +85,7 @@ func (suite *BaseRoutingSuite) RoutingConfig() *Config { handlerConfig := suite.BaseHandlerTestSuite.HandlerConfig() handlerConfig.SetAppNames(handlers.ApplicationTestServername()) handlerConfig.SetNotificationSender(suite.TestNotificationSender()) + handlerConfig.SetNotificationReceiver(suite.TestNotificationReceiver()) // Need this for any requests that will either retrieve or save files or their info. fakeS3 := storageTest.NewFakeS3Storage(true) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go new file mode 100644 index 00000000000..481769056c0 --- /dev/null +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -0,0 +1,41 @@ +package internalapi_test + +import ( + "net/http" + "net/http/httptest" + + "github.com/transcom/mymove/pkg/factory" + "github.com/transcom/mymove/pkg/models" + "github.com/transcom/mymove/pkg/uploader" +) + +func (suite *InternalAPISuite) TestUploads() { + suite.Run("Received message for upload", func() { + move := factory.BuildMove(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ + { + Model: move.Orders.UploadedOrders, + LinkOnly: true, + }, + { + Model: models.Upload{ + Filename: "FileName", + Bytes: int64(15), + ContentType: uploader.FileTypePDF, + }, + }, + }, nil) + file := suite.Fixture("test.pdf") + _, err := suite.HandlerConfig().FileStorer().Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) + + req := suite.NewAuthenticatedMilRequest("GET", "/internal/uploads/"+uploadUser1.ID.String()+"/status", nil, move.Orders.ServiceMember) + rr := httptest.NewRecorder() + + suite.SetupSiteHandler().ServeHTTP(rr, req) + + suite.Equal(http.StatusOK, rr.Code) + + // suite.Equal("text/eventstream", rr.Header().Get("content-type")) + }) +} diff --git a/pkg/notifications/mocks/NotificationReceiver.go b/pkg/notifications/mocks/NotificationReceiver.go new file mode 100644 index 00000000000..df8329e5f60 --- /dev/null +++ b/pkg/notifications/mocks/NotificationReceiver.go @@ -0,0 +1,133 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + appcontext "github.com/transcom/mymove/pkg/appcontext" + + notifications "github.com/transcom/mymove/pkg/notifications" +) + +// NotificationReceiver is an autogenerated mock type for the NotificationReceiver type +type NotificationReceiver struct { + mock.Mock +} + +// CloseoutQueue provides a mock function with given fields: appCtx, queueUrl +func (_m *NotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { + ret := _m.Called(appCtx, queueUrl) + + if len(ret) == 0 { + panic("no return value specified for CloseoutQueue") + } + + var r0 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) error); ok { + r0 = rf(appCtx, queueUrl) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// CreateQueueWithSubscription provides a mock function with given fields: appCtx, params +func (_m *NotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params notifications.NotificationQueueParams) (string, error) { + ret := _m.Called(appCtx, params) + + if len(ret) == 0 { + panic("no return value specified for CreateQueueWithSubscription") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) (string, error)); ok { + return rf(appCtx, params) + } + if rf, ok := ret.Get(0).(func(appcontext.AppContext, notifications.NotificationQueueParams) string); ok { + r0 = rf(appCtx, params) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(appcontext.AppContext, notifications.NotificationQueueParams) error); ok { + r1 = rf(appCtx, params) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetDefaultTopic provides a mock function with given fields: +func (_m *NotificationReceiver) GetDefaultTopic() (string, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetDefaultTopic") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func() (string, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ReceiveMessages provides a mock function with given fields: appCtx, queueUrl +func (_m *NotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]notifications.ReceivedMessage, error) { + ret := _m.Called(appCtx, queueUrl) + + if len(ret) == 0 { + panic("no return value specified for ReceiveMessages") + } + + var r0 []notifications.ReceivedMessage + var r1 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) ([]notifications.ReceivedMessage, error)); ok { + return rf(appCtx, queueUrl) + } + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string) []notifications.ReceivedMessage); ok { + r0 = rf(appCtx, queueUrl) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]notifications.ReceivedMessage) + } + } + + if rf, ok := ret.Get(1).(func(appcontext.AppContext, string) error); ok { + r1 = rf(appCtx, queueUrl) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewNotificationReceiver creates a new instance of NotificationReceiver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNotificationReceiver(t interface { + mock.TestingT + Cleanup(func()) +}) *NotificationReceiver { + mock := &NotificationReceiver{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 576fdc71bcc..cee3b300ac2 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -11,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" - "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/gofrs/uuid" "github.com/spf13/viper" "go.uber.org/zap" @@ -32,7 +31,7 @@ type NotificationQueueParams struct { //go:generate mockery --name NotificationReceiver type NotificationReceiver interface { CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) - ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) + ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error GetDefaultTopic() (string, error) } @@ -47,6 +46,12 @@ type NotificationReceiverContext struct { receiverCancelMap map[string]context.CancelFunc } +// ReceivedMessage standardizes the format of the received message +type ReceivedMessage struct { + MessageId string + Body *string +} + // NewNotificationReceiver returns a new NotificationReceiverContext func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ @@ -68,8 +73,6 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte queueArn := n.constructArn("sqs", queueName) topicArn := n.constructArn("sns", params.SubscriptionTopicName) - // Create queue - accessPolicy := fmt.Sprintf(`{ "Version": "2012-10-17", "Statement": [{ @@ -121,7 +124,7 @@ func (n NotificationReceiverContext) CreateQueueWithSubscription(appCtx appconte } // ReceiveMessages polls given queue continuously for messages for up to 20 seconds -func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { +func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { recCtx, cancelRecCtx := context.WithCancel(context.Background()) defer cancelRecCtx() n.receiverCancelMap[queueUrl] = cancelRecCtx @@ -132,7 +135,7 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex WaitTimeSeconds: 20, }) if err != nil && recCtx.Err() != context.Canceled { - appCtx.Logger().Info("Couldn't get messages from queue. Here's why: %v\n", zap.Error(err)) + appCtx.Logger().Info("Couldn't get messages from queue. Error: %v\n", zap.Error(err)) return nil, err } @@ -140,14 +143,20 @@ func (n NotificationReceiverContext) ReceiveMessages(appCtx appcontext.AppContex return nil, recCtx.Err() } - return result.Messages, recCtx.Err() -} + receivedMessages := make([]ReceivedMessage, len(result.Messages)) + for index, value := range result.Messages { + receivedMessages[index] = ReceivedMessage{ + MessageId: *value.MessageId, + Body: value.Body, + } + } -// map of queueUrl to context + return receivedMessages, recCtx.Err() +} // CloseoutQueue stops receiving messages and cleans up the queue and its subscriptions func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { - appCtx.Logger().Info("CLOSING OUT QUEUE CONTEXT") + appCtx.Logger().Info("Closing out queue: %v", zap.String("queueUrl", queueUrl)) if cancelFunc, exists := n.receiverCancelMap[queueUrl]; exists { cancelFunc() @@ -167,41 +176,26 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, _, err := n.sqsService.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ QueueUrl: &queueUrl, }) - if err != nil { - return err - } - return nil + return err } +// GetDefaultTopic returns the topic value set within the environment func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { - // Start waiting for tag updates v := viper.New() v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) v.AutomaticEnv() topicName := v.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) receiverBackend := v.GetString(cli.ReceiverBackendFlag) if topicName == "" && receiverBackend == "sns&sqs" { - return "", errors.New("aws_sns_object_tags_added_topic key not available.") + return "", errors.New("aws_sns_object_tags_added_topic key not available") } return topicName, nil } -// InitEmail initializes the email backend +// InitReceiver initializes the receiver backend func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { - // result, err := sesService.GetAccountSendingEnabled(context.Background(), input) - // if err != nil || result == nil || !result.Enabled { - // logger.Error("email sending not enabled", zap.Error(err)) - // return NewNotificationSender(nil, awsSESDomain, sysAdminEmail), err - // } - // return NewNotificationSender(sesService, awsSESDomain, sysAdminEmail), nil - // } - - // domain := "milmovelocal" - // logger.Info("Using local email backend", zap.String("domain", domain)) - // return NewStubNotificationSender(domain), nil - if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { // Setup notification receiver service with SNS & SQS backend dependencies awsSNSRegion := v.GetString(cli.AWSSNSRegionFlag) diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index 8ea3ccd12cb..5b44471f608 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -3,7 +3,6 @@ package notifications import ( "context" - "github.com/aws/aws-sdk-go-v2/service/sqs/types" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" @@ -25,24 +24,23 @@ func NewStubNotificationReceiver() StubNotificationReceiver { } func (n StubNotificationReceiver) CreateQueueWithSubscription(appCtx appcontext.AppContext, params NotificationQueueParams) (string, error) { - return "fakeQueueName", nil + return "stubQueueName", nil } -func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]types.Message, error) { - // TODO: sleep func here & swap types for return to avoid aws type - messageId := "fakeMessageId" - body := queueUrl + ":fakeMessageBody" - mockMessages := make([]types.Message, 1) - mockMessages = append(mockMessages, types.Message{ - MessageId: &messageId, +func (n StubNotificationReceiver) ReceiveMessages(appCtx appcontext.AppContext, queueUrl string) ([]ReceivedMessage, error) { + messageId := "stubMessageId" + body := queueUrl + ":stubMessageBody" + mockMessages := make([]ReceivedMessage, 1) + mockMessages[0] = ReceivedMessage{ + MessageId: messageId, Body: &body, - }) - appCtx.Logger().Debug("Receiving a fake message for queue: %v", zap.String("queueUrl", queueUrl)) + } + appCtx.Logger().Debug("Receiving a stubbed message for queue: %v", zap.String("queueUrl", queueUrl)) return mockMessages, nil } func (n StubNotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, queueUrl string) error { - appCtx.Logger().Debug("Closing out the fake queue.") + appCtx.Logger().Debug("Closing out the stubbed queue.") return nil } diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go new file mode 100644 index 00000000000..5add62ddb14 --- /dev/null +++ b/pkg/notifications/notification_receiver_test.go @@ -0,0 +1,46 @@ +package notifications + +import ( + "fmt" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" + + "github.com/transcom/mymove/pkg/testingsuite" +) + +type notificationReceiverSuite struct { + *testingsuite.PopTestSuite +} + +func TestNotificationReceiverSuite(t *testing.T) { + + hs := ¬ificationReceiverSuite{ + PopTestSuite: testingsuite.NewPopTestSuite(testingsuite.CurrentPackage(), + testingsuite.WithPerTestTransaction()), + } + suite.Run(t, hs) + hs.PopTestSuite.TearDown() +} + +func (suite *notificationReceiverSuite) TestNotificationReceiverLocalStub() { + v := viper.New() + localReceiver, err := InitReceiver(v, suite.Logger()) + + suite.NoError(err) + + queueParams := NotificationQueueParams{ + NamePrefix: "testPrefix", + } + createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + suite.NoError(err) + suite.NotContains(createdQueueUrl, queueParams.NamePrefix) + suite.Equal(createdQueueUrl, "stubQueueName") + + receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + suite.Len(receivedMessages, 1) + suite.Equal(receivedMessages[0].MessageId, "stubMessageId") + suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) +} diff --git a/pkg/storage/filesystem.go b/pkg/storage/filesystem.go index 702a7d372a4..f6e43583420 100644 --- a/pkg/storage/filesystem.go +++ b/pkg/storage/filesystem.go @@ -117,7 +117,7 @@ func (fs *Filesystem) Fetch(key string) (io.ReadCloser, error) { func (fs *Filesystem) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) // Assume anti-virus complete - tags["av_status"] = "CLEAN" + tags["av-status"] = "CLEAN" return tags, nil } diff --git a/pkg/storage/memory.go b/pkg/storage/memory.go index dcdd27ca200..4e171e40e9d 100644 --- a/pkg/storage/memory.go +++ b/pkg/storage/memory.go @@ -117,7 +117,7 @@ func (fs *Memory) Fetch(key string) (io.ReadCloser, error) { func (fs *Memory) Tags(_ string) (map[string]string, error) { tags := make(map[string]string) // Assume anti-virus complete - tags["av_status"] = "CLEAN" + tags["av-status"] = "CLEAN" return tags, nil } diff --git a/pkg/storage/test/s3.go b/pkg/storage/test/s3.go index da076681dfe..5f738e7b088 100644 --- a/pkg/storage/test/s3.go +++ b/pkg/storage/test/s3.go @@ -90,7 +90,8 @@ func (fake *FakeS3Storage) TempFileSystem() *afero.Afero { // Tags returns the tags for a specified key func (fake *FakeS3Storage) Tags(_ string) (map[string]string, error) { tags := map[string]string{ - "tagName": "tagValue", + "tagName": "tagValue", + "av-status": "CLEAN", // Assume anti-virus run } return tags, nil } From c4355c5fa125b6e92ca582e02f6898e81b2b5c4e Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 7 Jan 2025 15:51:40 +0000 Subject: [PATCH 20/24] B-21669 - working test under routing with stub. --- pkg/handlers/routing/internalapi_test/uploads_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index 481769056c0..fade7975cc6 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -11,10 +11,10 @@ import ( func (suite *InternalAPISuite) TestUploads() { suite.Run("Received message for upload", func() { - move := factory.BuildMove(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ { - Model: move.Orders.UploadedOrders, + Model: orders.UploadedOrders, LinkOnly: true, }, { @@ -29,13 +29,13 @@ func (suite *InternalAPISuite) TestUploads() { _, err := suite.HandlerConfig().FileStorer().Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) suite.NoError(err) - req := suite.NewAuthenticatedMilRequest("GET", "/internal/uploads/"+uploadUser1.ID.String()+"/status", nil, move.Orders.ServiceMember) + req := suite.NewAuthenticatedMilRequest("GET", "/internal/uploads/"+uploadUser1.Upload.ID.String()+"/status", nil, orders.ServiceMember) rr := httptest.NewRecorder() suite.SetupSiteHandler().ServeHTTP(rr, req) suite.Equal(http.StatusOK, rr.Code) - - // suite.Equal("text/eventstream", rr.Header().Get("content-type")) + suite.Equal("text/event-stream", rr.Header().Get("content-type")) + suite.Equal("id: 0\nevent: message\ndata: CLEAN\n\n", rr.Body.String()) }) } From f825beff472b4e76e7c7830a68f4eaedd10db8d0 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 7 Jan 2025 19:42:39 +0000 Subject: [PATCH 21/24] B-21669 - trying to mock dependencies. --- pkg/notifications/notification_receiver.go | 37 +++++-- .../notification_receiver_test.go | 99 +++++++++++++++---- 2 files changed, 110 insertions(+), 26 deletions(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index cee3b300ac2..74d272c8fd7 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -12,7 +12,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/gofrs/uuid" - "github.com/spf13/viper" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" @@ -38,6 +37,7 @@ type NotificationReceiver interface { // NotificationReceiverConext provides context to a notification Receiver. Maps use queueUrl for key type NotificationReceiverContext struct { + viper ViperType snsService *sns.Client sqsService *sqs.Client awsRegion string @@ -46,6 +46,19 @@ type NotificationReceiverContext struct { receiverCancelMap map[string]context.CancelFunc } +type SnsClient interface { + *sns.Client +} + +type SqsClient interface { + *sqs.Client +} + +type ViperType interface { + GetString(string) string + SetEnvKeyReplacer(*strings.Replacer) +} + // ReceivedMessage standardizes the format of the received message type ReceivedMessage struct { MessageId string @@ -53,8 +66,9 @@ type ReceivedMessage struct { } // NewNotificationReceiver returns a new NotificationReceiverContext -func NewNotificationReceiver(snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { +func NewNotificationReceiver(v ViperType, snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ + viper: v, snsService: snsService, sqsService: sqsService, awsRegion: awsRegion, @@ -182,11 +196,11 @@ func (n NotificationReceiverContext) CloseoutQueue(appCtx appcontext.AppContext, // GetDefaultTopic returns the topic value set within the environment func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { - v := viper.New() - v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - v.AutomaticEnv() - topicName := v.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) - receiverBackend := v.GetString(cli.ReceiverBackendFlag) + // v := viper.New() + n.viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + // v.AutomaticEnv() + topicName := n.viper.GetString(cli.AWSSNSObjectTagsAddedTopicFlag) + receiverBackend := n.viper.GetString(cli.ReceiverBackendFlag) if topicName == "" && receiverBackend == "sns&sqs" { return "", errors.New("aws_sns_object_tags_added_topic key not available") } @@ -194,7 +208,11 @@ func (n NotificationReceiverContext) GetDefaultTopic() (string, error) { } // InitReceiver initializes the receiver backend -func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, error) { +func InitReceiver(v ViperType, logger *zap.Logger) (NotificationReceiver, error) { + + // v := viper.New() + // v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + // v.AutomaticEnv() if v.GetString(cli.ReceiverBackendFlag) == "sns&sqs" { // Setup notification receiver service with SNS & SQS backend dependencies @@ -208,12 +226,13 @@ func InitReceiver(v *viper.Viper, logger *zap.Logger) (NotificationReceiver, err ) if err != nil { logger.Fatal("error loading sns aws config", zap.Error(err)) + return nil, err } snsService := sns.NewFromConfig(cfg) sqsService := sqs.NewFromConfig(cfg) - return NewNotificationReceiver(snsService, sqsService, awsSNSRegion, awsAccountId), nil + return NewNotificationReceiver(v, snsService, sqsService, awsSNSRegion, awsAccountId), nil } return NewStubNotificationReceiver(), nil diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index 5add62ddb14..34546ad5ed4 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -2,11 +2,15 @@ package notifications import ( "fmt" + "strings" "testing" "github.com/spf13/viper" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "github.com/transcom/mymove/pkg/cli" "github.com/transcom/mymove/pkg/testingsuite" ) @@ -24,23 +28,84 @@ func TestNotificationReceiverSuite(t *testing.T) { hs.PopTestSuite.TearDown() } -func (suite *notificationReceiverSuite) TestNotificationReceiverLocalStub() { - v := viper.New() - localReceiver, err := InitReceiver(v, suite.Logger()) - - suite.NoError(err) +type Viper struct { + mock.Mock +} - queueParams := NotificationQueueParams{ - NamePrefix: "testPrefix", +func (_m *Viper) GetString(key string) string { + switch key { + case cli.ReceiverBackendFlag: + return "sns&sqs" + case cli.AWSRegionFlag: + return "us-gov-west-1" + case cli.AWSSNSAccountId: + return "12345" + case cli.AWSSNSObjectTagsAddedTopicFlag: + return "fake_sns_topic" } - createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) - suite.NoError(err) - suite.NotContains(createdQueueUrl, queueParams.NamePrefix) - suite.Equal(createdQueueUrl, "stubQueueName") - - receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) - suite.NoError(err) - suite.Len(receivedMessages, 1) - suite.Equal(receivedMessages[0].MessageId, "stubMessageId") - suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) + return "" } + +func (_m *Viper) SetEnvKeyReplacer(_ *strings.Replacer) {} + +func (suite *notificationReceiverSuite) TestSuccessPath() { + + suite.Run("local backend - notification receiver stub", func() { + v := viper.New() + localReceiver, err := InitReceiver(v, suite.Logger()) + + suite.NoError(err) + suite.IsType(StubNotificationReceiver{}, localReceiver) + + queueParams := NotificationQueueParams{ + NamePrefix: "testPrefix", + } + createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + suite.NoError(err) + suite.NotContains(createdQueueUrl, queueParams.NamePrefix) + suite.Equal(createdQueueUrl, "stubQueueName") + + receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + suite.Len(receivedMessages, 1) + suite.Equal(receivedMessages[0].MessageId, "stubMessageId") + suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) + }) + + suite.Run("aws backend - notification receiver", func() { + v := Viper{} + + rec, _ := InitReceiver(&v, suite.Logger()) + suite.IsType(NotificationReceiverContext{}, rec) + defaultTopic, err := rec.GetDefaultTopic() + suite.Logger().Error("%s", zap.String("default topic", defaultTopic)) + suite.Equal("fake_sns_topic", defaultTopic) + suite.NoError(err) + + // queueParams := NotificationQueueParams{ + // NamePrefix: "testPrefix", + // } + // createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + // suite.NoError(err) + // suite.NotContains(createdQueueUrl, queueParams.NamePrefix) + // suite.Equal(createdQueueUrl, "stubQueueName") + + // receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + // suite.NoError(err) + // suite.Len(receivedMessages, 1) + // suite.Equal(receivedMessages[0].MessageId, "stubMessageId") + // suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) + }) + +} + +// func (suite *notificationReceiverSuite) TestNotificationReceiverInitReceiver + +// func (suite *notificationReceiverSuite) TestNotificationReceiverAWS() { +// v := viper.New() +// v.Set(cli.ReceiverBackendFlag, "sns&sqs") +// v.Set(cli.AWSSNSRegionFlag, "us-gov-west-1") +// v.Set(cli.AWSSNSAccountId, "12345") + +// awsReceiver, err := InitReceiver(v, suite.Logger()) +// } From bd2254bdcd43acec49fb5a0238beb3bba161aec7 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 7 Jan 2025 21:02:54 +0000 Subject: [PATCH 22/24] B-21669 - working test with mock sns & sqs services. --- pkg/notifications/notification_receiver.go | 13 ++- .../notification_receiver_stub.go | 2 +- .../notification_receiver_test.go | 107 +++++++++++++----- 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/pkg/notifications/notification_receiver.go b/pkg/notifications/notification_receiver.go index 74d272c8fd7..b1c95495bc7 100644 --- a/pkg/notifications/notification_receiver.go +++ b/pkg/notifications/notification_receiver.go @@ -38,8 +38,8 @@ type NotificationReceiver interface { // NotificationReceiverConext provides context to a notification Receiver. Maps use queueUrl for key type NotificationReceiverContext struct { viper ViperType - snsService *sns.Client - sqsService *sqs.Client + snsService SnsClient + sqsService SqsClient awsRegion string awsAccountId string queueSubscriptionMap map[string]string @@ -47,11 +47,14 @@ type NotificationReceiverContext struct { } type SnsClient interface { - *sns.Client + Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) + Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) } type SqsClient interface { - *sqs.Client + CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) + ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) + DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) } type ViperType interface { @@ -66,7 +69,7 @@ type ReceivedMessage struct { } // NewNotificationReceiver returns a new NotificationReceiverContext -func NewNotificationReceiver(v ViperType, snsService *sns.Client, sqsService *sqs.Client, awsRegion string, awsAccountId string) NotificationReceiverContext { +func NewNotificationReceiver(v ViperType, snsService SnsClient, sqsService SqsClient, awsRegion string, awsAccountId string) NotificationReceiverContext { return NotificationReceiverContext{ viper: v, snsService: snsService, diff --git a/pkg/notifications/notification_receiver_stub.go b/pkg/notifications/notification_receiver_stub.go index 5b44471f608..f87806b9451 100644 --- a/pkg/notifications/notification_receiver_stub.go +++ b/pkg/notifications/notification_receiver_stub.go @@ -45,5 +45,5 @@ func (n StubNotificationReceiver) CloseoutQueue(appCtx appcontext.AppContext, qu } func (n StubNotificationReceiver) GetDefaultTopic() (string, error) { - return "", nil + return "stubDefaultTopic", nil } diff --git a/pkg/notifications/notification_receiver_test.go b/pkg/notifications/notification_receiver_test.go index 34546ad5ed4..836ed986c19 100644 --- a/pkg/notifications/notification_receiver_test.go +++ b/pkg/notifications/notification_receiver_test.go @@ -1,14 +1,18 @@ package notifications import ( + "context" "fmt" "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/spf13/viper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "go.uber.org/zap" "github.com/transcom/mymove/pkg/cli" "github.com/transcom/mymove/pkg/testingsuite" @@ -28,6 +32,7 @@ func TestNotificationReceiverSuite(t *testing.T) { hs.PopTestSuite.TearDown() } +// mock - Viper type Viper struct { mock.Mock } @@ -45,9 +50,45 @@ func (_m *Viper) GetString(key string) string { } return "" } - func (_m *Viper) SetEnvKeyReplacer(_ *strings.Replacer) {} +// mock - SNS +type MockSnsClient struct { + mock.Mock +} + +func (_m *MockSnsClient) Subscribe(ctx context.Context, params *sns.SubscribeInput, optFns ...func(*sns.Options)) (*sns.SubscribeOutput, error) { + return &sns.SubscribeOutput{SubscriptionArn: aws.String("FakeSubscriptionArn")}, nil +} + +func (_m *MockSnsClient) Unsubscribe(ctx context.Context, params *sns.UnsubscribeInput, optFns ...func(*sns.Options)) (*sns.UnsubscribeOutput, error) { + return &sns.UnsubscribeOutput{}, nil +} + +// mock - SQS +type MockSqsClient struct { + mock.Mock +} + +func (_m *MockSqsClient) CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { + return &sqs.CreateQueueOutput{ + QueueUrl: aws.String("FakeQueueUrl"), + }, nil +} +func (_m *MockSqsClient) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) { + messages := make([]types.Message, 0) + messages = append(messages, types.Message{ + MessageId: aws.String("fakeMessageId"), + Body: aws.String(*params.QueueUrl + ":fakeMessageBody"), + }) + return &sqs.ReceiveMessageOutput{ + Messages: messages, + }, nil +} +func (_m *MockSqsClient) DeleteQueue(ctx context.Context, params *sqs.DeleteQueueInput, optFns ...func(*sqs.Options)) (*sqs.DeleteQueueOutput, error) { + return &sqs.DeleteQueueOutput{}, nil +} + func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Run("local backend - notification receiver stub", func() { @@ -57,6 +98,10 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.NoError(err) suite.IsType(StubNotificationReceiver{}, localReceiver) + defaultTopic, err := localReceiver.GetDefaultTopic() + suite.Equal("stubDefaultTopic", defaultTopic) + suite.NoError(err) + queueParams := NotificationQueueParams{ NamePrefix: "testPrefix", } @@ -72,40 +117,42 @@ func (suite *notificationReceiverSuite) TestSuccessPath() { suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) }) - suite.Run("aws backend - notification receiver", func() { + suite.Run("aws backend - notification receiver init", func() { v := Viper{} - rec, _ := InitReceiver(&v, suite.Logger()) - suite.IsType(NotificationReceiverContext{}, rec) - defaultTopic, err := rec.GetDefaultTopic() - suite.Logger().Error("%s", zap.String("default topic", defaultTopic)) + receiver, _ := InitReceiver(&v, suite.Logger()) + suite.IsType(NotificationReceiverContext{}, receiver) + defaultTopic, err := receiver.GetDefaultTopic() suite.Equal("fake_sns_topic", defaultTopic) suite.NoError(err) - - // queueParams := NotificationQueueParams{ - // NamePrefix: "testPrefix", - // } - // createdQueueUrl, err := localReceiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) - // suite.NoError(err) - // suite.NotContains(createdQueueUrl, queueParams.NamePrefix) - // suite.Equal(createdQueueUrl, "stubQueueName") - - // receivedMessages, err := localReceiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) - // suite.NoError(err) - // suite.Len(receivedMessages, 1) - // suite.Equal(receivedMessages[0].MessageId, "stubMessageId") - // suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:stubMessageBody", createdQueueUrl)) }) -} + suite.Run("aws backend - notification receiver with mock services", func() { + v := Viper{} + snsService := MockSnsClient{} + sqsService := MockSqsClient{} -// func (suite *notificationReceiverSuite) TestNotificationReceiverInitReceiver + receiver := NewNotificationReceiver(&v, &snsService, &sqsService, "", "") + suite.IsType(NotificationReceiverContext{}, receiver) -// func (suite *notificationReceiverSuite) TestNotificationReceiverAWS() { -// v := viper.New() -// v.Set(cli.ReceiverBackendFlag, "sns&sqs") -// v.Set(cli.AWSSNSRegionFlag, "us-gov-west-1") -// v.Set(cli.AWSSNSAccountId, "12345") + defaultTopic, err := receiver.GetDefaultTopic() + suite.Equal("fake_sns_topic", defaultTopic) + suite.NoError(err) -// awsReceiver, err := InitReceiver(v, suite.Logger()) -// } + queueParams := NotificationQueueParams{ + NamePrefix: "testPrefix", + } + createdQueueUrl, err := receiver.CreateQueueWithSubscription(suite.AppContextForTest(), queueParams) + suite.NoError(err) + suite.Equal("FakeQueueUrl", createdQueueUrl) + + receivedMessages, err := receiver.ReceiveMessages(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + suite.Len(receivedMessages, 1) + suite.Equal(receivedMessages[0].MessageId, "fakeMessageId") + suite.Equal(*receivedMessages[0].Body, fmt.Sprintf("%s:fakeMessageBody", createdQueueUrl)) + + err = receiver.CloseoutQueue(suite.AppContextForTest(), createdQueueUrl) + suite.NoError(err) + }) +} From 6e122f4bc33e41a3b86627f9cb0b37683d7dde67 Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Tue, 7 Jan 2025 22:52:40 +0000 Subject: [PATCH 23/24] B-21669 - working close call on server; cleanup for handler. --- pkg/handlers/internalapi/uploads.go | 48 ++++++++++++++++------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index 7901cae3d19..e5de7e9034d 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -14,6 +14,7 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/gobuffalo/validate/v3" "github.com/gofrs/uuid" + "github.com/pkg/errors" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" @@ -276,8 +277,14 @@ const ( AVStatusTypeINFECTED AVStatusType = "INFECTED" ) -func constructEventStreamMessage(id int, data string) []byte { - return []byte(fmt.Sprintf("id: %s\nevent: message\ndata: %s\n\n", strconv.Itoa(id), data)) +func writeEventStreamMessage(rw http.ResponseWriter, producer runtime.Producer, id int, event string, data string) { + resProcess := []byte(fmt.Sprintf("id: %s\nevent: %s\ndata: %s\n\n", strconv.Itoa(id), event, data)) + if produceErr := producer.Produce(rw, resProcess); produceErr != nil { + panic(produceErr) + } + if f, ok := rw.(http.Flusher); ok { + f.Flush() + } } func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { @@ -305,17 +312,12 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer } else { uploadStatus = AVStatusType(tags["av-status"]) } - resProcess := constructEventStreamMessage(0, string(uploadStatus)) - if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) - } - if f, ok := rw.(http.Flusher); ok { - f.Flush() - } + writeEventStreamMessage(rw, producer, 0, "message", string(uploadStatus)) if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { - return + writeEventStreamMessage(rw, producer, 1, "close", "Connection closed") + return // skip notification loop since object already tagged from anti-virus } // Start waiting for tag updates @@ -352,7 +354,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer _ = o.receiver.CloseoutQueue(o.appCtx, queueUrl) }() - id_counter := 0 + id_counter := 1 // Run for 120 seconds, 20 second long polling for receiver, 6 times for range 6 { o.appCtx.Logger().Info("Receiving...") @@ -376,25 +378,27 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer uploadStatus = AVStatusType(tags["av-status"]) } - resProcess := constructEventStreamMessage(id_counter, string(uploadStatus)) - if produceErr := producer.Produce(rw, resProcess); produceErr != nil { - panic(produceErr) // let the recovery middleware deal with this + writeEventStreamMessage(rw, producer, id_counter, "message", string(uploadStatus)) + + if uploadStatus == AVStatusTypeCLEAN || uploadStatus == AVStatusTypeINFECTED { + return errors.New("connection_closed") } - return nil + + return err }) - if errTransaction != nil { - o.appCtx.Logger().Error(err.Error()) + if errTransaction != nil && errTransaction.Error() == "connection_closed" { + id_counter++ + writeEventStreamMessage(rw, producer, id_counter, "close", "Connection closed") + break } - } - if f, ok := rw.(http.Flusher); ok { - f.Flush() + if errTransaction != nil { + panic(errTransaction) // let the recovery middleware deal with this + } } id_counter++ } - - // TODO: add a close here after ends } // Handle returns status of an upload From 2fa55a12b172894a97fcadd4dedf5d668f57fdda Mon Sep 17 00:00:00 2001 From: ryan-mchugh Date: Wed, 8 Jan 2025 18:42:27 +0000 Subject: [PATCH 24/24] B-21669 - fix tests and error handling on endpoint. --- pkg/handlers/internalapi/uploads.go | 63 +++++++++++-------- pkg/handlers/internalapi/uploads_test.go | 48 +++++++++++--- .../routing/internalapi_test/uploads_test.go | 2 +- 3 files changed, 79 insertions(+), 34 deletions(-) diff --git a/pkg/handlers/internalapi/uploads.go b/pkg/handlers/internalapi/uploads.go index e5de7e9034d..834d2124d43 100644 --- a/pkg/handlers/internalapi/uploads.go +++ b/pkg/handlers/internalapi/uploads.go @@ -259,10 +259,11 @@ type GetUploadStatusHandler struct { } type CustomNewUploadStatusOK struct { - params uploadop.GetUploadStatusParams - appCtx appcontext.AppContext - receiver notifications.NotificationReceiver - storer storage.FileStorer + params uploadop.GetUploadStatusParams + storageKey string + appCtx appcontext.AppContext + receiver notifications.NotificationReceiver + storer storage.FileStorer } // AVStatusType represents the type of the anti-virus status, whether it is still processing, clean or infected @@ -289,23 +290,8 @@ func writeEventStreamMessage(rw http.ResponseWriter, producer runtime.Producer, func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - // TODO: add check for permissions to view upload - - uploadId := o.params.UploadID.String() - - uploadUUID, err := uuid.FromString(uploadId) - if err != nil { - uploadop.NewGetUploadStatusInternalServerError().WriteResponse(rw, producer) - panic(err) - } - // Check current tag before event-driven wait for anti-virus - uploaded, err := models.FetchUserUploadFromUploadID(o.appCtx.DB(), o.appCtx.Session(), uploadUUID) - if err != nil { - panic(err) - } - - tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + tags, err := o.storer.Tags(o.storageKey) var uploadStatus AVStatusType if err != nil || len(tags) == 0 { uploadStatus = AVStatusTypePROCESSING @@ -335,7 +321,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer ] } } - }`, uploadId) + }`, o.params.UploadID) notificationParams := notifications.NotificationQueueParams{ SubscriptionTopicName: topicName, @@ -370,7 +356,7 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer if len(messages) != 0 { errTransaction := o.appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - tags, err := o.storer.Tags(uploaded.Upload.StorageKey) + tags, err := o.storer.Tags(o.storageKey) if err != nil || len(tags) == 0 { uploadStatus = AVStatusTypePROCESSING @@ -405,11 +391,36 @@ func (o *CustomNewUploadStatusOK) WriteResponse(rw http.ResponseWriter, producer func (h GetUploadStatusHandler) Handle(params uploadop.GetUploadStatusParams) middleware.Responder { return h.AuditableAppContextFromRequestWithErrors(params.HTTPRequest, func(appCtx appcontext.AppContext) (middleware.Responder, error) { + + handleError := func(err error) (middleware.Responder, error) { + appCtx.Logger().Error("GetUploadStatusHandler error", zap.Error(err)) + switch errors.Cause(err) { + case models.ErrFetchForbidden: + return uploadop.NewGetUploadStatusForbidden(), err + case models.ErrFetchNotFound: + return uploadop.NewGetUploadStatusNotFound(), err + default: + return uploadop.NewGetUploadStatusInternalServerError(), err + } + } + + uploadId := params.UploadID.String() + uploadUUID, err := uuid.FromString(uploadId) + if err != nil { + return handleError(err) + } + + uploaded, err := models.FetchUserUploadFromUploadID(appCtx.DB(), appCtx.Session(), uploadUUID) + if err != nil { + return handleError(err) + } + return &CustomNewUploadStatusOK{ - params: params, - appCtx: h.AppContextFromRequest(params.HTTPRequest), - receiver: h.NotificationReceiver(), - storer: h.FileStorer(), + params: params, + storageKey: uploaded.Upload.StorageKey, + appCtx: h.AppContextFromRequest(params.HTTPRequest), + receiver: h.NotificationReceiver(), + storer: h.FileStorer(), }, nil }) } diff --git a/pkg/handlers/internalapi/uploads_test.go b/pkg/handlers/internalapi/uploads_test.go index bc07c4a5619..143dfa465eb 100644 --- a/pkg/handlers/internalapi/uploads_test.go +++ b/pkg/handlers/internalapi/uploads_test.go @@ -449,15 +449,14 @@ func (suite *HandlerSuite) TestDeleteUploadHandlerSuccessEvenWithS3Failure() { suite.NotNil(queriedUpload.DeletedAt) } -// TODO: functioning test func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { fakeS3 := storageTest.NewFakeS3Storage(true) localReceiver := notifications.StubNotificationReceiver{} - move := factory.BuildMove(suite.DB(), nil, nil) + orders := factory.BuildOrder(suite.DB(), nil, nil) uploadUser1 := factory.BuildUserUpload(suite.DB(), []factory.Customization{ { - Model: move.Orders.UploadedOrders, + Model: orders.UploadedOrders, LinkOnly: true, }, { @@ -470,10 +469,11 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { }, nil) file := suite.Fixture(FixturePDF) - fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + _, err := fakeS3.Store(uploadUser1.Upload.StorageKey, file.Data, "somehash", nil) + suite.NoError(err) params := uploadop.NewGetUploadStatusParams() - params.UploadID = strfmt.UUID(uploadUser1.ID.String()) + params.UploadID = strfmt.UUID(uploadUser1.Upload.ID.String()) req := &http.Request{} req = suite.AuthenticateRequest(req, uploadUser1.Document.ServiceMember) @@ -490,8 +490,42 @@ func (suite *HandlerSuite) TestGetUploadStatusHandlerSuccess() { suite.True(ok) queriedUpload := models.Upload{} - err := suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) - suite.Nil(err) + err = suite.DB().Find(&queriedUpload, uploadUser1.Upload.ID) + suite.NoError(err) +} + +func (suite *HandlerSuite) TestGetUploadStatusHandlerFailure() { + suite.Run("Error on no match for uploadId", func() { + orders := factory.BuildOrder(suite.DB(), factory.GetTraitActiveServiceMemberUser(), nil) + + uploadUUID := uuid.Must(uuid.NewV4()) + + params := uploadop.NewGetUploadStatusParams() + params.UploadID = strfmt.UUID(uploadUUID.String()) + + req := &http.Request{} + req = suite.AuthenticateRequest(req, orders.ServiceMember) + params.HTTPRequest = req + + fakeS3 := storageTest.NewFakeS3Storage(true) + localReceiver := notifications.StubNotificationReceiver{} + + handlerConfig := suite.HandlerConfig() + handlerConfig.SetFileStorer(fakeS3) + handlerConfig.SetNotificationReceiver(localReceiver) + uploadInformationFetcher := upload.NewUploadInformationFetcher() + handler := GetUploadStatusHandler{handlerConfig, uploadInformationFetcher} + + response := handler.Handle(params) + _, ok := response.(*uploadop.GetUploadStatusNotFound) + suite.True(ok) + + queriedUpload := models.Upload{} + err := suite.DB().Find(&queriedUpload, uploadUUID) + suite.Error(err) + }) + + // TODO: ADD A FORBIDDEN TEST } func (suite *HandlerSuite) TestCreatePPMUploadsHandlerSuccess() { diff --git a/pkg/handlers/routing/internalapi_test/uploads_test.go b/pkg/handlers/routing/internalapi_test/uploads_test.go index fade7975cc6..3fe89e8927d 100644 --- a/pkg/handlers/routing/internalapi_test/uploads_test.go +++ b/pkg/handlers/routing/internalapi_test/uploads_test.go @@ -36,6 +36,6 @@ func (suite *InternalAPISuite) TestUploads() { suite.Equal(http.StatusOK, rr.Code) suite.Equal("text/event-stream", rr.Header().Get("content-type")) - suite.Equal("id: 0\nevent: message\ndata: CLEAN\n\n", rr.Body.String()) + suite.Equal("id: 0\nevent: message\ndata: CLEAN\n\nid: 1\nevent: close\ndata: Connection closed\n\n", rr.Body.String()) }) }