From 6520adf3ba7ee99beb21656a54d0921a1d51784d Mon Sep 17 00:00:00 2001 From: Eric McGarry <46828798+mcgarrye@users.noreply.github.com> Date: Tue, 4 Feb 2025 16:30:40 -0500 Subject: [PATCH] release: refactor and extend zip code (#4588) (#1062) --- api/src/controllers/application.controller.ts | 14 ++- api/src/controllers/lottery.controller.ts | 5 +- .../services/application-exporter.service.ts | 107 ++++++++++-------- api/src/utilities/zip-export.ts | 39 +++++++ .../application-exporter.service.spec.ts | 28 ++--- sites/partners/src/lib/hooks.ts | 60 ++++------ .../src/pages/api/adapter/[...backendUrl].ts | 7 +- .../listings/[id]/applications/index.tsx | 5 +- .../src/pages/listings/[id]/lottery.tsx | 8 +- 9 files changed, 154 insertions(+), 119 deletions(-) create mode 100644 api/src/utilities/zip-export.ts diff --git a/api/src/controllers/application.controller.ts b/api/src/controllers/application.controller.ts index 6c302f5a2f..678d5b6b54 100644 --- a/api/src/controllers/application.controller.ts +++ b/api/src/controllers/application.controller.ts @@ -115,15 +115,19 @@ export class ApplicationController { summary: 'Get applications as csv', operationId: 'listAsCsv', }) - @Header('Content-Type', 'text/csv') + @Header('Content-Type', 'application/zip') @UseInterceptors(ExportLogInterceptor) async listAsCsv( @Request() req: ExpressRequest, - @Res({ passthrough: true }) res: Response, @Query(new ValidationPipe(defaultValidationPipeOptions)) queryParams: ApplicationCsvQueryParams, ): Promise { - return await this.applicationExportService.csvExport(req, res, queryParams); + return await this.applicationExportService.exporter( + req, + queryParams, + false, + false, + ); } @Get(`spreadsheet`) @@ -139,11 +143,11 @@ export class ApplicationController { @Query(new ValidationPipe(defaultValidationPipeOptions)) queryParams: ApplicationCsvQueryParams, ): Promise { - return await this.applicationExportService.spreadsheetExport( + return await this.applicationExportService.exporter( req, - res, queryParams, false, + true, ); } diff --git a/api/src/controllers/lottery.controller.ts b/api/src/controllers/lottery.controller.ts index cc9f12da00..a71965fb46 100644 --- a/api/src/controllers/lottery.controller.ts +++ b/api/src/controllers/lottery.controller.ts @@ -84,10 +84,11 @@ export class LotteryController { @Query(new ValidationPipe(defaultValidationPipeOptions)) queryParams: ApplicationCsvQueryParams, ): Promise { - return await this.applicationExporterService.spreadsheetExport( + return await this.applicationExporterService.exporter( req, - res, queryParams, + true, + true, ); } diff --git a/api/src/services/application-exporter.service.ts b/api/src/services/application-exporter.service.ts index 899d5e68a7..29c01f9861 100644 --- a/api/src/services/application-exporter.service.ts +++ b/api/src/services/application-exporter.service.ts @@ -1,9 +1,9 @@ import { ForbiddenException, Injectable, StreamableFile } from '@nestjs/common'; import { MultiselectQuestionsApplicationSectionEnum } from '@prisma/client'; -import archiver from 'archiver'; +import dayjs from 'dayjs'; import Excel, { Column } from 'exceljs'; -import { Request as ExpressRequest, Response } from 'express'; -import fs, { createReadStream } from 'fs'; +import { Request as ExpressRequest } from 'express'; +import fs, { createReadStream, ReadStream } from 'fs'; import { join } from 'path'; import { view } from './application.service'; import { Application } from '../dtos/applications/application.dto'; @@ -21,6 +21,7 @@ import { PrismaService } from './prisma.service'; import { CsvHeader } from '../types/CsvExportInterface'; import { getExportHeaders } from '../utilities/application-export-helpers'; import { mapTo } from '../utilities/mapTo'; +import { zipExport } from '../utilities/zip-export'; view.csv = { ...view.details, @@ -43,31 +44,69 @@ export class ApplicationExporterService { private permissionService: PermissionService, ) {} - // csv export functions /** * - * @param queryParams * @param req + * @param queryParams + * @param isLottery a boolean indicating if the export is a lottery + * @param isSpreadsheet a boolean indicating if the export is a spreadsheet * @returns a promise containing a streamable file */ - async csvExport( + async exporter( req: ExpressRequest, - res: Response, queryParams: QueryParams, + isLottery: boolean, + isSpreadsheet: boolean, ): Promise { const user = mapTo(User, req['user']); await this.authorizeExport(user, queryParams.id); + let filename: string; + let readStream: ReadStream; + let zipFilename: string; + const now = new Date(); + const dateString = dayjs(now).format('YYYY-MM-DD_HH-mm'); + if (isLottery) { + readStream = await this.spreadsheetExport(queryParams, user.id, true); + zipFilename = `listing-${queryParams.id}-lottery-${ + user.id + }-${now.getTime()}`; + filename = `lottery-${queryParams.id}-${dateString}`; + } else { + if (isSpreadsheet) { + readStream = await this.spreadsheetExport(queryParams, user.id, false); + } else { + readStream = await this.csvExport(queryParams, user.id); + } + zipFilename = `listing-${queryParams.id}-applications-${ + user.id + }-${now.getTime()}`; + filename = `applications-${queryParams.id}-${dateString}`; + } + + return await zipExport(readStream, zipFilename, filename, isSpreadsheet); + } + + // csv export functions + /** + * + * @param queryParams + * @param user_id + * @returns a promise containing a file read stream + */ + async csvExport( + queryParams: QueryParams, + user_id: string, + ): Promise { const filename = join( process.cwd(), - `src/temp/listing-${queryParams.id}-applications-${ - user.id - }-${new Date().getTime()}.csv`, + `src/temp/listing-${ + queryParams.id + }-applications-${user_id}-${new Date().getTime()}.csv`, ); await this.createCsv(filename, queryParams); - const file = createReadStream(filename); - return new StreamableFile(file); + return createReadStream(filename); } /** @@ -319,23 +358,20 @@ export class ApplicationExporterService { /** * * @param queryParams - * @param req - * @returns generates the lottery export file via helper function and returns the streamable file + * @param user + * @param forLottery + * @returns generates the applications or lottery spreadsheet export and returns a promise containing a file read stream */ async spreadsheetExport( - req: ExpressRequest, - res: Response, queryParams: QueryParams, + user_id: string, forLottery = true, - ): Promise { - const user = mapTo(User, req['user']); - await this.authorizeExport(user, queryParams.id); - + ): Promise { const filename = join( process.cwd(), `src/temp/${forLottery ? 'lottery-' : ''}listing-${ queryParams.id - }-applications-${user.id}-${new Date().getTime()}.xlsx`, + }-applications-${user_id}-${new Date().getTime()}.xlsx`, ); const workbook = new Excel.stream.xlsx.WorkbookWriter({ @@ -343,13 +379,6 @@ export class ApplicationExporterService { useSharedStrings: false, }); - const zipFilePath = join( - process.cwd(), - `src/temp/${forLottery ? 'lottery-' : ''}listing-${ - queryParams.id - }-applications-${user.id}-${new Date().getTime()}.zip`, - ); - await this.createSpreadsheets( workbook, { @@ -359,27 +388,7 @@ export class ApplicationExporterService { ); await workbook.commit(); - const readStream = createReadStream(filename); - - return new Promise((resolve) => { - // Create a writable stream to the zip file - const output = fs.createWriteStream(zipFilePath); - const archive = archiver('zip', { - zlib: { level: 9 }, - }); - output.on('close', () => { - const zipFile = createReadStream(zipFilePath); - resolve(new StreamableFile(zipFile)); - }); - - archive.pipe(output); - archive.append(readStream, { - name: `${forLottery ? 'lottery-' : ''}${ - queryParams.id - }-${new Date().getTime()}.xlsx`, - }); - archive.finalize(); - }); + return createReadStream(filename); } /** diff --git a/api/src/utilities/zip-export.ts b/api/src/utilities/zip-export.ts new file mode 100644 index 0000000000..e7841f44b0 --- /dev/null +++ b/api/src/utilities/zip-export.ts @@ -0,0 +1,39 @@ +import { StreamableFile } from '@nestjs/common'; +import archiver from 'archiver'; +import fs, { createReadStream, ReadStream } from 'fs'; +import { join } from 'path'; + +/** + * + * @param readStream + * @param zipFilename + * @param filename + * @param isSpreadsheet + * @returns a promise containing a streamable file + */ +export const zipExport = ( + readStream: ReadStream, + zipFilename: string, + filename: string, + isSpreadsheet: boolean, +): Promise => { + const zipFilePath = join(process.cwd(), `src/temp/${zipFilename}.zip`); + + return new Promise((resolve) => { + // Create a writable stream to the zip file + const output = fs.createWriteStream(zipFilePath); + const archive = archiver('zip', { + zlib: { level: 9 }, + }); + output.on('close', () => { + const zipFile = createReadStream(zipFilePath); + resolve(new StreamableFile(zipFile)); + }); + + archive.pipe(output); + archive.append(readStream, { + name: `${filename}.${isSpreadsheet ? 'xlsx' : 'csv'}`, + }); + archive.finalize(); + }); +}; diff --git a/api/test/unit/services/application-exporter.service.spec.ts b/api/test/unit/services/application-exporter.service.spec.ts index e500ea3fa8..63ebf5bcc6 100644 --- a/api/test/unit/services/application-exporter.service.spec.ts +++ b/api/test/unit/services/application-exporter.service.spec.ts @@ -3,7 +3,6 @@ import { PassThrough } from 'stream'; import { Test, TestingModule } from '@nestjs/testing'; import { MultiselectQuestionsApplicationSectionEnum } from '@prisma/client'; import { HttpModule } from '@nestjs/axios'; -import { Request as ExpressRequest, Response } from 'express'; import { PrismaService } from '../../../src/services/prisma.service'; import { ApplicationCsvQueryParams } from '../../../src/dtos/applications/application-csv-query-params.dto'; import { User } from '../../../src/dtos/users/user.dto'; @@ -97,12 +96,11 @@ describe('Testing application export service', () => { ]); const exportResponse = await service.csvExport( - { user: requestingUser } as unknown as ExpressRequest, - {} as unknown as Response, { id: randomUUID(), includeDemographics: false, } as unknown as ApplicationCsvQueryParams, + requestingUser.id, ); const headerRow = @@ -111,7 +109,7 @@ describe('Testing application export service', () => { '"application 0 firstName","application 0 middleName","application 0 lastName","application 0 birthDay","application 0 birthMonth","application 0 birthYear","application 0 emailaddress","application 0 phoneNumber","application 0 phoneNumberType","additionalPhoneNumber 0","application 0 applicantAddress street","application 0 applicantAddress street2","application 0 applicantAddress city","application 0 applicantAddress state","application 0 applicantAddress zipCode",,,,,,,,,,,,,,,,,,"income 0","per month",,,,"true","true","true","Studio,One Bedroom",,,,,,,,,,,,,,,,,,,'; const mockedStream = new PassThrough(); - exportResponse.getStream().pipe(mockedStream); + exportResponse.pipe(mockedStream); // In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it const readable = await new Promise((resolve) => { @@ -156,12 +154,11 @@ describe('Testing application export service', () => { ]); const exportResponse = await service.csvExport( - { user: requestingUser } as unknown as ExpressRequest, - {} as unknown as Response, { id: 'test', includeDemographics: true, } as unknown as ApplicationCsvQueryParams, + requestingUser.id, ); const headerRow = @@ -170,7 +167,7 @@ describe('Testing application export service', () => { '"application 0 firstName","application 0 middleName","application 0 lastName","application 0 birthDay","application 0 birthMonth","application 0 birthYear","application 0 emailaddress","application 0 phoneNumber","application 0 phoneNumberType","additionalPhoneNumber 0","application 0 applicantAddress street","application 0 applicantAddress street2","application 0 applicantAddress city","application 0 applicantAddress state","application 0 applicantAddress zipCode",,,,,,,,,,,,,,,,,,"income 0","per month",,,,"true","true","true","Studio,One Bedroom",,,,,,"Indigenous",,,,"Other"'; const mockedStream = new PassThrough(); - exportResponse.getStream().pipe(mockedStream); + exportResponse.pipe(mockedStream); const readable = await new Promise((resolve) => { mockedStream.on('data', async (d) => { const value = Buffer.from(d).toString(); @@ -227,13 +224,12 @@ describe('Testing application export service', () => { .spyOn({ unitTypeToReadable }, 'unitTypeToReadable') .mockReturnValue('Studio'); const exportResponse = await service.csvExport( - { user: requestingUser } as unknown as ExpressRequest, - {} as unknown as Response, - { id: randomUUID() } as unknown as ApplicationCsvQueryParams, + { listingId: randomUUID() } as unknown as ApplicationCsvQueryParams, + requestingUser.id, ); const mockedStream = new PassThrough(); - exportResponse.getStream().pipe(mockedStream); + exportResponse.pipe(mockedStream); // In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it const readable = await new Promise((resolve) => { @@ -290,16 +286,15 @@ describe('Testing application export service', () => { .spyOn({ unitTypeToReadable }, 'unitTypeToReadable') .mockReturnValue('Studio'); const exportResponse = await service.csvExport( - { user: requestingUser } as unknown as ExpressRequest, - {} as unknown as Response, { id: randomUUID(), timeZone: 'America/New_York', } as unknown as ApplicationCsvQueryParams, + requestingUser.id, ); const mockedStream = new PassThrough(); - exportResponse.getStream().pipe(mockedStream); + exportResponse.pipe(mockedStream); // In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it const readable = await new Promise((resolve) => { @@ -359,16 +354,15 @@ describe('Testing application export service', () => { .spyOn({ unitTypeToReadable }, 'unitTypeToReadable') .mockReturnValue('Studio'); const exportResponse = await service.csvExport( - { user: requestingUser } as unknown as ExpressRequest, - {} as unknown as Response, { listingId: randomUUID(), timeZone: 'America/New_York', } as unknown as ApplicationCsvQueryParams, + requestingUser.id, ); const mockedStream = new PassThrough(); - exportResponse.getStream().pipe(mockedStream); + exportResponse.pipe(mockedStream); // In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it const readable = await new Promise((resolve) => { diff --git a/sites/partners/src/lib/hooks.ts b/sites/partners/src/lib/hooks.ts index b42cdf0b74..bf0ff51a53 100644 --- a/sites/partners/src/lib/hooks.ts +++ b/sites/partners/src/lib/hooks.ts @@ -512,37 +512,11 @@ export const createDateStringFromNow = (format = "YYYY-MM-DD_HH:mm:ss"): string return dayjs(now).format(format) } -export const useApplicationsExport = ( +export const useZipExport = ( listingId: string, includeDemographics: boolean, - spreadSheetExport = false -) => { - if (spreadSheetExport) { - // eslint-disable-next-line react-hooks/rules-of-hooks - return useSpreadsheetExport(listingId, includeDemographics, false) - } - - // eslint-disable-next-line react-hooks/rules-of-hooks - const { applicationsService } = useContext(AuthContext) - - // eslint-disable-next-line react-hooks/rules-of-hooks - const res = useCsvExport( - () => - applicationsService.listAsCsv({ - id: listingId, - includeDemographics, - timeZone: dayjs.tz.guess(), - }), - `applications-${listingId}-${createDateStringFromNow()}.csv` - ) - - return { onExport: res.onExport, exportLoading: res.csvExportLoading } -} - -export const useSpreadsheetExport = ( - listingId: string, - includeDemographics: boolean, - forLottery: boolean + isLottery: boolean, + isSpreadsheet = false ) => { const { applicationsService, lotteryService } = useContext(AuthContext) const [exportLoading, setExportLoading] = useState(false) @@ -551,24 +525,36 @@ export const useSpreadsheetExport = ( const onExport = useCallback(async () => { setExportLoading(true) try { - const content = forLottery - ? await lotteryService.lotteryResults( - { id: listingId, includeDemographics }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let content: any + if (isLottery) { + content = await lotteryService.lotteryResults( + { id: listingId, includeDemographics, timeZone: dayjs.tz.guess() }, + { responseType: "arraybuffer" } + ) + } else { + if (isSpreadsheet) { + content = await applicationsService.listAsSpreadsheet( + { id: listingId, includeDemographics, timeZone: dayjs.tz.guess() }, { responseType: "arraybuffer" } ) - : await applicationsService.listAsSpreadsheet( - { id: listingId, includeDemographics }, + } else { + content = await applicationsService.listAsCsv( + { id: listingId, includeDemographics, timeZone: dayjs.tz.guess() }, { responseType: "arraybuffer" } ) + } + } + const blob = new Blob([new Uint8Array(content)], { type: "application/zip" }) const url = window.URL.createObjectURL(blob) const link = document.createElement("a") link.href = url - const now = new Date() - const dateString = dayjs(now).format("YYYY-MM-DD_HH-mm") link.setAttribute( "download", - `${listingId}-${dateString}-${forLottery ? "lottery" : "applications"}.zip` + `${isLottery ? "lottery" : "applications"}-${listingId}-${createDateStringFromNow( + "YYYY-MM-DD_HH-mm" + )}.zip` ) document.body.appendChild(link) link.click() diff --git a/sites/partners/src/pages/api/adapter/[...backendUrl].ts b/sites/partners/src/pages/api/adapter/[...backendUrl].ts index eae1ee6bff..5780381b35 100644 --- a/sites/partners/src/pages/api/adapter/[...backendUrl].ts +++ b/sites/partners/src/pages/api/adapter/[...backendUrl].ts @@ -13,7 +13,12 @@ import { logger } from "../../../logger" */ // all endpoints that return a zip file -const zipEndpoints = ["listings/csv", "lottery/getLotteryResults", "applications/spreadsheet"] +const zipEndpoints = [ + "listings/csv", + "lottery/getLotteryResults", + "applications/spreadsheet", + "applications/csv", +] export default async (req: NextApiRequest, res: NextApiResponse) => { const jar = new CookieJar() diff --git a/sites/partners/src/pages/listings/[id]/applications/index.tsx b/sites/partners/src/pages/listings/[id]/applications/index.tsx index 07b0e99c36..def2942f02 100644 --- a/sites/partners/src/pages/listings/[id]/applications/index.tsx +++ b/sites/partners/src/pages/listings/[id]/applications/index.tsx @@ -19,7 +19,7 @@ import { useSingleListingData, useFlaggedApplicationsList, useApplicationsData, - useApplicationsExport, + useZipExport, } from "../../../../lib/hooks" import { ListingStatusBar } from "../../../../components/listings/ListingStatusBar" import Layout from "../../../../layouts" @@ -50,12 +50,13 @@ const ApplicationsList = () => { ) const includeDemographicsPartner = profile?.userRoles?.isPartner && listingJurisdiction?.enablePartnerDemographics - const { onExport, exportLoading } = useApplicationsExport( + const { onExport, exportLoading } = useZipExport( listingId, (profile?.userRoles?.isAdmin || profile?.userRoles?.isJurisdictionalAdmin || includeDemographicsPartner) ?? false, + false, !!process.env.applicationExportAsSpreadsheet ) diff --git a/sites/partners/src/pages/listings/[id]/lottery.tsx b/sites/partners/src/pages/listings/[id]/lottery.tsx index f9bfae0dc6..535df31ffc 100644 --- a/sites/partners/src/pages/listings/[id]/lottery.tsx +++ b/sites/partners/src/pages/listings/[id]/lottery.tsx @@ -28,11 +28,7 @@ import ListingGuard from "../../../components/shared/ListingGuard" import { NavigationHeader } from "../../../components/shared/NavigationHeader" import { ListingStatusBar } from "../../../components/listings/ListingStatusBar" import { logger } from "../../../logger" -import { - useFlaggedApplicationsMeta, - useLotteryActivityLog, - useSpreadsheetExport, -} from "../../../lib/hooks" +import { useFlaggedApplicationsMeta, useLotteryActivityLog, useZipExport } from "../../../lib/hooks" dayjs.extend(advancedFormat) import styles from "../../../../styles/lottery.module.scss" @@ -65,7 +61,7 @@ const Lottery = (props: { listing: Listing | undefined }) => { const includeDemographicsPartner = profile?.userRoles?.isPartner && listingJurisdiction?.enablePartnerDemographics - const { onExport, exportLoading } = useSpreadsheetExport( + const { onExport, exportLoading } = useZipExport( listing?.id, (profile?.userRoles?.isAdmin || profile?.userRoles?.isJurisdictionalAdmin ||