Skip to content

Commit

Permalink
release: refactor and extend zip code (bloom-housing#4588) (#1062)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mcgarrye
mcgarrye authored Feb 4, 2025
1 parent 54fe4b6 commit 6520adf
Show file tree
Hide file tree
Showing 9 changed files with 154 additions and 119 deletions.
14 changes: 9 additions & 5 deletions api/src/controllers/application.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,19 @@ export class ApplicationController {
summary: 'Get applications as csv',
operationId: 'listAsCsv',
})
@Header('Content-Type', 'text/csv')
@Header('Content-Type', 'application/zip')
@UseInterceptors(ExportLogInterceptor)
async listAsCsv(
@Request() req: ExpressRequest,
@Res({ passthrough: true }) res: Response,
@Query(new ValidationPipe(defaultValidationPipeOptions))
queryParams: ApplicationCsvQueryParams,
): Promise<StreamableFile> {
return await this.applicationExportService.csvExport(req, res, queryParams);
return await this.applicationExportService.exporter(
req,
queryParams,
false,
false,
);
}

@Get(`spreadsheet`)
Expand All @@ -139,11 +143,11 @@ export class ApplicationController {
@Query(new ValidationPipe(defaultValidationPipeOptions))
queryParams: ApplicationCsvQueryParams,
): Promise<StreamableFile> {
return await this.applicationExportService.spreadsheetExport(
return await this.applicationExportService.exporter(
req,
res,
queryParams,
false,
true,
);
}

Expand Down
5 changes: 3 additions & 2 deletions api/src/controllers/lottery.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,11 @@ export class LotteryController {
@Query(new ValidationPipe(defaultValidationPipeOptions))
queryParams: ApplicationCsvQueryParams,
): Promise<StreamableFile> {
return await this.applicationExporterService.spreadsheetExport(
return await this.applicationExporterService.exporter(
req,
res,
queryParams,
true,
true,
);
}

Expand Down
107 changes: 58 additions & 49 deletions api/src/services/application-exporter.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { ForbiddenException, Injectable, StreamableFile } from '@nestjs/common';
import { MultiselectQuestionsApplicationSectionEnum } from '@prisma/client';
import archiver from 'archiver';
import dayjs from 'dayjs';
import Excel, { Column } from 'exceljs';
import { Request as ExpressRequest, Response } from 'express';
import fs, { createReadStream } from 'fs';
import { Request as ExpressRequest } from 'express';
import fs, { createReadStream, ReadStream } from 'fs';
import { join } from 'path';
import { view } from './application.service';
import { Application } from '../dtos/applications/application.dto';
Expand All @@ -21,6 +21,7 @@ import { PrismaService } from './prisma.service';
import { CsvHeader } from '../types/CsvExportInterface';
import { getExportHeaders } from '../utilities/application-export-helpers';
import { mapTo } from '../utilities/mapTo';
import { zipExport } from '../utilities/zip-export';

view.csv = {
...view.details,
Expand All @@ -43,31 +44,69 @@ export class ApplicationExporterService {
private permissionService: PermissionService,
) {}

// csv export functions
/**
*
* @param queryParams
* @param req
* @param queryParams
* @param isLottery a boolean indicating if the export is a lottery
* @param isSpreadsheet a boolean indicating if the export is a spreadsheet
* @returns a promise containing a streamable file
*/
async csvExport<QueryParams extends ApplicationCsvQueryParams>(
async exporter<QueryParams extends ApplicationCsvQueryParams>(
req: ExpressRequest,
res: Response,
queryParams: QueryParams,
isLottery: boolean,
isSpreadsheet: boolean,
): Promise<StreamableFile> {
const user = mapTo(User, req['user']);
await this.authorizeExport(user, queryParams.id);

let filename: string;
let readStream: ReadStream;
let zipFilename: string;
const now = new Date();
const dateString = dayjs(now).format('YYYY-MM-DD_HH-mm');
if (isLottery) {
readStream = await this.spreadsheetExport(queryParams, user.id, true);
zipFilename = `listing-${queryParams.id}-lottery-${
user.id
}-${now.getTime()}`;
filename = `lottery-${queryParams.id}-${dateString}`;
} else {
if (isSpreadsheet) {
readStream = await this.spreadsheetExport(queryParams, user.id, false);
} else {
readStream = await this.csvExport(queryParams, user.id);
}
zipFilename = `listing-${queryParams.id}-applications-${
user.id
}-${now.getTime()}`;
filename = `applications-${queryParams.id}-${dateString}`;
}

return await zipExport(readStream, zipFilename, filename, isSpreadsheet);
}

// csv export functions
/**
*
* @param queryParams
* @param user_id
* @returns a promise containing a file read stream
*/
async csvExport<QueryParams extends ApplicationCsvQueryParams>(
queryParams: QueryParams,
user_id: string,
): Promise<ReadStream> {
const filename = join(
process.cwd(),
`src/temp/listing-${queryParams.id}-applications-${
user.id
}-${new Date().getTime()}.csv`,
`src/temp/listing-${
queryParams.id
}-applications-${user_id}-${new Date().getTime()}.csv`,
);

await this.createCsv(filename, queryParams);
const file = createReadStream(filename);
return new StreamableFile(file);
return createReadStream(filename);
}

/**
Expand Down Expand Up @@ -319,37 +358,27 @@ export class ApplicationExporterService {
/**
*
* @param queryParams
* @param req
* @returns generates the lottery export file via helper function and returns the streamable file
* @param user
* @param forLottery
* @returns generates the applications or lottery spreadsheet export and returns a promise containing a file read stream
*/
async spreadsheetExport<QueryParams extends ApplicationCsvQueryParams>(
req: ExpressRequest,
res: Response,
queryParams: QueryParams,
user_id: string,
forLottery = true,
): Promise<StreamableFile> {
const user = mapTo(User, req['user']);
await this.authorizeExport(user, queryParams.id);

): Promise<ReadStream> {
const filename = join(
process.cwd(),
`src/temp/${forLottery ? 'lottery-' : ''}listing-${
queryParams.id
}-applications-${user.id}-${new Date().getTime()}.xlsx`,
}-applications-${user_id}-${new Date().getTime()}.xlsx`,
);

const workbook = new Excel.stream.xlsx.WorkbookWriter({
filename,
useSharedStrings: false,
});

const zipFilePath = join(
process.cwd(),
`src/temp/${forLottery ? 'lottery-' : ''}listing-${
queryParams.id
}-applications-${user.id}-${new Date().getTime()}.zip`,
);

await this.createSpreadsheets(
workbook,
{
Expand All @@ -359,27 +388,7 @@ export class ApplicationExporterService {
);

await workbook.commit();
const readStream = createReadStream(filename);

return new Promise((resolve) => {
// Create a writable stream to the zip file
const output = fs.createWriteStream(zipFilePath);
const archive = archiver('zip', {
zlib: { level: 9 },
});
output.on('close', () => {
const zipFile = createReadStream(zipFilePath);
resolve(new StreamableFile(zipFile));
});

archive.pipe(output);
archive.append(readStream, {
name: `${forLottery ? 'lottery-' : ''}${
queryParams.id
}-${new Date().getTime()}.xlsx`,
});
archive.finalize();
});
return createReadStream(filename);
}

/**
Expand Down
39 changes: 39 additions & 0 deletions api/src/utilities/zip-export.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { StreamableFile } from '@nestjs/common';
import archiver from 'archiver';
import fs, { createReadStream, ReadStream } from 'fs';
import { join } from 'path';

/**
*
* @param readStream
* @param zipFilename
* @param filename
* @param isSpreadsheet
* @returns a promise containing a streamable file
*/
export const zipExport = (
readStream: ReadStream,
zipFilename: string,
filename: string,
isSpreadsheet: boolean,
): Promise<StreamableFile> => {
const zipFilePath = join(process.cwd(), `src/temp/${zipFilename}.zip`);

return new Promise((resolve) => {
// Create a writable stream to the zip file
const output = fs.createWriteStream(zipFilePath);
const archive = archiver('zip', {
zlib: { level: 9 },
});
output.on('close', () => {
const zipFile = createReadStream(zipFilePath);
resolve(new StreamableFile(zipFile));
});

archive.pipe(output);
archive.append(readStream, {
name: `${filename}.${isSpreadsheet ? 'xlsx' : 'csv'}`,
});
archive.finalize();
});
};
28 changes: 11 additions & 17 deletions api/test/unit/services/application-exporter.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { PassThrough } from 'stream';
import { Test, TestingModule } from '@nestjs/testing';
import { MultiselectQuestionsApplicationSectionEnum } from '@prisma/client';
import { HttpModule } from '@nestjs/axios';
import { Request as ExpressRequest, Response } from 'express';
import { PrismaService } from '../../../src/services/prisma.service';
import { ApplicationCsvQueryParams } from '../../../src/dtos/applications/application-csv-query-params.dto';
import { User } from '../../../src/dtos/users/user.dto';
Expand Down Expand Up @@ -97,12 +96,11 @@ describe('Testing application export service', () => {
]);

const exportResponse = await service.csvExport(
{ user: requestingUser } as unknown as ExpressRequest,
{} as unknown as Response,
{
id: randomUUID(),
includeDemographics: false,
} as unknown as ApplicationCsvQueryParams,
requestingUser.id,
);

const headerRow =
Expand All @@ -111,7 +109,7 @@ describe('Testing application export service', () => {
'"application 0 firstName","application 0 middleName","application 0 lastName","application 0 birthDay","application 0 birthMonth","application 0 birthYear","application 0 emailaddress","application 0 phoneNumber","application 0 phoneNumberType","additionalPhoneNumber 0","application 0 applicantAddress street","application 0 applicantAddress street2","application 0 applicantAddress city","application 0 applicantAddress state","application 0 applicantAddress zipCode",,,,,,,,,,,,,,,,,,"income 0","per month",,,,"true","true","true","Studio,One Bedroom",,,,,,,,,,,,,,,,,,,';

const mockedStream = new PassThrough();
exportResponse.getStream().pipe(mockedStream);
exportResponse.pipe(mockedStream);

// In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it
const readable = await new Promise((resolve) => {
Expand Down Expand Up @@ -156,12 +154,11 @@ describe('Testing application export service', () => {
]);

const exportResponse = await service.csvExport(
{ user: requestingUser } as unknown as ExpressRequest,
{} as unknown as Response,
{
id: 'test',
includeDemographics: true,
} as unknown as ApplicationCsvQueryParams,
requestingUser.id,
);

const headerRow =
Expand All @@ -170,7 +167,7 @@ describe('Testing application export service', () => {
'"application 0 firstName","application 0 middleName","application 0 lastName","application 0 birthDay","application 0 birthMonth","application 0 birthYear","application 0 emailaddress","application 0 phoneNumber","application 0 phoneNumberType","additionalPhoneNumber 0","application 0 applicantAddress street","application 0 applicantAddress street2","application 0 applicantAddress city","application 0 applicantAddress state","application 0 applicantAddress zipCode",,,,,,,,,,,,,,,,,,"income 0","per month",,,,"true","true","true","Studio,One Bedroom",,,,,,"Indigenous",,,,"Other"';

const mockedStream = new PassThrough();
exportResponse.getStream().pipe(mockedStream);
exportResponse.pipe(mockedStream);
const readable = await new Promise((resolve) => {
mockedStream.on('data', async (d) => {
const value = Buffer.from(d).toString();
Expand Down Expand Up @@ -227,13 +224,12 @@ describe('Testing application export service', () => {
.spyOn({ unitTypeToReadable }, 'unitTypeToReadable')
.mockReturnValue('Studio');
const exportResponse = await service.csvExport(
{ user: requestingUser } as unknown as ExpressRequest,
{} as unknown as Response,
{ id: randomUUID() } as unknown as ApplicationCsvQueryParams,
{ listingId: randomUUID() } as unknown as ApplicationCsvQueryParams,
requestingUser.id,
);

const mockedStream = new PassThrough();
exportResponse.getStream().pipe(mockedStream);
exportResponse.pipe(mockedStream);

// In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it
const readable = await new Promise((resolve) => {
Expand Down Expand Up @@ -290,16 +286,15 @@ describe('Testing application export service', () => {
.spyOn({ unitTypeToReadable }, 'unitTypeToReadable')
.mockReturnValue('Studio');
const exportResponse = await service.csvExport(
{ user: requestingUser } as unknown as ExpressRequest,
{} as unknown as Response,
{
id: randomUUID(),
timeZone: 'America/New_York',
} as unknown as ApplicationCsvQueryParams,
requestingUser.id,
);

const mockedStream = new PassThrough();
exportResponse.getStream().pipe(mockedStream);
exportResponse.pipe(mockedStream);

// In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it
const readable = await new Promise((resolve) => {
Expand Down Expand Up @@ -359,16 +354,15 @@ describe('Testing application export service', () => {
.spyOn({ unitTypeToReadable }, 'unitTypeToReadable')
.mockReturnValue('Studio');
const exportResponse = await service.csvExport(
{ user: requestingUser } as unknown as ExpressRequest,
{} as unknown as Response,
{
listingId: randomUUID(),
timeZone: 'America/New_York',
} as unknown as ApplicationCsvQueryParams,
requestingUser.id,
);

const mockedStream = new PassThrough();
exportResponse.getStream().pipe(mockedStream);
exportResponse.pipe(mockedStream);

// In order to make sure the last expect statements are properly hit we need to wrap in a promise and resolve it
const readable = await new Promise((resolve) => {
Expand Down
Loading

0 comments on commit 6520adf

Please sign in to comment.