From 5a0dc0fcc6894622c77c8ca7c3445ad90cb99e88 Mon Sep 17 00:00:00 2001
From: jer3k <99355997+jer3k@users.noreply.github.com>
Date: Tue, 5 Nov 2024 09:44:03 -0800
Subject: [PATCH 1/3] feat: GEO-1199 Load PowerBi by report name (#839)
---
.../src/components/AnalyticsPage.vue | 60 ++++++++++++-------
backend/src/external/services/powerbi-api.ts | 19 +++++-
.../src/external/services/powerbi-service.ts | 60 ++++++++++++++++++-
.../src/v1/services/analytic-service.spec.ts | 10 ++--
backend/src/v1/services/analytic-service.ts | 8 +--
5 files changed, 121 insertions(+), 36 deletions(-)
diff --git a/admin-frontend/src/components/AnalyticsPage.vue b/admin-frontend/src/components/AnalyticsPage.vue
index 177ac3442..c2304feae 100644
--- a/admin-frontend/src/components/AnalyticsPage.vue
+++ b/admin-frontend/src/components/AnalyticsPage.vue
@@ -1,7 +1,7 @@
<template>
<div v-if="isAnalyticsAvailable" class="w-100 overflow-x-auto">
<div
- v-for="[name, details] in resourceDetails"
+ v-for="[name, details] in powerBiDetailsPerResource"
:key="name"
class="powerbi-container"
>
@@ -36,12 +36,12 @@ type PowerBiDetails = {
const isAnalyticsAvailable =
(window as any).config?.IS_ADMIN_ANALYTICS_AVAILABLE?.toUpperCase() == 'TRUE';
-const resourceDetails = createDefaultPowerBiDetailsMap([
+const powerBiDetailsPerResource = createDefaultPowerBiDetailsMap([
POWERBI_RESOURCE.ANALYTICS,
]);
if (isAnalyticsAvailable) {
- getPowerBiAccessToken(resourceDetails);
+ getPowerBiAccessToken(powerBiDetailsPerResource);
}
/** Create a Map containing the details of the resources. */
@@ -60,31 +60,13 @@ function createDefaultPowerBiDetailsMap(
accessToken: undefined,
tokenType: models.TokenType.Embed,
},
- css: { width: '200px', height: '400px' },
+ css: { width: '1280px', height: '720px' },
// eventHandlersMap - https://learn.microsoft.com/en-us/javascript/api/overview/powerbi/handle-events#report-events
eventHandlersMap: new Map([
[
'loaded', // The loaded event is raised when the report initializes.
() => {
- /** Set the css size of the report to be the size of the maximum page of all pages. */
- const setCssSize = async () => {
- const pages = await resourceDetails.get(name)!.report?.getPages();
- if (pages) {
- const sizes = pages.reduce(
- (prev, current) => ({
- width: Math.max(prev.width, current.defaultSize.width ?? 0),
- height: Math.max(
- prev.height,
- current.defaultSize.height ?? 0,
- ),
- }),
- { width: 0, height: 0 },
- );
- resourceDetails.get(name)!.css.width = sizes.width + 'px';
- resourceDetails.get(name)!.css.height = sizes.height + 'px';
- }
- };
- setCssSize();
+ setCssSize(name);
},
],
]),
@@ -126,6 +108,38 @@ async function getPowerBiAccessToken(
const msToExpiry = Duration.between(now, expiry).minusMinutes(1).toMillis();
setTimeout(getPowerBiAccessToken, msToExpiry);
}
+
+/**
+ * Set the css size of the report to be the size of the maximum page of all pages.
+ * Warning: The navigation pane is not redrawn when increasing the display size which
+ * leaves a gray area where there should be the navigation pane. You can manually refresh
+ * the iframe by right clicking on the gray area and selecting 'refresh frame'.
+ * @param name
+ * @param refresh Reloads the iframe. Loading takes longer, but the navigation pane will be drawn correctly
+ */
+async function setCssSize(name: POWERBI_RESOURCE, refresh: boolean = false) {
+ const details = powerBiDetailsPerResource.get(name)!;
+ if (!details.report) return;
+ const pages = await details.report.getPages();
+ if (pages) {
+ const sizes = pages.reduce(
+ (prev, current) => ({
+ width: Math.max(prev.width, current.defaultSize.width ?? 0),
+ height: Math.max(prev.height, current.defaultSize.height ?? 0),
+ }),
+ { width: 0, height: 0 },
+ );
+
+ if (
+ details.css.width != sizes.width + 'px' ||
+ details.css.height != sizes.height + 'px'
+ ) {
+ details.css.width = sizes.width + 'px';
+ details.css.height = sizes.height + 'px';
+ if (refresh) details.report.iframe.src += ''; // Allegedly, this is how to refresh an iframe
+ }
+ }
+}
</script>
<style lang="scss">
diff --git a/backend/src/external/services/powerbi-api.ts b/backend/src/external/services/powerbi-api.ts
index a91fbeaa8..588acd04a 100644
--- a/backend/src/external/services/powerbi-api.ts
+++ b/backend/src/external/services/powerbi-api.ts
@@ -53,10 +53,21 @@ export class Api {
body,
config,
);
+
+ /** https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-reports */
+ public getReports = (
+ url: Group_Url,
+ config: AxiosRequestConfig,
+ ): Promise<AxiosResponse<Reports>> =>
+ this.api.get<Reports>(
+ `/v1.0/myorg/groups/${url.workspaceId}/reports`,
+ config,
+ );
}
type DashboardInGroup_Url = { workspaceId: string; dashboardId: string };
type ReportInGroup_Url = { workspaceId: string; reportId: string };
+type Group_Url = { workspaceId: string };
/** https://learn.microsoft.com/en-us/rest/api/power-bi/embed-token/dashboards-generate-token-in-group#request-body */
export type GenerateTokenForDashboardInGroup_Body = {
@@ -82,7 +93,6 @@ export type GenerateToken_Body = {
/** https://learn.microsoft.com/en-us/rest/api/power-bi/embed-token/generate-token#embedtoken */
export type EmbedToken = {
- '@odata.context': string;
token: string;
tokenId: string;
expiration: string;
@@ -90,7 +100,6 @@ export type EmbedToken = {
/** https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-report-in-group#report */
export type Report = {
- '@odata.context': string;
id: string;
reportType: string;
name: string;
@@ -104,9 +113,13 @@ export type Report = {
subscriptions: [];
};
+/** https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-reports#reports */
+export type Reports = {
+ value: Report[];
+};
+
/** https://learn.microsoft.com/en-us/rest/api/power-bi/dashboards/get-dashboard-in-group#dashboard */
export type Dashboard = {
- '@odata.context': string;
id: string;
displayName: string;
isReadOnly: boolean;
diff --git a/backend/src/external/services/powerbi-service.ts b/backend/src/external/services/powerbi-service.ts
index 24e6a4412..c8ce5fde8 100644
--- a/backend/src/external/services/powerbi-service.ts
+++ b/backend/src/external/services/powerbi-service.ts
@@ -23,6 +23,7 @@ type EmbedConfig = {
};
export type ReportInWorkspace = { workspaceId: string; reportId: string };
+export type ReportNameInWorkspace = { workspaceId: string; reportName: string };
/**
* Class to authenticate and make use of the PowerBi REST API.
@@ -39,6 +40,63 @@ export class PowerBiService {
this.powerBiApi = new PowerBi.Api(powerBiUrl);
}
+ /**
+ * Get embed params for multiple report in multiple workspace. Search reports by name
+ * https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-report-in-group
+ * @return EmbedConfig object
+ */
+ public async getEmbedParamsForReportsByName(
+ reportNameInWorkspace: ReportNameInWorkspace[],
+ ): Promise<EmbedConfig> {
+ try {
+ const header = await this.getEntraAuthorizationHeader();
+
+ const workspaces = uniq(reportNameInWorkspace.map((x) => x.workspaceId));
+
+ const reportsPerWorkspace: Record<string, PowerBi.Report[]> = {};
+
+ // Get all the reports in each workspace by calling the PowerBI REST API
+ await Promise.all(
+ workspaces.map(async (id) => {
+ const reports = await this.powerBiApi.getReports(
+ { workspaceId: id },
+ {
+ headers: header,
+ },
+ );
+ reportsPerWorkspace[id] = reports.data.value;
+ }),
+ );
+
+ // Limit the found reports to only the ones requested
+ const reports = reportNameInWorkspace.map((res) =>
+ reportsPerWorkspace[res.workspaceId].find(
+ (x) => x.name == res.reportName,
+ ),
+ );
+
+ // Get Embed token multiple resources
+ const embedToken = await this.getEmbedTokenForV2Workspace(
+ uniq(reports.map((report) => report.id)),
+ uniq(reports.map((report) => report.datasetId)),
+ uniq(reportNameInWorkspace.map((res) => res.workspaceId)),
+ );
+
+ // Add report data for embedding
+ const reportDetails: PowerBiResource[] = reports.map((report) => ({
+ id: report.id,
+ name: report.name,
+ embedUrl: report.embedUrl,
+ }));
+
+ return { embedToken: embedToken, resources: reportDetails };
+ } catch (err) {
+ if (err instanceof AxiosError && err?.response?.data)
+ err.message = JSON.stringify(err.response.data);
+ throw err;
+ }
+ }
+
/**
* https://learn.microsoft.com/en-us/rest/api/power-bi/dashboards/get-dashboard-in-group
*/
@@ -77,7 +135,7 @@ export class PowerBiService {
}
/**
- * Get embed params for a single report for a single workspace
+ * Get embed params for multiple reports in multiple workspace
* https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-report-in-group
* @return EmbedConfig object
*/
diff --git a/backend/src/v1/services/analytic-service.spec.ts b/backend/src/v1/services/analytic-service.spec.ts
index dba6d9854..2b6713e27 100644
--- a/backend/src/v1/services/analytic-service.spec.ts
+++ b/backend/src/v1/services/analytic-service.spec.ts
@@ -4,14 +4,14 @@ import {
PowerBiResourceName,
} from './analytic-service';
-const mockGetEmbedParamsForReports = jest.fn();
+const mockgetEmbedParamsForReportsByName = jest.fn();
jest.mock('../../external/services/powerbi-service', () => {
const actual = jest.requireActual('../../external/services/powerbi-service');
return {
...actual,
PowerBiService: jest.fn().mockImplementation(() => {
return {
- getEmbedParamsForReports: mockGetEmbedParamsForReports,
+ getEmbedParamsForReportsByName: mockgetEmbedParamsForReportsByName,
};
}),
};
@@ -40,7 +40,7 @@ describe('getEmbedInfo', () => {
expiry: '2024',
};
- mockGetEmbedParamsForReports.mockResolvedValue({
+ mockgetEmbedParamsForReportsByName.mockResolvedValue({
resources: [
{ id: output.resources[0].id, embedUrl: output.resources[0].embedUrl },
{ id: output.resources[1].id, embedUrl: output.resources[1].embedUrl },
@@ -51,7 +51,7 @@ describe('getEmbedInfo', () => {
PowerBiResourceName.Analytics,
PowerBiResourceName.Analytics,
]);
- expect(mockGetEmbedParamsForReports).toHaveBeenCalledTimes(1);
+ expect(mockgetEmbedParamsForReportsByName).toHaveBeenCalledTimes(1);
expect(json).toMatchObject(output);
});
@@ -62,6 +62,6 @@ describe('getEmbedInfo', () => {
'invalid' as never,
]),
).rejects.toThrow('Invalid resource names');
- expect(mockGetEmbedParamsForReports).not.toHaveBeenCalled();
+ expect(mockgetEmbedParamsForReportsByName).not.toHaveBeenCalled();
});
});
diff --git a/backend/src/v1/services/analytic-service.ts b/backend/src/v1/services/analytic-service.ts
index 59c1cfb8b..33d895b50 100644
--- a/backend/src/v1/services/analytic-service.ts
+++ b/backend/src/v1/services/analytic-service.ts
@@ -1,7 +1,7 @@
import { config } from '../../config';
import {
PowerBiService,
- ReportInWorkspace,
+ ReportNameInWorkspace,
} from '../../external/services/powerbi-service';
// Embed info for Reports, Dashboards, and other resources
@@ -15,10 +15,10 @@ export enum PowerBiResourceName {
Analytics = 'Analytics',
}
-const resourceIds: Record<PowerBiResourceName, ReportInWorkspace> = {
+const resourceIds: Record<PowerBiResourceName, ReportNameInWorkspace> = {
Analytics: {
workspaceId: config.get('powerbi:analytics:workspaceId'),
- reportId: config.get('powerbi:analytics:analyticsId'),
+ reportName: config.get('powerbi:analytics:analyticsId'),
},
};
@@ -46,7 +46,7 @@ export const analyticsService = {
config.get('entra:tenantId'),
);
- const embedParams = await powerBi.getEmbedParamsForReports(
+ const embedParams = await powerBi.getEmbedParamsForReportsByName(
resourceNames.map((name) => resourceIds[name]),
);
From c3c353cc50b27b943e4ca6546cc7e5bf1150bd3d Mon Sep 17 00:00:00 2001
From: jer3k <99355997+jer3k@users.noreply.github.com>
Date: Tue, 5 Nov 2024 13:49:30 -0800
Subject: [PATCH 2/3] feat: GEO-381 Snowplow analytics button (#838)
---
.github/workflows/.deploy.yml | 1 +
admin-frontend/Caddyfile | 3 ++-
admin-frontend/src/App.vue | 4 ++++
admin-frontend/src/components/AnalyticsPage.vue | 14 ++++++++++++++
.../admin-frontend/templates/deployment.yaml | 2 ++
charts/fin-pay-transparency/templates/secret.yaml | 1 +
charts/fin-pay-transparency/values-dev.yaml | 1 +
7 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/.deploy.yml b/.github/workflows/.deploy.yml
index d5cdacb0f..6cc93001d 100644
--- a/.github/workflows/.deploy.yml
+++ b/.github/workflows/.deploy.yml
@@ -168,6 +168,7 @@ jobs:
--set-string crunchy.pgBackRest.s3.accessKey="${{ secrets.S3_ACCESS_KEY }}" \
--set-string crunchy.pgBackRest.s3.secretKey="${{ secrets.S3_SECRET_ACCESS_KEY }}" \
--set-string global.secrets.clamavApiKey="${{ secrets.CLAMAV_API_KEY }}" \
+ --set-string global.secrets.snowplowUrl="${{ secrets.SNOWPLOW_URL }}" \
${{ inputs.params }} \
--timeout "$DEPLOY_TIMEOUT"m ./${{ github.event.repository.name }}-${{ steps.vars.outputs.semver }}.tgz
diff --git a/admin-frontend/Caddyfile b/admin-frontend/Caddyfile
index c83f7f573..6f6390bba 100644
--- a/admin-frontend/Caddyfile
+++ b/admin-frontend/Caddyfile
@@ -20,7 +20,8 @@
}
respond `window.config = {
"IS_ADMIN_DASHBOARD_AVAILABLE":"{$IS_ADMIN_DASHBOARD_AVAILABLE}",
- "IS_ADMIN_ANALYTICS_AVAILABLE":"{$IS_ADMIN_ANALYTICS_AVAILABLE}"
+ "IS_ADMIN_ANALYTICS_AVAILABLE":"{$IS_ADMIN_ANALYTICS_AVAILABLE}",
+ "SNOWPLOW_URL":"{$SNOWPLOW_URL}"
};`
}
root * /app/dist
diff --git a/admin-frontend/src/App.vue b/admin-frontend/src/App.vue
index 3cdd83af1..661c88c5e 100644
--- a/admin-frontend/src/App.vue
+++ b/admin-frontend/src/App.vue
@@ -118,6 +118,10 @@ a:hover {
background-color: transparent !important;
}
+.v-card-title {
+ font-size: 1.35rem !important; //default is 1.25, but BCSans font looks wonky at that size.
+}
+
.theme--light.application {
background: #f1f1f1;
}
diff --git a/admin-frontend/src/components/AnalyticsPage.vue b/admin-frontend/src/components/AnalyticsPage.vue
index c2304feae..a326e23e4 100644
--- a/admin-frontend/src/components/AnalyticsPage.vue
+++ b/admin-frontend/src/components/AnalyticsPage.vue
@@ -1,4 +1,15 @@
<template>
+ <v-btn
+ v-tooltip:bottom-end="'Only authorized users can access this data'"
+ append-icon="mdi-open-in-new"
+ class="ml-auto btn-primary"
+ style="margin-top: -40px"
+ rel="noopener"
+ target="_blank"
+ :href="sanitizeUrl(snowplowUrl)"
+ >Web Traffic Analytics</v-btn
+ >
+
<div v-if="isAnalyticsAvailable" class="w-100 overflow-x-auto">
<div
v-for="[name, details] in powerBiDetailsPerResource"
@@ -25,6 +36,7 @@ import ApiService from '../services/apiService';
import { ZonedDateTime, Duration } from '@js-joda/core';
import { POWERBI_RESOURCE } from '../utils/constant';
import { NotificationService } from '../services/notificationService';
+import { sanitizeUrl } from '@braintree/sanitize-url';
type PowerBiDetails = {
config: IReportEmbedConfiguration;
@@ -33,6 +45,8 @@ type PowerBiDetails = {
eventHandlersMap: Map<string, EventHandler>;
};
+const snowplowUrl = (window as any).config?.SNOWPLOW_URL;
+
const isAnalyticsAvailable =
(window as any).config?.IS_ADMIN_ANALYTICS_AVAILABLE?.toUpperCase() == 'TRUE';
diff --git a/charts/fin-pay-transparency/charts/admin-frontend/templates/deployment.yaml b/charts/fin-pay-transparency/charts/admin-frontend/templates/deployment.yaml
index 82654bcfd..57e2c2c8c 100644
--- a/charts/fin-pay-transparency/charts/admin-frontend/templates/deployment.yaml
+++ b/charts/fin-pay-transparency/charts/admin-frontend/templates/deployment.yaml
@@ -49,6 +49,8 @@ spec:
value: "{{ .Values.env.isAdminDashboardAvailable }}"
- name: IS_ADMIN_ANALYTICS_AVAILABLE
value: "{{ .Values.env.isAdminAnalyticsAvailable }}"
+ - name: SNOWPLOW_URL
+ value: "{{ .Values.global.secrets.snowplowUrl }}"
- name: CLAMAV_API_KEY
valueFrom:
secretKeyRef:
diff --git a/charts/fin-pay-transparency/templates/secret.yaml b/charts/fin-pay-transparency/templates/secret.yaml
index 6de223119..16da11ac8 100644
--- a/charts/fin-pay-transparency/templates/secret.yaml
+++ b/charts/fin-pay-transparency/templates/secret.yaml
@@ -62,4 +62,5 @@ data:
S3_BUCKET_NAME: {{ .Values.global.secrets.s3Bucket | b64enc | quote }}
S3_ENDPOINT: {{ .Values.global.secrets.s3Endpoint | b64enc | quote }}
CLAMAV_API_KEY: {{ .Values.global.secrets.clamavApiKey | b64enc | quote }}
+ SNOWPLOW_URL: {{ .Values.global.secrets.snowplowUrl | b64enc | quote }}
diff --git a/charts/fin-pay-transparency/values-dev.yaml b/charts/fin-pay-transparency/values-dev.yaml
index 3005fde8b..e4c490f3f 100644
--- a/charts/fin-pay-transparency/values-dev.yaml
+++ b/charts/fin-pay-transparency/values-dev.yaml
@@ -158,6 +158,7 @@ admin-frontend:
env:
isAdminDashboardAvailable: true
isAdminAnalyticsAvailable: true
+ isAdminAnalyticsAvailable: true
database:
enabled: false
crunchy:
From 61307fb053e2360ecca5f0a39f822371ee268d31 Mon Sep 17 00:00:00 2001
From: jer3k <99355997+jer3k@users.noreply.github.com>
Date: Wed, 6 Nov 2024 15:14:59 -0800
Subject: [PATCH 3/3] fix: GEO-1007 s3 now reports deleted objects (#841)
---
backend/src/external/services/s3-api.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend/src/external/services/s3-api.ts b/backend/src/external/services/s3-api.ts
index 33b12220c..18a09bcbb 100644
--- a/backend/src/external/services/s3-api.ts
+++ b/backend/src/external/services/s3-api.ts
@@ -124,7 +124,7 @@ export const deleteFiles = async (ids: string[]): Promise<Set<string>> => {
try {
// Get all the files stored under each id
const filesPerId = await Promise.all(
- ids.map((id) => getFileList(s3Client, id)), //TODO: try deleting the folders instead of each individual file. https://stackoverflow.com/a/73367823
+ ids.map((id) => getFileList(s3Client, id)),
);
const idsWithNoFiles = ids.filter(
(id, index) => filesPerId[index].length === 0,
@@ -150,7 +150,7 @@ export const deleteFiles = async (ids: string[]): Promise<Set<string>> => {
// report any errors
responsePerGroup.forEach((r) =>
- r.Errors.forEach((e) => {
+ r.Errors?.forEach((e) => {
if (e.Code == 'NoSuchKey') idsWithNoFiles.push(getIdFromKey(e.Key));
logger.error(e.Message);
}),
@@ -158,7 +158,7 @@ export const deleteFiles = async (ids: string[]): Promise<Set<string>> => {
// Return the id of all successful deleted
const successfulIds = responsePerGroup.flatMap((r) =>
- r.Deleted.reduce((acc, x) => {
+ r.Deleted?.reduce((acc, x) => {
acc.push(getIdFromKey(x.Key));
return acc;
}, [] as string[]),