From 449cc66a9bdc03b0fec04fc7663acb90b9951ea0 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Tue, 6 Feb 2024 21:20:12 -0600 Subject: [PATCH 01/15] Load from static content --- src/public/app/view.html | 14 +- src/public/auth/view.html | 9 +- src/public/static/bulma-tooltip.min.css | 2 + src/public/static/bulma.min.css | 1 + src/public/static/htmx.min.js | 1 + src/public/static/material-symbols.css | 0 src/public/static/sse.js | 355 ++++++++++++++++++++++++ 7 files changed, 368 insertions(+), 14 deletions(-) create mode 100644 src/public/static/bulma-tooltip.min.css create mode 100644 src/public/static/bulma.min.css create mode 100644 src/public/static/htmx.min.js create mode 100644 src/public/static/material-symbols.css create mode 100644 src/public/static/sse.js diff --git a/src/public/app/view.html b/src/public/app/view.html index 08930e61..49958246 100644 --- a/src/public/app/view.html +++ b/src/public/app/view.html @@ -2,15 +2,11 @@ CHT User Management Tool - - - - - + + + + + ")}}function Pr(){var e=te().querySelector('meta[name="htmx-config"]');if(e){return S(e.content)}else{return null}}function kr(){var e=Pr();if(e){Y.config=se(Y.config,e)}}Nr(function(){kr();Ir();var e=te().body;Pt(e);var t=te().querySelectorAll("[hx-trigger='restored'],[data-hx-trigger='restored']");e.addEventListener("htmx:abort",function(e){var t=e.target;var r=ie(t);if(r&&r.xhr){r.xhr.abort()}});var r=window.onpopstate;window.onpopstate=function(e){if(e.state&&e.state.htmx){Gt();ae(t,function(e){fe(e,"htmx:restored",{document:te(),triggerEvent:fe})})}else{if(r){r(e)}}};setTimeout(function(){fe(e,"htmx:load",{});e=null},0)});return Y}()}); \ No newline at end of file diff --git a/src/public/static/material-symbols.css b/src/public/static/material-symbols.css new file mode 100644 index 00000000..e69de29b diff --git a/src/public/static/sse.js b/src/public/static/sse.js new file mode 100644 index 00000000..943d80ac --- /dev/null +++ b/src/public/static/sse.js @@ -0,0 +1,355 @@ +/* +Server Sent Events Extension +============================ +This extension adds support for Server Sent Events to htmx. See /www/extensions/sse.md for usage instructions. + +*/ + +(function() { + + /** @type {import("../htmx").HtmxInternalApi} */ + var api; + + htmx.defineExtension("sse", { + + /** + * Init saves the provided reference to the internal HTMX API. + * + * @param {import("../htmx").HtmxInternalApi} api + * @returns void + */ + init: function(apiRef) { + // store a reference to the internal API. + api = apiRef; + + // set a function in the public API for creating new EventSource objects + if (htmx.createEventSource == undefined) { + htmx.createEventSource = createEventSource; + } + }, + + /** + * onEvent handles all events passed to this extension. + * + * @param {string} name + * @param {Event} evt + * @returns void + */ + onEvent: function(name, evt) { + + switch (name) { + + case "htmx:beforeCleanupElement": + var internalData = api.getInternalData(evt.target) + // Try to remove remove an EventSource when elements are removed + if (internalData.sseEventSource) { + internalData.sseEventSource.close(); + } + + return; + + // Try to create EventSources when elements are processed + case "htmx:afterProcessNode": + ensureEventSourceOnElement(evt.target); + registerSSE(evt.target); + } + } + }); + + /////////////////////////////////////////////// + // HELPER FUNCTIONS + /////////////////////////////////////////////// + + + /** + * createEventSource is the default method for creating new EventSource objects. + * it is hoisted into htmx.config.createEventSource to be overridden by the user, if needed. + * + * @param {string} url + * @returns EventSource + */ + function createEventSource(url) { + return new EventSource(url, { withCredentials: true }); + } + + function splitOnWhitespace(trigger) { + return trigger.trim().split(/\s+/); + } + + function getLegacySSEURL(elt) { + var legacySSEValue = api.getAttributeValue(elt, "hx-sse"); + if (legacySSEValue) { + var values = splitOnWhitespace(legacySSEValue); + for (var i = 0; i < values.length; i++) { + var value = values[i].split(/:(.+)/); + if (value[0] === "connect") { + return value[1]; + } + } + } + } + + function getLegacySSESwaps(elt) { + var legacySSEValue = api.getAttributeValue(elt, "hx-sse"); + var returnArr = []; + if (legacySSEValue != null) { + var values = splitOnWhitespace(legacySSEValue); + for (var i = 0; i < values.length; i++) { + var value = values[i].split(/:(.+)/); + if (value[0] === "swap") { + returnArr.push(value[1]); + } + } + } + return returnArr; + } + + /** + * registerSSE looks for attributes that can contain sse events, right + * now hx-trigger and sse-swap and adds listeners based on these attributes too + * the closest event source + * + * @param {HTMLElement} elt + */ + function registerSSE(elt) { + // Find closest existing event source + var sourceElement = api.getClosestMatch(elt, hasEventSource); + if (sourceElement == null) { + // api.triggerErrorEvent(elt, "htmx:noSSESourceError") + return null; // no eventsource in parentage, orphaned element + } + + // Set internalData and source + var internalData = api.getInternalData(sourceElement); + var source = internalData.sseEventSource; + + // Add message handlers for every `sse-swap` attribute + queryAttributeOnThisOrChildren(elt, "sse-swap").forEach(function(child) { + + var sseSwapAttr = api.getAttributeValue(child, "sse-swap"); + if (sseSwapAttr) { + var sseEventNames = sseSwapAttr.split(","); + } else { + var sseEventNames = getLegacySSESwaps(child); + } + + for (var i = 0; i < sseEventNames.length; i++) { + var sseEventName = sseEventNames[i].trim(); + var listener = function(event) { + + // If the source is missing then close SSE + if (maybeCloseSSESource(sourceElement)) { + return; + } + + // If the body no longer contains the element, remove the listener + if (!api.bodyContains(child)) { + source.removeEventListener(sseEventName, listener); + } + + // swap the response into the DOM and trigger a notification + swap(child, event.data); + api.triggerEvent(elt, "htmx:sseMessage", event); + }; + + // Register the new listener + api.getInternalData(child).sseEventListener = listener; + source.addEventListener(sseEventName, listener); + } + }); + + // Add message handlers for every `hx-trigger="sse:*"` attribute + queryAttributeOnThisOrChildren(elt, "hx-trigger").forEach(function(child) { + + var sseEventName = api.getAttributeValue(child, "hx-trigger"); + if (sseEventName == null) { + return; + } + + // Only process hx-triggers for events with the "sse:" prefix + if (sseEventName.slice(0, 4) != "sse:") { + return; + } + + // remove the sse: prefix from here on out + sseEventName = sseEventName.substr(4); + + var listener = function() { + if (maybeCloseSSESource(sourceElement)) { + return + } + + if (!api.bodyContains(child)) { + source.removeEventListener(sseEventName, listener); + } + } + }); + } + + /** + * ensureEventSourceOnElement creates a new EventSource connection on the provided element. + * If a usable EventSource already exists, then it is returned. If not, then a new EventSource + * is created and stored in the element's internalData. + * @param {HTMLElement} elt + * @param {number} retryCount + * @returns {EventSource | null} + */ + function ensureEventSourceOnElement(elt, retryCount) { + + if (elt == null) { + return null; + } + + // handle extension source creation attribute + queryAttributeOnThisOrChildren(elt, "sse-connect").forEach(function(child) { + var sseURL = api.getAttributeValue(child, "sse-connect"); + if (sseURL == null) { + return; + } + + ensureEventSource(child, sseURL, retryCount); + }); + + // handle legacy sse, remove for HTMX2 + queryAttributeOnThisOrChildren(elt, "hx-sse").forEach(function(child) { + var sseURL = getLegacySSEURL(child); + if (sseURL == null) { + return; + } + + ensureEventSource(child, sseURL, retryCount); + }); + + } + + function ensureEventSource(elt, url, retryCount) { + var source = htmx.createEventSource(url); + + source.onerror = function(err) { + + // Log an error event + api.triggerErrorEvent(elt, "htmx:sseError", { error: err, source: source }); + + // If parent no longer exists in the document, then clean up this EventSource + if (maybeCloseSSESource(elt)) { + return; + } + + // Otherwise, try to reconnect the EventSource + if (source.readyState === EventSource.CLOSED) { + retryCount = retryCount || 0; + var timeout = Math.random() * (2 ^ retryCount) * 500; + window.setTimeout(function() { + ensureEventSourceOnElement(elt, Math.min(7, retryCount + 1)); + }, timeout); + } + }; + + source.onopen = function(evt) { + api.triggerEvent(elt, "htmx:sseOpen", { source: source }); + } + + api.getInternalData(elt).sseEventSource = source; + } + + /** + * maybeCloseSSESource confirms that the parent element still exists. + * If not, then any associated SSE source is closed and the function returns true. + * + * @param {HTMLElement} elt + * @returns boolean + */ + function maybeCloseSSESource(elt) { + if (!api.bodyContains(elt)) { + var source = api.getInternalData(elt).sseEventSource; + if (source != undefined) { + source.close(); + // source = null + return true; + } + } + return false; + } + + /** + * queryAttributeOnThisOrChildren returns all nodes that contain the requested attributeName, INCLUDING THE PROVIDED ROOT ELEMENT. + * + * @param {HTMLElement} elt + * @param {string} attributeName + */ + function queryAttributeOnThisOrChildren(elt, attributeName) { + + var result = []; + + // If the parent element also contains the requested attribute, then add it to the results too. + if (api.hasAttribute(elt, attributeName)) { + result.push(elt); + } + + // Search all child nodes that match the requested attribute + elt.querySelectorAll("[" + attributeName + "], [data-" + attributeName + "]").forEach(function(node) { + result.push(node); + }); + + return result; + } + + /** + * @param {HTMLElement} elt + * @param {string} content + */ + function swap(elt, content) { + + api.withExtensions(elt, function(extension) { + content = extension.transformResponse(content, null, elt); + }); + + var swapSpec = api.getSwapSpecification(elt); + var target = api.getTarget(elt); + var settleInfo = api.makeSettleInfo(elt); + + api.selectAndSwap(swapSpec.swapStyle, target, elt, content, settleInfo); + + settleInfo.elts.forEach(function(elt) { + if (elt.classList) { + elt.classList.add(htmx.config.settlingClass); + } + api.triggerEvent(elt, 'htmx:beforeSettle'); + }); + + // Handle settle tasks (with delay if requested) + if (swapSpec.settleDelay > 0) { + setTimeout(doSettle(settleInfo), swapSpec.settleDelay); + } else { + doSettle(settleInfo)(); + } + } + + /** + * doSettle mirrors much of the functionality in htmx that + * settles elements after their content has been swapped. + * TODO: this should be published by htmx, and not duplicated here + * @param {import("../htmx").HtmxSettleInfo} settleInfo + * @returns () => void + */ + function doSettle(settleInfo) { + + return function() { + settleInfo.tasks.forEach(function(task) { + task.call(); + }); + + settleInfo.elts.forEach(function(elt) { + if (elt.classList) { + elt.classList.remove(htmx.config.settlingClass); + } + api.triggerEvent(elt, 'htmx:afterSettle'); + }); + } + } + + function hasEventSource(node) { + return api.getInternalData(node).sseEventSource != null; + } + +})(); From 16532a0fc22a3ca6ae0a11931af744eababe2637 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Tue, 6 Feb 2024 22:14:34 -0600 Subject: [PATCH 02/15] Working solution based around modal --- .vscode/launch.json | 18 +++ src/public/app/fragment_home.html | 15 +- src/public/app/nav.html | 42 +++++- src/public/place/list.html | 218 ++++++++++++++------------- src/public/place/list_event.html | 2 + src/public/place/progress_modal.html | 43 ++++++ src/routes/add-place.ts | 7 +- src/routes/app.ts | 26 ++-- src/routes/authentication.ts | 11 ++ src/routes/events.ts | 40 +++-- src/services/place.ts | 6 +- src/services/progress-model.ts | 21 +++ src/services/session-cache.ts | 2 - src/services/upload-manager.ts | 35 +++-- test/services/progress-model.spec.ts | 43 ++++++ 15 files changed, 355 insertions(+), 174 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 src/public/place/list_event.html create mode 100644 src/public/place/progress_modal.html create mode 100644 src/services/progress-model.ts create mode 100644 test/services/progress-model.spec.ts diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..2cc188ca --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/dist/index.js", + "envFile": "${workspaceFolder}/.env" + } + ] +} \ No newline at end of file diff --git a/src/public/app/fragment_home.html b/src/public/app/fragment_home.html index 85cae48b..6dbdbb74 100644 --- a/src/public/app/fragment_home.html +++ b/src/public/app/fragment_home.html @@ -1,8 +1,13 @@
-
-
- {% include "place/list.html" %} -
+
+ {% include "place/list.html" %} + {% include "place/progress_modal.html" %}
\ No newline at end of file diff --git a/src/public/app/nav.html b/src/public/app/nav.html index d05ef2cb..38a031ab 100644 --- a/src/public/app/nav.html +++ b/src/public/app/nav.html @@ -70,7 +70,7 @@
- + group_add Upload @@ -99,5 +99,45 @@ $target.classList.toggle('is-active'); }); }); + + function closeAllModals() { + (document.querySelectorAll('.modal') || []).forEach(($modal) => { + closeModal($modal); + }); + } + + function openModal($el) { + $el.classList.add('is-active'); + } + + function closeModal($el) { + $el.classList.remove('is-active'); + } + + // Add a click event on buttons to open a specific modal + (document.querySelectorAll('.modal-trigger') || []).forEach(($trigger) => { + const modal = $trigger.dataset.target; + const $target = document.getElementById(modal); + + $trigger.addEventListener('click', () => { + openModal($target); + }); + }); + + // Add a click event on various child elements to close the parent modal + (document.querySelectorAll('.modal-background, .modal-action, .modal-close, .modal-card-head .delete, .modal-card-foot .button') || []).forEach(($close) => { + const $target = $close.closest('.modal'); + + $close.addEventListener('click', () => { + closeModal($target); + }); + }); + + // Add a keyboard event to close all modals + document.addEventListener('keydown', (event) => { + if (event.key === "Escape") { + closeAllModals(); + } + }); }); diff --git a/src/public/place/list.html b/src/public/place/list.html index a8694946..199520ab 100644 --- a/src/public/place/list.html +++ b/src/public/place/list.html @@ -1,121 +1,123 @@ -{% for contactType in contactTypes %} -
-

{{contactType.friendly}}

- {% if contactType.places.length > 0 %} - - - - {% for hierarchy in contactType.hierarchy %} - - {% endfor %} - {% for property in contactType.place_properties %} - - {% endfor %} - {% for contact_property in contactType.contact_properties %} - - {% endfor %} - - - - - - {% for place in contactType.places %} - - {% for hierarchy in contactType.hierarchy %} - {% capture propertyName %}hierarchy_{{ hierarchy.property_name }}{% endcapture %} - {% - include "components/list_cell.html" - propertyName=propertyName - property=hierarchy - values=place.hierarchyProperties - linkTo=place.resolvedHierarchy[hierarchy.level] - %} - {% endfor %} +
+ {% for contactType in contactTypes %} +
+

{{contactType.friendly}}

+ {% if contactType.places.length > 0 %} +
{{ hierarchy.friendly_name }}{{property.friendly_name}}{{contact_property.friendly_name}}
+ + + {% for hierarchy in contactType.hierarchy %} + + {% endfor %} + {% for property in contactType.place_properties %} + + {% endfor %} + {% for contact_property in contactType.contact_properties %} + + {% endfor %} + + + + + + {% for place in contactType.places %} + + {% for hierarchy in contactType.hierarchy %} + {% capture propertyName %}hierarchy_{{ hierarchy.property_name }}{% endcapture %} + {% + include "components/list_cell.html" + propertyName=propertyName + property=hierarchy + values=place.hierarchyProperties + linkTo=place.resolvedHierarchy[hierarchy.level] + %} + {% endfor %} - {% for property in contactType.place_properties %} - {% capture propertyName %}place_{{ property.property_name }}{% endcapture %} + {% for property in contactType.place_properties %} + {% capture propertyName %}place_{{ property.property_name }}{% endcapture %} - {% - include "components/list_cell.html" - propertyName=propertyName - property=property - values=place.properties - %} - {% endfor %} - - {% for contact_property in contactType.contact_properties %} - {% capture propertyName %}contact_{{ contact_property.property_name }}{% endcapture %} - {% - include "components/list_cell.html" - propertyName=propertyName - property=contact_property - values=place.contact.properties - %} - {% endfor %} - - + {% + include "components/list_cell.html" + propertyName=propertyName + property=property + values=place.properties + %} + {% endfor %} - + + - - {% endfor%} - -
{{ hierarchy.friendly_name }}{{property.friendly_name}}{{contact_property.friendly_name}}
- {% capture tag_text %}{% if place.validationErrors == empty %}{{ place.state }}{% else %}INVALID{% endif %}{% endcapture %} - {% capture tag_class %} - {% if place.state == 'failure' or place.validationErrors != empty %}is-danger{% endif %} - {% if place.uploadError %}has-tooltip-arrow has-tooltipl-multiline{% endif %} - {% endcapture %} - - {{ tag_text }} - - - {% if place.creationDetails.password %} - {% capture explanation %}Username: {{ place.creationDetails.username }} -Password: {{ place.creationDetails.password }}{% endcapture %} + {% for contact_property in contactType.contact_properties %} + {% capture propertyName %}contact_{{ contact_property.property_name }}{% endcapture %} + {% + include "components/list_cell.html" + propertyName=propertyName + property=contact_property + values=place.contact.properties + %} + {% endfor %} + + + {% capture tag_text %}{% if place.validationErrors == empty %}{{ place.state }}{% else %}INVALID{% endif %}{% endcapture %} + {% capture tag_class %} + {% if place.state == 'failure' or place.validationErrors != empty %}is-danger{% endif %} + {% if place.uploadError %}has-tooltip-arrow has-tooltipl-multiline{% endif %} + {% endcapture %} - key + {{ tag_text }} - {% else %} - - edit - - - refresh - - {% if place.validationErrors == empty %} - - upload + + {% if place.creationDetails.password %} + {% capture explanation %}Username: {{ place.creationDetails.username }} + Password: {{ place.creationDetails.password }}{% endcapture %} + + key + + {% else %} + + edit + + + refresh + {% if place.validationErrors == empty %} + + upload + + {%endif%} {%endif%} - {%endif%} - delete -
- {% endif %} + delete + + + {% endfor%} + + + {% endif %} -
-
- - add Add {{contactType.friendly}} - -
- Create New - Replace Existing - Upload from CSV - Move +
+
+ + add Add {{contactType.friendly}} + +
+ Create New + Replace Existing + Upload from CSV + Move +
-
-{% endfor %} \ No newline at end of file + {% endfor %} +
\ No newline at end of file diff --git a/src/public/place/list_event.html b/src/public/place/list_event.html new file mode 100644 index 00000000..eb24e2a1 --- /dev/null +++ b/src/public/place/list_event.html @@ -0,0 +1,2 @@ +{% include "place/list.html" %} +{% include "place/progress_modal.html" %} \ No newline at end of file diff --git a/src/public/place/progress_modal.html b/src/public/place/progress_modal.html new file mode 100644 index 00000000..d99b6d50 --- /dev/null +++ b/src/public/place/progress_modal.html @@ -0,0 +1,43 @@ + \ No newline at end of file diff --git a/src/routes/add-place.ts b/src/routes/add-place.ts index 35e5e634..912f72e2 100644 --- a/src/routes/add-place.ts +++ b/src/routes/add-place.ts @@ -122,7 +122,7 @@ export default async function addPlace(fastify: FastifyInstance) { await RemotePlaceResolver.resolveOne(place, sessionCache, chtApi, { fuzz: true }); place.validate(); - fastify.uploadManager.refresh(req.sessionCache); + fastify.uploadManager.triggerRefresh(place.id); }); fastify.post('/place/upload/:id', async (req) => { @@ -135,14 +135,13 @@ export default async function addPlace(fastify: FastifyInstance) { const chtApi = new ChtApi(req.chtSession); const uploadManager: UploadManager = fastify.uploadManager; - await uploadManager.doUpload([place], chtApi); - fastify.uploadManager.refresh(req.sessionCache); + uploadManager.doUpload([place], chtApi); }); fastify.post('/place/remove/:id', async (req) => { const { id } = req.params as any; const sessionCache: SessionCache = req.sessionCache; sessionCache.removePlace(id); - fastify.uploadManager.refresh(req.sessionCache); + fastify.uploadManager.triggerRefresh(id); }); } diff --git a/src/routes/app.ts b/src/routes/app.ts index 8b15409a..d4d722cd 100644 --- a/src/routes/app.ts +++ b/src/routes/app.ts @@ -1,11 +1,12 @@ import { FastifyInstance } from 'fastify'; + +import { ChtApi } from '../lib/cht-api'; import { Config } from '../config'; +import ProgressModel from '../services/progress-model'; +import RemotePlaceCache from '../lib/remote-place-cache'; +import RemotePlaceResolver from '../lib/remote-place-resolver'; import SessionCache from '../services/session-cache'; import { UploadManager } from '../services/upload-manager'; -import { PlaceUploadState } from '../services/place'; -import { ChtApi } from '../lib/cht-api'; -import RemotePlaceResolver from '../lib/remote-place-resolver'; -import RemotePlaceCache from '../lib/remote-place-cache'; export default async function sessionCache(fastify: FastifyInstance) { fastify.get('/', async (req, resp) => { @@ -18,8 +19,6 @@ export default async function sessionCache(fastify: FastifyInstance) { const contactType = Config.getContactType(placeTypeName); const sessionCache: SessionCache = req.sessionCache; - const failed = sessionCache.getPlaces({ state: PlaceUploadState.FAILURE }); - const scheduledJobs = sessionCache.getPlaces({ state: PlaceUploadState.SCHEDULED }); const placeData = contactTypes.map((item) => { return { ...item, @@ -30,15 +29,12 @@ export default async function sessionCache(fastify: FastifyInstance) { const tmplData = { view: 'list', + session: req.chtSession, logo: Config.getLogoBase64(), + op, contactType, contactTypes: placeData, - sessionState: sessionCache.state, - hasFailedJobs: failed.length > 0, - failedJobCount: failed.length, - scheduledJobCount: scheduledJobs.length, - session: req.chtSession, - op, + progress: new ProgressModel(sessionCache), }; return resp.view('src/public/app/view.html', tmplData); @@ -47,7 +43,7 @@ export default async function sessionCache(fastify: FastifyInstance) { fastify.post('/app/remove-all', async (req) => { const sessionCache: SessionCache = req.sessionCache; sessionCache.removeAll(); - fastify.uploadManager.refresh(req.sessionCache); + fastify.uploadManager.triggerRefresh(undefined); }); fastify.post('/app/refresh-all', async (req) => { @@ -60,7 +56,7 @@ export default async function sessionCache(fastify: FastifyInstance) { await RemotePlaceResolver.resolve(places, sessionCache, chtApi, { fuzz: true }); places.forEach(p => p.validate()); - fastify.uploadManager.refresh(req.sessionCache); + fastify.uploadManager.triggerRefresh(undefined); }); // initiates place creation via the job manager @@ -68,9 +64,7 @@ export default async function sessionCache(fastify: FastifyInstance) { const uploadManager: UploadManager = fastify.uploadManager; const sessionCache: SessionCache = req.sessionCache; - uploadManager.eventedSessionStateChange(sessionCache, 'in_progress'); const chtApi = new ChtApi(req.chtSession); uploadManager.doUpload(sessionCache.getPlaces(), chtApi); - uploadManager.eventedSessionStateChange(sessionCache, 'done'); }); } diff --git a/src/routes/authentication.ts b/src/routes/authentication.ts index 20d85c28..f6920584 100644 --- a/src/routes/authentication.ts +++ b/src/routes/authentication.ts @@ -1,4 +1,6 @@ import { FastifyInstance, FastifyRequest } from 'fastify'; +import fs from 'fs'; +import path from 'path'; import Auth from '../lib/authentication'; import { ChtApi } from '../lib/cht-api'; @@ -26,6 +28,15 @@ export default async function authentication(fastify: FastifyInstance) { return resp.redirect('/login'); }); + fastify.get('/static/:filename', unauthenticatedOptions, async req => { + const params: any = req.params; + const filename = params.filename; + + const fullPath = path.resolve(`./src/public/static/${filename}`); + const content = fs.readFileSync(fullPath); + return content; + }); + fastify.post('/authenticate', unauthenticatedOptions, async (req, resp) => { const data: any = req.body; const { username, password, domain } = data; diff --git a/src/routes/events.ts b/src/routes/events.ts index 1d8215be..482c0131 100644 --- a/src/routes/events.ts +++ b/src/routes/events.ts @@ -1,23 +1,23 @@ import { FastifyInstance } from 'fastify'; import { Config } from '../config'; -import { PlaceUploadState } from '../services/place'; -import SessionCache, { SessionCacheUploadState } from '../services/session-cache'; +import SessionCache from '../services/session-cache'; +import ProgressModel from '../services/progress-model'; export default async function events(fastify: FastifyInstance) { - fastify.get('/events/places_list', async (req, resp) => { + fastify.get('/events/places/all', async (req, resp) => { const sessionCache: SessionCache = req.sessionCache; const contactTypes = Config.contactTypes(); - const placeData = contactTypes.map((item) => { - return { - ...item, - places: sessionCache.getPlaces({ type: item.name }), - hierarchy: Config.getHierarchyWithReplacement(item, 'desc'), - }; - }); - return resp.view('src/public/place/list.html', { + const placeData = contactTypes.map(item => ({ + ...item, + places: sessionCache.getPlaces({ type: item.name }), + hierarchy: Config.getHierarchyWithReplacement(item, 'desc'), + })); + + return resp.view('src/public/place/list_event.html', { contactTypes: placeData, session: req.chtSession, + progress: new ProgressModel(sessionCache), }); }); @@ -25,19 +25,17 @@ export default async function events(fastify: FastifyInstance) { const { uploadManager } = fastify; resp.hijack(); - const placesChangeListener = (arg: PlaceUploadState) => { - resp.sse({ event: 'places_state_change', data: arg }); - }; - uploadManager.on('places_state_change', placesChangeListener); - - const sessionStateListener = (arg: SessionCacheUploadState) => { - resp.sse({ event: 'session_state_change', data: arg }); + const placesChangeListener = (arg: string = '*') => { + console.log('place_state_change', arguments); + resp.sse({ event: 'place_state_change', data: arg }); }; - uploadManager.on('session_state_change', sessionStateListener); + + uploadManager.on('refresh_table', placesChangeListener); + uploadManager.on('refresh_table_row', placesChangeListener); req.socket.on('close', () => { - uploadManager.removeListener('places_state_change', placesChangeListener); - uploadManager.removeListener('session_state_change', sessionStateListener); + uploadManager.removeListener('refresh_table', placesChangeListener); + uploadManager.removeListener('refresh_table_row', placesChangeListener); }); }); } diff --git a/src/services/place.ts b/src/services/place.ts index 998c281e..e645e3f8 100644 --- a/src/services/place.ts +++ b/src/services/place.ts @@ -22,7 +22,7 @@ export enum PlaceUploadState { FAILURE = 'failure', PENDING = 'pending', SCHEDULED = 'scheduled', - IN_PROGESS = 'in_progress', + IN_PROGRESS = 'in_progress', } const PLACE_PREFIX = 'place_'; @@ -210,6 +210,10 @@ export default class Place { return username; } + public get hasValidationErrors() : boolean { + return Object.keys(this.validationErrors as any).length > 0; + } + public get isDependant() : boolean { return !!this.resolvedHierarchy.find(hierarchy => hierarchy?.type === 'local'); } diff --git a/src/services/progress-model.ts b/src/services/progress-model.ts new file mode 100644 index 00000000..d3858c9e --- /dev/null +++ b/src/services/progress-model.ts @@ -0,0 +1,21 @@ +import { PlaceUploadState } from './place'; +import SessionCache from './session-cache'; + +export default class ProgressModel { + public completeCount: number; + public pendingCount: number; + public percent: string; + public successCount: number; + public remainingCount: number; + public failureCount: number; + + constructor(sessionCache: SessionCache) { + this.successCount = sessionCache.getPlaces({ state: PlaceUploadState.SUCCESS }).length; + this.failureCount = sessionCache.getPlaces({ state: PlaceUploadState.FAILURE }).length; + this.completeCount = this.successCount + this.failureCount; + this.pendingCount = sessionCache.getPlaces().filter(r => !r.hasValidationErrors).length; + const percentage = this.pendingCount > 0 ? this.completeCount / this.pendingCount : 0; + this.remainingCount = this.pendingCount - this.completeCount; + this.percent = Math.round(percentage * 100.0) + '%' + } +} diff --git a/src/services/session-cache.ts b/src/services/session-cache.ts index 12a59cf2..f32a8ac6 100644 --- a/src/services/session-cache.ts +++ b/src/services/session-cache.ts @@ -5,8 +5,6 @@ export type SessionCacheUploadState = 'in_progress' | 'done' | 'pending'; export default class SessionCache { private static caches: Map = new Map(); - - public state: SessionCacheUploadState = 'pending'; private places: { [key: string]: Place } = {}; private constructor() {} diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index bf372b62..6bc13c30 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -3,7 +3,6 @@ import { ChtApi, PlacePayload } from '../lib/cht-api'; import Place, { PlaceUploadState } from './place'; import { UserPayload } from './user-payload'; -import SessionCache, { SessionCacheUploadState } from './session-cache'; import { UploadReplacementPlace } from './upload.replacement'; import { UploadNewPlace } from './upload.new'; import { Config } from '../config'; @@ -19,7 +18,7 @@ export interface Uploader { export class UploadManager extends EventEmitter { doUpload = async (places: Place[], chtApi: ChtApi) => { - const placesNeedingUpload = places.filter(p => !p.isCreated && Object.keys(p.validationErrors as any).length === 0); + const placesNeedingUpload = places.filter(p => !p.isCreated && !p.hasValidationErrors); this.eventedPlaceStateChange(placesNeedingUpload, PlaceUploadState.SCHEDULED); const independants = placesNeedingUpload.filter(p => !p.isDependant); @@ -37,7 +36,7 @@ export class UploadManager extends EventEmitter { } private async uploadSinglePlace(place: Place, chtApi: ChtApi) { - this.eventedPlaceStateChange(place, PlaceUploadState.IN_PROGESS); + this.eventedPlaceStateChange(place, PlaceUploadState.IN_PROGRESS); try { const uploader: Uploader = place.hierarchyProperties.replacement ? new UploadReplacementPlace(chtApi) : new UploadNewPlace(chtApi); @@ -80,24 +79,28 @@ export class UploadManager extends EventEmitter { } } - public refresh(sessionCache: SessionCache) { - this.emit('session_state_change', sessionCache.state); - this.emit('places_state_change', PlaceUploadState.PENDING); - } - - public eventedSessionStateChange(sessionCache: SessionCache, state: SessionCacheUploadState) { - sessionCache.state = state; - this.emit('session_state_change', state); + public triggerRefresh(place_id: string | undefined) { + if (place_id) { + this.emit('refresh_table_row', place_id); + } else { + this.emit('refresh_table'); + } } private eventedPlaceStateChange = (subject: Place | Place[], state: PlaceUploadState) => { - if (Array.isArray(subject)) { - subject.forEach(place => place.state = state); - } else { - subject.state = state; + if (!Array.isArray(subject)) { + subject = [subject]; + } + + if (subject.length > 1) { + this.triggerRefresh(undefined); + return; } - this.emit('places_state_change', state); + subject.forEach(place => { + place.state = state; + this.triggerRefresh(place.id); + }); }; } diff --git a/test/services/progress-model.spec.ts b/test/services/progress-model.spec.ts new file mode 100644 index 00000000..7f24ab15 --- /dev/null +++ b/test/services/progress-model.spec.ts @@ -0,0 +1,43 @@ +import { expect } from 'chai'; + +import { PlaceUploadState } from '../../src/services/place'; +import ProgressModel from '../../src/services/progress-model'; +import SessionCache from '../../src/services/session-cache'; + +describe('services/progress-model.ts', () => { + it('one place with each state', () => { + const sessionCache = new SessionCache(); + sessionCache.savePlaces( + { id: 'fail', state: PlaceUploadState.FAILURE }, + { id: 'in-progress', state: PlaceUploadState.IN_PROGRESS }, + { id: 'pending', state: PlaceUploadState.PENDING }, + { id: 'scheduled', state: PlaceUploadState.SCHEDULED }, + { id: 'success', state: PlaceUploadState.SUCCESS }, + { id: 'validation-error', state: PlaceUploadState.PENDING, hasValidationErrors: true }, + ); + + const progressModel = new ProgressModel(sessionCache); + expect(progressModel).to.deep.eq({ + failureCount: 1, + successCount: 1, + remainingCount: 3, + completeCount: 2, + pendingCount: 5, + percent: '40%', + }); + }); + + it('empty session', () => { + const sessionCache = new SessionCache(); + const progressModel = new ProgressModel(sessionCache); + expect(progressModel).to.deep.eq({ + failureCount: 0, + successCount: 0, + completeCount: 0, + remainingCount: 0, + pendingCount: 0, + percent: '0%', + }); + }); +}); + From 1792e5c2c67faeac52583dcf5ce087c3a909e03c Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Thu, 8 Feb 2024 09:48:47 +0700 Subject: [PATCH 03/15] Change from modal to top-of-page directive --- src/public/app/fragment_home.html | 2 +- src/public/app/nav.html | 2 +- src/public/components/search_input.html | 1 - src/public/place/directive.html | 13 ++++++ src/public/place/directive_1_get_started.html | 22 ++++++++++ .../place/directive_2_prompt_upload.html | 15 +++++++ src/public/place/directive_3_in_progress.html | 17 ++++++++ src/public/place/directive_4_prompt_save.html | 16 +++++++ src/public/place/list.html | 20 +-------- src/public/place/list_event.html | 4 +- src/public/place/progress_modal.html | 43 ------------------- src/services/progress-model.ts | 2 + 12 files changed, 91 insertions(+), 66 deletions(-) create mode 100644 src/public/place/directive.html create mode 100644 src/public/place/directive_1_get_started.html create mode 100644 src/public/place/directive_2_prompt_upload.html create mode 100644 src/public/place/directive_3_in_progress.html create mode 100644 src/public/place/directive_4_prompt_save.html delete mode 100644 src/public/place/progress_modal.html diff --git a/src/public/app/fragment_home.html b/src/public/app/fragment_home.html index 6dbdbb74..4af292f8 100644 --- a/src/public/app/fragment_home.html +++ b/src/public/app/fragment_home.html @@ -7,7 +7,7 @@ hx-swap="none" hx-trigger="sse:place_state_change" > + {% include "place/directive.html" %} {% include "place/list.html" %} - {% include "place/progress_modal.html" %}
\ No newline at end of file diff --git a/src/public/app/nav.html b/src/public/app/nav.html index 38a031ab..28178a5c 100644 --- a/src/public/app/nav.html +++ b/src/public/app/nav.html @@ -66,7 +66,7 @@
cloud - Connected to {{ session.authInfo.friendly}} as {{ session.username }} + Connected to {{ session.authInfo.friendly }} as {{ session.username }}
diff --git a/src/public/components/search_input.html b/src/public/components/search_input.html index 839edbb7..2ac5a7e7 100644 --- a/src/public/components/search_input.html +++ b/src/public/components/search_input.html @@ -17,7 +17,6 @@ hx-target="#search_results_{{ property_name }}" hx-swap="innerHTML" {% if include.required %} required {% endif %} {% if include.data[property_name] %}value="{{ include.data[property_name] }}" {% endif %} - {% if oob %} hx-swap-oob="true" {%endif%} hx-encoding="application/x-www-form-urlencoded" />
diff --git a/src/public/place/directive.html b/src/public/place/directive.html new file mode 100644 index 00000000..7ae8dc96 --- /dev/null +++ b/src/public/place/directive.html @@ -0,0 +1,13 @@ +
+
+ {% if progress.inProgressCount > 0 %} + {% include "place/directive_3_in_progress.html" %} + {% elsif progress.successCount > 0 %} + {% include "place/directive_4_prompt_save.html" %} + {% elsif progress.pendingCount > 0 %} + {% include "place/directive_2_prompt_upload.html" %} + {% else %} + {% include "place/directive_1_get_started.html" %} + {% endif %} +
+
diff --git a/src/public/place/directive_1_get_started.html b/src/public/place/directive_1_get_started.html new file mode 100644 index 00000000..78f549ac --- /dev/null +++ b/src/public/place/directive_1_get_started.html @@ -0,0 +1,22 @@ +

+ Getting Started... +

+

+ To get started, add some users which you want to create on the {{ session.authInfo.friendly }} instance. +

+ +{% for contactType in contactTypes %} +
+
+ + add Add {{contactType.friendly}} + +
+ Create New + Replace Existing + Upload from CSV + Move +
+
+
+{% endfor %} \ No newline at end of file diff --git a/src/public/place/directive_2_prompt_upload.html b/src/public/place/directive_2_prompt_upload.html new file mode 100644 index 00000000..6287c6f5 --- /dev/null +++ b/src/public/place/directive_2_prompt_upload.html @@ -0,0 +1,15 @@ +

+ Upload When Ready +

+

+ You now have some staged users below which are ready for upload. + When everything looks right, upload them to {{ session.authInfo.friendly }}. +

+ +
+
+ + group_add Upload + +
+
diff --git a/src/public/place/directive_3_in_progress.html b/src/public/place/directive_3_in_progress.html new file mode 100644 index 00000000..9c747190 --- /dev/null +++ b/src/public/place/directive_3_in_progress.html @@ -0,0 +1,17 @@ +

+ Upload In Progress... +

+ +{{ progress.percent }} + +
+
+ {{ progress.completeCount }} successes +
+
+ {{ progress.failureCount }} failures +
+
+ {{ progress.remainingCount }} remaining +
+
\ No newline at end of file diff --git a/src/public/place/directive_4_prompt_save.html b/src/public/place/directive_4_prompt_save.html new file mode 100644 index 00000000..07605207 --- /dev/null +++ b/src/public/place/directive_4_prompt_save.html @@ -0,0 +1,16 @@ +

+ Download Resulting User Credentials +

+

+ Some users have been successfully uploaded to the {{ session.authInfo.friendly }} instance. + Usernames and passwords are now available for download by clicking the button below. +

+ +
+
+ + save_as + Save Credentials + +
+
diff --git a/src/public/place/list.html b/src/public/place/list.html index 199520ab..07ee2289 100644 --- a/src/public/place/list.html +++ b/src/public/place/list.html @@ -1,11 +1,10 @@ -
+
{% for contactType in contactTypes %}
-

{{contactType.friendly}}

{% if contactType.places.length > 0 %} +

{{contactType.friendly}}

@@ -104,20 +103,5 @@

{{contactType.friendly}}

{% endif %} - -
-
- - add Add {{contactType.friendly}} - -
- Create New - Replace Existing - Upload from CSV - Move -
-
-
-
{% endfor %}
\ No newline at end of file diff --git a/src/public/place/list_event.html b/src/public/place/list_event.html index eb24e2a1..fab2ab40 100644 --- a/src/public/place/list_event.html +++ b/src/public/place/list_event.html @@ -1,2 +1,2 @@ -{% include "place/list.html" %} -{% include "place/progress_modal.html" %} \ No newline at end of file +{% include "place/directive.html" %} +{% include "place/list.html" %} \ No newline at end of file diff --git a/src/public/place/progress_modal.html b/src/public/place/progress_modal.html deleted file mode 100644 index d99b6d50..00000000 --- a/src/public/place/progress_modal.html +++ /dev/null @@ -1,43 +0,0 @@ - \ No newline at end of file diff --git a/src/services/progress-model.ts b/src/services/progress-model.ts index d3858c9e..ca9775f3 100644 --- a/src/services/progress-model.ts +++ b/src/services/progress-model.ts @@ -8,12 +8,14 @@ export default class ProgressModel { public successCount: number; public remainingCount: number; public failureCount: number; + public inProgressCount: number; constructor(sessionCache: SessionCache) { this.successCount = sessionCache.getPlaces({ state: PlaceUploadState.SUCCESS }).length; this.failureCount = sessionCache.getPlaces({ state: PlaceUploadState.FAILURE }).length; this.completeCount = this.successCount + this.failureCount; this.pendingCount = sessionCache.getPlaces().filter(r => !r.hasValidationErrors).length; + this.inProgressCount = sessionCache.getPlaces().filter(r => r.state === PlaceUploadState.IN_PROGRESS || r.state === PlaceUploadState.SCHEDULED).length; const percentage = this.pendingCount > 0 ? this.completeCount / this.pendingCount : 0; this.remainingCount = this.pendingCount - this.completeCount; this.percent = Math.round(percentage * 100.0) + '%' From b5e442d63d750f39ea572a0d6c866f1bf2ee613d Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Thu, 8 Feb 2024 10:32:01 +0700 Subject: [PATCH 04/15] Review and testing --- .eslintrc | 3 +- src/public/app/nav.html | 49 +------------------ .../components/button_save_credentials.html | 3 ++ src/public/components/button_upload.html | 3 ++ src/public/place/directive.html | 40 ++++++++++----- src/public/place/directive_3_in_progress.html | 14 +----- src/public/place/directive_4_prompt_save.html | 9 ++-- src/public/static/material-symbols.css | 23 +++++++++ src/services/place.ts | 4 +- src/services/progress-model.ts | 19 ++++--- src/services/session-cache.ts | 2 +- src/services/upload-manager.ts | 2 +- test/services/progress-model.spec.ts | 18 ++++--- 13 files changed, 92 insertions(+), 97 deletions(-) create mode 100644 src/public/components/button_save_credentials.html create mode 100644 src/public/components/button_upload.html diff --git a/.eslintrc b/.eslintrc index 3350a286..d5a0146c 100644 --- a/.eslintrc +++ b/.eslintrc @@ -9,7 +9,8 @@ }, "ignorePatterns": [ "**/node_modules/**", - "**/dist/**" + "**/dist/**", + "**/static/**" ], "plugins": ["promise", "node"], "rules": { diff --git a/src/public/app/nav.html b/src/public/app/nav.html index 28178a5c..1d95bfdb 100644 --- a/src/public/app/nav.html +++ b/src/public/app/nav.html @@ -70,13 +70,8 @@
- - group_add Upload - - - - save_as Save Credentials - + {% include "components/button_upload.html" className="navbar-item" %} + {% include "components/button_save_credentials.html" className="navbar-item" %} logout Logout @@ -99,45 +94,5 @@ $target.classList.toggle('is-active'); }); }); - - function closeAllModals() { - (document.querySelectorAll('.modal') || []).forEach(($modal) => { - closeModal($modal); - }); - } - - function openModal($el) { - $el.classList.add('is-active'); - } - - function closeModal($el) { - $el.classList.remove('is-active'); - } - - // Add a click event on buttons to open a specific modal - (document.querySelectorAll('.modal-trigger') || []).forEach(($trigger) => { - const modal = $trigger.dataset.target; - const $target = document.getElementById(modal); - - $trigger.addEventListener('click', () => { - openModal($target); - }); - }); - - // Add a click event on various child elements to close the parent modal - (document.querySelectorAll('.modal-background, .modal-action, .modal-close, .modal-card-head .delete, .modal-card-foot .button') || []).forEach(($close) => { - const $target = $close.closest('.modal'); - - $close.addEventListener('click', () => { - closeModal($target); - }); - }); - - // Add a keyboard event to close all modals - document.addEventListener('keydown', (event) => { - if (event.key === "Escape") { - closeAllModals(); - } - }); }); diff --git a/src/public/components/button_save_credentials.html b/src/public/components/button_save_credentials.html new file mode 100644 index 00000000..f1445a22 --- /dev/null +++ b/src/public/components/button_save_credentials.html @@ -0,0 +1,3 @@ + + save_as Save Credentials + \ No newline at end of file diff --git a/src/public/components/button_upload.html b/src/public/components/button_upload.html new file mode 100644 index 00000000..5d5f7459 --- /dev/null +++ b/src/public/components/button_upload.html @@ -0,0 +1,3 @@ + + group_add Upload + \ No newline at end of file diff --git a/src/public/place/directive.html b/src/public/place/directive.html index 7ae8dc96..403fbe6e 100644 --- a/src/public/place/directive.html +++ b/src/public/place/directive.html @@ -1,13 +1,29 @@ -
-
- {% if progress.inProgressCount > 0 %} - {% include "place/directive_3_in_progress.html" %} - {% elsif progress.successCount > 0 %} - {% include "place/directive_4_prompt_save.html" %} - {% elsif progress.pendingCount > 0 %} - {% include "place/directive_2_prompt_upload.html" %} - {% else %} - {% include "place/directive_1_get_started.html" %} - {% endif %} +
+
+
+
+ {% if progress.inProgressCount > 0 %} + {% include "place/directive_3_in_progress.html" %} + {% elsif progress.successCount > 0 %} + {% include "place/directive_4_prompt_save.html" %} + {% elsif progress.stagedCount > 0 %} + {% include "place/directive_2_prompt_upload.html" %} + {% else %} + {% include "place/directive_1_get_started.html" %} + {% endif %} +
+
-
+ +
+
+ Summary: +
    +
  • {{ progress.stagedCount }} staged
    • +
  • {{ progress.validationErrorCount }} validation errors
    • +
  • {{ progress.completeCount }} uploaded
    • +
  • {{ progress.failureCount }} failures
    • +
+
+
+
diff --git a/src/public/place/directive_3_in_progress.html b/src/public/place/directive_3_in_progress.html index 9c747190..97555635 100644 --- a/src/public/place/directive_3_in_progress.html +++ b/src/public/place/directive_3_in_progress.html @@ -2,16 +2,4 @@ Upload In Progress...

-{{ progress.percent }} - -
-
- {{ progress.completeCount }} successes -
-
- {{ progress.failureCount }} failures -
-
- {{ progress.remainingCount }} remaining -
-
\ No newline at end of file +{{ progress.percent }} \ No newline at end of file diff --git a/src/public/place/directive_4_prompt_save.html b/src/public/place/directive_4_prompt_save.html index 07605207..75158ae0 100644 --- a/src/public/place/directive_4_prompt_save.html +++ b/src/public/place/directive_4_prompt_save.html @@ -1,16 +1,13 @@

Download Resulting User Credentials

+

- Some users have been successfully uploaded to the {{ session.authInfo.friendly }} instance. - Usernames and passwords are now available for download by clicking the button below. + Usernames and passwords are now available for download. Click the button below to download the credentials as a file.

- - save_as - Save Credentials - + {% include "components/button_save_credentials.html" %}
diff --git a/src/public/static/material-symbols.css b/src/public/static/material-symbols.css index e69de29b..6b95c180 100644 --- a/src/public/static/material-symbols.css +++ b/src/public/static/material-symbols.css @@ -0,0 +1,23 @@ +/* fallback */ +@font-face { + font-family: 'Material Symbols Outlined'; + font-style: normal; + font-weight: 400; + src: url(https://fonts.gstatic.com/s/materialsymbolsoutlined/v161/kJF1BvYX7BgnkSrUwT8OhrdQw4oELdPIeeII9v6oDMzByHX9rA6RzaxHMPdY43zj-jCxv3fzvRNU22ZXGJpEpjC_1v-p_4MrImHCIJIZrDCvHOej.woff2) format('woff2'); +} + +.material-symbols-outlined { + font-family: 'Material Symbols Outlined'; + font-weight: normal; + font-style: normal; + font-size: 24px; + line-height: 1; + letter-spacing: normal; + text-transform: none; + display: inline-block; + white-space: nowrap; + word-wrap: normal; + direction: ltr; + -webkit-font-feature-settings: 'liga'; + -webkit-font-smoothing: antialiased; +} diff --git a/src/services/place.ts b/src/services/place.ts index e645e3f8..40a6aa34 100644 --- a/src/services/place.ts +++ b/src/services/place.ts @@ -20,7 +20,7 @@ export type UserCreationDetails = { export enum PlaceUploadState { SUCCESS = 'success', FAILURE = 'failure', - PENDING = 'pending', + STAGED = 'staged', SCHEDULED = 'scheduled', IN_PROGRESS = 'in_progress', } @@ -57,7 +57,7 @@ export default class Place { this.contact = new Contact(type); this.properties = {}; this.hierarchyProperties = {}; - this.state = PlaceUploadState.PENDING; + this.state = PlaceUploadState.STAGED; this.resolvedHierarchy = []; } diff --git a/src/services/progress-model.ts b/src/services/progress-model.ts index ca9775f3..25974aa6 100644 --- a/src/services/progress-model.ts +++ b/src/services/progress-model.ts @@ -3,21 +3,26 @@ import SessionCache from './session-cache'; export default class ProgressModel { public completeCount: number; - public pendingCount: number; + public stagedCount: number; public percent: string; public successCount: number; - public remainingCount: number; public failureCount: number; public inProgressCount: number; + public validationErrorCount: number; + public totalCount: number; constructor(sessionCache: SessionCache) { this.successCount = sessionCache.getPlaces({ state: PlaceUploadState.SUCCESS }).length; this.failureCount = sessionCache.getPlaces({ state: PlaceUploadState.FAILURE }).length; + this.validationErrorCount = sessionCache.getPlaces().filter(r => r.hasValidationErrors).length; + + const inProgressStates = [PlaceUploadState.IN_PROGRESS, PlaceUploadState.SCHEDULED]; + this.inProgressCount = sessionCache.getPlaces().filter(r => inProgressStates.includes(r.state)).length; + this.totalCount = sessionCache.getPlaces().length; + this.completeCount = this.successCount + this.failureCount; - this.pendingCount = sessionCache.getPlaces().filter(r => !r.hasValidationErrors).length; - this.inProgressCount = sessionCache.getPlaces().filter(r => r.state === PlaceUploadState.IN_PROGRESS || r.state === PlaceUploadState.SCHEDULED).length; - const percentage = this.pendingCount > 0 ? this.completeCount / this.pendingCount : 0; - this.remainingCount = this.pendingCount - this.completeCount; - this.percent = Math.round(percentage * 100.0) + '%' + this.stagedCount = this.totalCount - this.validationErrorCount - this.completeCount; + const percentage = this.stagedCount > 0 ? this.completeCount / this.totalCount : 0; + this.percent = Math.round(percentage * 100.0) + '%'; } } diff --git a/src/services/session-cache.ts b/src/services/session-cache.ts index f32a8ac6..b64b5bcc 100644 --- a/src/services/session-cache.ts +++ b/src/services/session-cache.ts @@ -1,7 +1,7 @@ import Place, { PlaceUploadState } from './place'; import { ChtSession } from '../lib/cht-api'; -export type SessionCacheUploadState = 'in_progress' | 'done' | 'pending'; +export type SessionCacheUploadState = 'in_progress' | 'done' | 'staged'; export default class SessionCache { private static caches: Map = new Map(); diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index 6bc13c30..bfa07ef7 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -8,7 +8,7 @@ import { UploadNewPlace } from './upload.new'; import { Config } from '../config'; import RemotePlaceCache from '../lib/remote-place-cache'; -const UPLOAD_BATCH_SIZE = 10; +const UPLOAD_BATCH_SIZE = 1; export interface Uploader { handleContact (payload: PlacePayload): Promise; diff --git a/test/services/progress-model.spec.ts b/test/services/progress-model.spec.ts index 7f24ab15..db2f6479 100644 --- a/test/services/progress-model.spec.ts +++ b/test/services/progress-model.spec.ts @@ -10,20 +10,22 @@ describe('services/progress-model.ts', () => { sessionCache.savePlaces( { id: 'fail', state: PlaceUploadState.FAILURE }, { id: 'in-progress', state: PlaceUploadState.IN_PROGRESS }, - { id: 'pending', state: PlaceUploadState.PENDING }, + { id: 'staged', state: PlaceUploadState.STAGED }, { id: 'scheduled', state: PlaceUploadState.SCHEDULED }, { id: 'success', state: PlaceUploadState.SUCCESS }, - { id: 'validation-error', state: PlaceUploadState.PENDING, hasValidationErrors: true }, + { id: 'validation-error', state: PlaceUploadState.STAGED, hasValidationErrors: true }, ); const progressModel = new ProgressModel(sessionCache); expect(progressModel).to.deep.eq({ failureCount: 1, successCount: 1, - remainingCount: 3, + validationErrorCount: 1, completeCount: 2, - pendingCount: 5, - percent: '40%', + inProgressCount: 2, + stagedCount: 3, + totalCount: 6, + percent: '33%', }); }); @@ -33,9 +35,11 @@ describe('services/progress-model.ts', () => { expect(progressModel).to.deep.eq({ failureCount: 0, successCount: 0, + validationErrorCount: 0, completeCount: 0, - remainingCount: 0, - pendingCount: 0, + inProgressCount: 0, + stagedCount: 0, + totalCount: 0, percent: '0%', }); }); From b61f464b950e856ea0e8ebb108eb45739d6fe692 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Thu, 8 Feb 2024 10:33:08 +0700 Subject: [PATCH 05/15] Revert use of static resources --- src/public/app/view.html | 14 +- src/public/auth/view.html | 9 +- src/public/static/bulma-tooltip.min.css | 2 - src/public/static/bulma.min.css | 1 - src/public/static/htmx.min.js | 1 - src/public/static/material-symbols.css | 23 -- src/public/static/sse.js | 355 ------------------------ src/routes/authentication.ts | 9 - 8 files changed, 14 insertions(+), 400 deletions(-) delete mode 100644 src/public/static/bulma-tooltip.min.css delete mode 100644 src/public/static/bulma.min.css delete mode 100644 src/public/static/htmx.min.js delete mode 100644 src/public/static/material-symbols.css delete mode 100644 src/public/static/sse.js diff --git a/src/public/app/view.html b/src/public/app/view.html index 49958246..08930e61 100644 --- a/src/public/app/view.html +++ b/src/public/app/view.html @@ -2,11 +2,15 @@ CHT User Management Tool - - - - - + + + + + ")}}function Pr(){var e=te().querySelector('meta[name="htmx-config"]');if(e){return S(e.content)}else{return null}}function kr(){var e=Pr();if(e){Y.config=se(Y.config,e)}}Nr(function(){kr();Ir();var e=te().body;Pt(e);var t=te().querySelectorAll("[hx-trigger='restored'],[data-hx-trigger='restored']");e.addEventListener("htmx:abort",function(e){var t=e.target;var r=ie(t);if(r&&r.xhr){r.xhr.abort()}});var r=window.onpopstate;window.onpopstate=function(e){if(e.state&&e.state.htmx){Gt();ae(t,function(e){fe(e,"htmx:restored",{document:te(),triggerEvent:fe})})}else{if(r){r(e)}}};setTimeout(function(){fe(e,"htmx:load",{});e=null},0)});return Y}()}); \ No newline at end of file diff --git a/src/public/static/material-symbols.css b/src/public/static/material-symbols.css deleted file mode 100644 index 6b95c180..00000000 --- a/src/public/static/material-symbols.css +++ /dev/null @@ -1,23 +0,0 @@ -/* fallback */ -@font-face { - font-family: 'Material Symbols Outlined'; - font-style: normal; - font-weight: 400; - src: url(https://fonts.gstatic.com/s/materialsymbolsoutlined/v161/kJF1BvYX7BgnkSrUwT8OhrdQw4oELdPIeeII9v6oDMzByHX9rA6RzaxHMPdY43zj-jCxv3fzvRNU22ZXGJpEpjC_1v-p_4MrImHCIJIZrDCvHOej.woff2) format('woff2'); -} - -.material-symbols-outlined { - font-family: 'Material Symbols Outlined'; - font-weight: normal; - font-style: normal; - font-size: 24px; - line-height: 1; - letter-spacing: normal; - text-transform: none; - display: inline-block; - white-space: nowrap; - word-wrap: normal; - direction: ltr; - -webkit-font-feature-settings: 'liga'; - -webkit-font-smoothing: antialiased; -} diff --git a/src/public/static/sse.js b/src/public/static/sse.js deleted file mode 100644 index 943d80ac..00000000 --- a/src/public/static/sse.js +++ /dev/null @@ -1,355 +0,0 @@ -/* -Server Sent Events Extension -============================ -This extension adds support for Server Sent Events to htmx. See /www/extensions/sse.md for usage instructions. - -*/ - -(function() { - - /** @type {import("../htmx").HtmxInternalApi} */ - var api; - - htmx.defineExtension("sse", { - - /** - * Init saves the provided reference to the internal HTMX API. - * - * @param {import("../htmx").HtmxInternalApi} api - * @returns void - */ - init: function(apiRef) { - // store a reference to the internal API. - api = apiRef; - - // set a function in the public API for creating new EventSource objects - if (htmx.createEventSource == undefined) { - htmx.createEventSource = createEventSource; - } - }, - - /** - * onEvent handles all events passed to this extension. - * - * @param {string} name - * @param {Event} evt - * @returns void - */ - onEvent: function(name, evt) { - - switch (name) { - - case "htmx:beforeCleanupElement": - var internalData = api.getInternalData(evt.target) - // Try to remove remove an EventSource when elements are removed - if (internalData.sseEventSource) { - internalData.sseEventSource.close(); - } - - return; - - // Try to create EventSources when elements are processed - case "htmx:afterProcessNode": - ensureEventSourceOnElement(evt.target); - registerSSE(evt.target); - } - } - }); - - /////////////////////////////////////////////// - // HELPER FUNCTIONS - /////////////////////////////////////////////// - - - /** - * createEventSource is the default method for creating new EventSource objects. - * it is hoisted into htmx.config.createEventSource to be overridden by the user, if needed. - * - * @param {string} url - * @returns EventSource - */ - function createEventSource(url) { - return new EventSource(url, { withCredentials: true }); - } - - function splitOnWhitespace(trigger) { - return trigger.trim().split(/\s+/); - } - - function getLegacySSEURL(elt) { - var legacySSEValue = api.getAttributeValue(elt, "hx-sse"); - if (legacySSEValue) { - var values = splitOnWhitespace(legacySSEValue); - for (var i = 0; i < values.length; i++) { - var value = values[i].split(/:(.+)/); - if (value[0] === "connect") { - return value[1]; - } - } - } - } - - function getLegacySSESwaps(elt) { - var legacySSEValue = api.getAttributeValue(elt, "hx-sse"); - var returnArr = []; - if (legacySSEValue != null) { - var values = splitOnWhitespace(legacySSEValue); - for (var i = 0; i < values.length; i++) { - var value = values[i].split(/:(.+)/); - if (value[0] === "swap") { - returnArr.push(value[1]); - } - } - } - return returnArr; - } - - /** - * registerSSE looks for attributes that can contain sse events, right - * now hx-trigger and sse-swap and adds listeners based on these attributes too - * the closest event source - * - * @param {HTMLElement} elt - */ - function registerSSE(elt) { - // Find closest existing event source - var sourceElement = api.getClosestMatch(elt, hasEventSource); - if (sourceElement == null) { - // api.triggerErrorEvent(elt, "htmx:noSSESourceError") - return null; // no eventsource in parentage, orphaned element - } - - // Set internalData and source - var internalData = api.getInternalData(sourceElement); - var source = internalData.sseEventSource; - - // Add message handlers for every `sse-swap` attribute - queryAttributeOnThisOrChildren(elt, "sse-swap").forEach(function(child) { - - var sseSwapAttr = api.getAttributeValue(child, "sse-swap"); - if (sseSwapAttr) { - var sseEventNames = sseSwapAttr.split(","); - } else { - var sseEventNames = getLegacySSESwaps(child); - } - - for (var i = 0; i < sseEventNames.length; i++) { - var sseEventName = sseEventNames[i].trim(); - var listener = function(event) { - - // If the source is missing then close SSE - if (maybeCloseSSESource(sourceElement)) { - return; - } - - // If the body no longer contains the element, remove the listener - if (!api.bodyContains(child)) { - source.removeEventListener(sseEventName, listener); - } - - // swap the response into the DOM and trigger a notification - swap(child, event.data); - api.triggerEvent(elt, "htmx:sseMessage", event); - }; - - // Register the new listener - api.getInternalData(child).sseEventListener = listener; - source.addEventListener(sseEventName, listener); - } - }); - - // Add message handlers for every `hx-trigger="sse:*"` attribute - queryAttributeOnThisOrChildren(elt, "hx-trigger").forEach(function(child) { - - var sseEventName = api.getAttributeValue(child, "hx-trigger"); - if (sseEventName == null) { - return; - } - - // Only process hx-triggers for events with the "sse:" prefix - if (sseEventName.slice(0, 4) != "sse:") { - return; - } - - // remove the sse: prefix from here on out - sseEventName = sseEventName.substr(4); - - var listener = function() { - if (maybeCloseSSESource(sourceElement)) { - return - } - - if (!api.bodyContains(child)) { - source.removeEventListener(sseEventName, listener); - } - } - }); - } - - /** - * ensureEventSourceOnElement creates a new EventSource connection on the provided element. - * If a usable EventSource already exists, then it is returned. If not, then a new EventSource - * is created and stored in the element's internalData. - * @param {HTMLElement} elt - * @param {number} retryCount - * @returns {EventSource | null} - */ - function ensureEventSourceOnElement(elt, retryCount) { - - if (elt == null) { - return null; - } - - // handle extension source creation attribute - queryAttributeOnThisOrChildren(elt, "sse-connect").forEach(function(child) { - var sseURL = api.getAttributeValue(child, "sse-connect"); - if (sseURL == null) { - return; - } - - ensureEventSource(child, sseURL, retryCount); - }); - - // handle legacy sse, remove for HTMX2 - queryAttributeOnThisOrChildren(elt, "hx-sse").forEach(function(child) { - var sseURL = getLegacySSEURL(child); - if (sseURL == null) { - return; - } - - ensureEventSource(child, sseURL, retryCount); - }); - - } - - function ensureEventSource(elt, url, retryCount) { - var source = htmx.createEventSource(url); - - source.onerror = function(err) { - - // Log an error event - api.triggerErrorEvent(elt, "htmx:sseError", { error: err, source: source }); - - // If parent no longer exists in the document, then clean up this EventSource - if (maybeCloseSSESource(elt)) { - return; - } - - // Otherwise, try to reconnect the EventSource - if (source.readyState === EventSource.CLOSED) { - retryCount = retryCount || 0; - var timeout = Math.random() * (2 ^ retryCount) * 500; - window.setTimeout(function() { - ensureEventSourceOnElement(elt, Math.min(7, retryCount + 1)); - }, timeout); - } - }; - - source.onopen = function(evt) { - api.triggerEvent(elt, "htmx:sseOpen", { source: source }); - } - - api.getInternalData(elt).sseEventSource = source; - } - - /** - * maybeCloseSSESource confirms that the parent element still exists. - * If not, then any associated SSE source is closed and the function returns true. - * - * @param {HTMLElement} elt - * @returns boolean - */ - function maybeCloseSSESource(elt) { - if (!api.bodyContains(elt)) { - var source = api.getInternalData(elt).sseEventSource; - if (source != undefined) { - source.close(); - // source = null - return true; - } - } - return false; - } - - /** - * queryAttributeOnThisOrChildren returns all nodes that contain the requested attributeName, INCLUDING THE PROVIDED ROOT ELEMENT. - * - * @param {HTMLElement} elt - * @param {string} attributeName - */ - function queryAttributeOnThisOrChildren(elt, attributeName) { - - var result = []; - - // If the parent element also contains the requested attribute, then add it to the results too. - if (api.hasAttribute(elt, attributeName)) { - result.push(elt); - } - - // Search all child nodes that match the requested attribute - elt.querySelectorAll("[" + attributeName + "], [data-" + attributeName + "]").forEach(function(node) { - result.push(node); - }); - - return result; - } - - /** - * @param {HTMLElement} elt - * @param {string} content - */ - function swap(elt, content) { - - api.withExtensions(elt, function(extension) { - content = extension.transformResponse(content, null, elt); - }); - - var swapSpec = api.getSwapSpecification(elt); - var target = api.getTarget(elt); - var settleInfo = api.makeSettleInfo(elt); - - api.selectAndSwap(swapSpec.swapStyle, target, elt, content, settleInfo); - - settleInfo.elts.forEach(function(elt) { - if (elt.classList) { - elt.classList.add(htmx.config.settlingClass); - } - api.triggerEvent(elt, 'htmx:beforeSettle'); - }); - - // Handle settle tasks (with delay if requested) - if (swapSpec.settleDelay > 0) { - setTimeout(doSettle(settleInfo), swapSpec.settleDelay); - } else { - doSettle(settleInfo)(); - } - } - - /** - * doSettle mirrors much of the functionality in htmx that - * settles elements after their content has been swapped. - * TODO: this should be published by htmx, and not duplicated here - * @param {import("../htmx").HtmxSettleInfo} settleInfo - * @returns () => void - */ - function doSettle(settleInfo) { - - return function() { - settleInfo.tasks.forEach(function(task) { - task.call(); - }); - - settleInfo.elts.forEach(function(elt) { - if (elt.classList) { - elt.classList.remove(htmx.config.settlingClass); - } - api.triggerEvent(elt, 'htmx:afterSettle'); - }); - } - } - - function hasEventSource(node) { - return api.getInternalData(node).sseEventSource != null; - } - -})(); diff --git a/src/routes/authentication.ts b/src/routes/authentication.ts index f6920584..02bb802d 100644 --- a/src/routes/authentication.ts +++ b/src/routes/authentication.ts @@ -28,15 +28,6 @@ export default async function authentication(fastify: FastifyInstance) { return resp.redirect('/login'); }); - fastify.get('/static/:filename', unauthenticatedOptions, async req => { - const params: any = req.params; - const filename = params.filename; - - const fullPath = path.resolve(`./src/public/static/${filename}`); - const content = fs.readFileSync(fullPath); - return content; - }); - fastify.post('/authenticate', unauthenticatedOptions, async (req, resp) => { const data: any = req.body; const { username, password, domain } = data; From 077ab528ceb2dee227c6d74857bc1e9cafd9d76d Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Thu, 8 Feb 2024 10:44:27 +0700 Subject: [PATCH 06/15] Pass eslint --- src/routes/authentication.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/authentication.ts b/src/routes/authentication.ts index 02bb802d..20d85c28 100644 --- a/src/routes/authentication.ts +++ b/src/routes/authentication.ts @@ -1,6 +1,4 @@ import { FastifyInstance, FastifyRequest } from 'fastify'; -import fs from 'fs'; -import path from 'path'; import Auth from '../lib/authentication'; import { ChtApi } from '../lib/cht-api'; From c953fbeb150b811a87d20e1a8e188818ee40dccd Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Fri, 9 Feb 2024 15:55:52 +0700 Subject: [PATCH 07/15] Quick prototype --- package-lock.json | 23 +++++++++++++++++++ package.json | 1 + scripts/retry-logic-server.js | 34 ++++++++++++++++++++++++++++ src/lib/axios-retry-config.ts | 13 +++++++++++ src/lib/cht-api.ts | 10 +++++++- src/services/upload-manager.ts | 17 +++++++++----- test/services/upload-manager.spec.ts | 4 ++-- 7 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 scripts/retry-logic-server.js create mode 100644 src/lib/axios-retry-config.ts diff --git a/package-lock.json b/package-lock.json index 11aa4583..3d071adf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@types/node": "^20.8.8", "@types/uuid": "^9.0.6", "axios": "^1.5.1", + "axios-retry": "^4.0.0", "csv": "^6.3.5", "dotenv": "^16.3.1", "fastify": "^4.23.2", @@ -868,6 +869,17 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-retry": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.0.0.tgz", + "integrity": "sha512-F6P4HVGITD/v4z9Lw2mIA24IabTajvpDZmKa6zq/gGwn57wN5j1P3uWrAV0+diqnW6kTM2fTqmWNfgYWGmMuiA==", + "dependencies": { + "is-retry-allowed": "^2.2.0" + }, + "peerDependencies": { + "axios": "0.x || 1.x" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2321,6 +2333,17 @@ "node": ">=8" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", diff --git a/package.json b/package.json index 87a09b47..971b5188 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@types/node": "^20.8.8", "@types/uuid": "^9.0.6", "axios": "^1.5.1", + "axios-retry": "^4.0.0", "csv": "^6.3.5", "dotenv": "^16.3.1", "fastify": "^4.23.2", diff --git a/scripts/retry-logic-server.js b/scripts/retry-logic-server.js new file mode 100644 index 00000000..7af90dcb --- /dev/null +++ b/scripts/retry-logic-server.js @@ -0,0 +1,34 @@ +const express = require('express'); + +const app = express(); + +// Middleware function to set timeout for all POST requests +const timeoutMiddleware = (req, res, next) => { + // Set timeout duration (in milliseconds) + const TIMEOUT_DURATION = 5000; // 5 seconds + + // Set timeout for this request + req.setTimeout(TIMEOUT_DURATION, () => { + // Handle timeout + res.status(408).send('Request Timeout'); + }); + + // Continue to next middleware + next(); +}; + +// Apply timeout middleware for all routes +app.use(timeoutMiddleware); + +// Route for handling POST requests +app.all('*', (req, res) => { + // Simulate long processing time to trigger timeout + setTimeout(() => { + res.send('POST request handled successfully'); + }, 6000); // This will exceed the timeout and trigger a timeout response +}); + +// Start the server +app.listen(3556, () => { + console.log(`Server is listening on port ${3556}`); +}); diff --git a/src/lib/axios-retry-config.ts b/src/lib/axios-retry-config.ts new file mode 100644 index 00000000..a59cb090 --- /dev/null +++ b/src/lib/axios-retry-config.ts @@ -0,0 +1,13 @@ +import { AxiosError } from 'axios'; +import isRetryAllowed from 'is-retry-allowed'; + +const axiosRetryConfig = { + retries: 5, + retryDelay: () => 1000, + retryCondition: isRetryAllowed, + onRetry: (retryCount: number, error: AxiosError) => { + console.log(`${retryCount} retry for ${error.request.url}`); + }, +}; + +export default axiosRetryConfig; \ No newline at end of file diff --git a/src/lib/cht-api.ts b/src/lib/cht-api.ts index 96df8209..d6e18313 100644 --- a/src/lib/cht-api.ts +++ b/src/lib/cht-api.ts @@ -1,8 +1,12 @@ import _ from 'lodash'; import axios, { AxiosHeaders } from 'axios'; +import axiosRetry from 'axios-retry'; +import axiosRetryConfig from './axios-retry-config'; import { UserPayload } from '../services/user-payload'; import { AuthenticationInfo, Config, ContactType } from '../config'; +axiosRetry(axios, axiosRetryConfig); + const { NODE_ENV } = process.env; @@ -164,7 +168,11 @@ export class ChtApi { createUser = async (user: UserPayload): Promise => { const url = `${this.protocolAndHost}/api/v1/users`; console.log('axios.post', url); - await axios.post(url, user, this.authorizationOptions()); + const axiosRequestionConfig = { + ...this.authorizationOptions(), + 'axios-retry': { retries: 0 }, // upload-manager handles retries for this + }; + await axios.post(url, user, axiosRequestionConfig); }; getParentAndSibling = async (parentId: string, contactType: ContactType): Promise<{ parent: any; sibling: any }> => { diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index bf372b62..e57f0642 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -1,13 +1,14 @@ import EventEmitter from 'events'; -import { ChtApi, PlacePayload } from '../lib/cht-api'; +import axiosRetryConfig from '../lib/axios-retry-config'; +import { ChtApi, PlacePayload } from '../lib/cht-api'; +import { Config } from '../config'; import Place, { PlaceUploadState } from './place'; -import { UserPayload } from './user-payload'; +import RemotePlaceCache from '../lib/remote-place-cache'; import SessionCache, { SessionCacheUploadState } from './session-cache'; -import { UploadReplacementPlace } from './upload.replacement'; import { UploadNewPlace } from './upload.new'; -import { Config } from '../config'; -import RemotePlaceCache from '../lib/remote-place-cache'; +import { UploadReplacementPlace } from './upload.replacement'; +import { UserPayload } from './user-payload'; const UPLOAD_BATCH_SIZE = 10; @@ -108,7 +109,11 @@ async function tryCreateUser (userPayload: UserPayload, chtApi: ChtApi): Promise return userPayload; } catch (err: any) { if (err?.response?.status !== 400) { - throw err; + if (axiosRetryConfig.retryCondition(err)) { + continue; + } else { + throw err; + } } const msg = err.response?.data?.error?.message || err.response?.data; diff --git a/test/services/upload-manager.spec.ts b/test/services/upload-manager.spec.ts index 026b3976..4823fee2 100644 --- a/test/services/upload-manager.spec.ts +++ b/test/services/upload-manager.spec.ts @@ -195,7 +195,7 @@ describe('upload-manager.ts', () => { const { remotePlace, sessionCache, chtApi } = await createMocks(); chtApi.createUser - .throws('timeout') + .throws({ code: 'CERT_REJECTED', toString: () => 'upload-error' }) .onSecondCall().resolves(); const chu_name = 'new chu'; @@ -204,7 +204,7 @@ describe('upload-manager.ts', () => { const uploadManager = new UploadManager(); await uploadManager.doUpload(sessionCache.getPlaces(), chtApi); expect(chu.isCreated).to.be.false; - expect(chu.uploadError).to.include('timeout'); + expect(chu.uploadError).to.include('upload-error'); expect(chu.creationDetails).to.deep.eq({ contactId: 'created-contact-id', placeId: 'created-place-id', From 631b61708d1d71182e436b0ac3df03c89ac35f60 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Sun, 11 Feb 2024 11:16:09 +0700 Subject: [PATCH 08/15] Test end to end. upload-manager tests --- scripts/retry-logic-server.js | 31 ++++++++++++----- src/lib/axios-retry-config.ts | 13 ++++--- src/services/upload-manager.ts | 26 +++++++------- test/e2e-request.spec.ts | 5 --- test/lib/axios-retry-config.spec.ts | 26 ++++++++++++++ test/services/upload-manager.spec.ts | 51 +++++++++++++++++++++++++++- 6 files changed, 120 insertions(+), 32 deletions(-) delete mode 100644 test/e2e-request.spec.ts create mode 100644 test/lib/axios-retry-config.spec.ts diff --git a/scripts/retry-logic-server.js b/scripts/retry-logic-server.js index 7af90dcb..1c6ca1cb 100644 --- a/scripts/retry-logic-server.js +++ b/scripts/retry-logic-server.js @@ -4,28 +4,41 @@ const app = express(); // Middleware function to set timeout for all POST requests const timeoutMiddleware = (req, res, next) => { - // Set timeout duration (in milliseconds) - const TIMEOUT_DURATION = 5000; // 5 seconds + const TIMEOUT_DURATION = 1000; - // Set timeout for this request req.setTimeout(TIMEOUT_DURATION, () => { // Handle timeout res.status(408).send('Request Timeout'); }); - // Continue to next middleware next(); }; -// Apply timeout middleware for all routes +app.post('/_session', (req, res) => { + res.set('Set-Cookie', 'AuthSession=abc123'); + res.status(200).send('OK'); +}); + +app.get('/medic/_design/medic-client/_view/contacts_by_type_freetext', (req, res) => { + const startkey = JSON.parse(req.query.startkey); + console.log('contacts_by_type_freetext', startkey); + const DATA = [ + {"id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","key":["b_sub_county","name:malava"],"value":"false false b_sub_county malava","doc":{"_id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","_rev":"1-cd20b7095c20172237867233b0375eda","parent":{"_id":"95d9abd1-7c17-41b1-af98-595509f96631"},"type":"contact","is_name_generated":"false","name":"Malava","external_id":"","contact":{"_id":"1e3d8375-6ab4-4409-be3f-3324db7658e9"},"geolocation":"","meta":{"created_by":"medic","created_by_person_uuid":"","created_by_place_uuid":""},"contact_type":"b_sub_county","reported_date":1702573623984,"form_version":{"time":1700734737921,"sha256":"755eac18545f4282c2a3e04c6a8857a2c4d5142f36ed86f5c93b3228ec6fad7a"}}}, + {"id":"2926bf4c-63eb-433d-a2b4-274fd05d2f1c","key":["c_community_health_unit","name:chu"],"value":"false false c_community_health_unit chu","doc":{"_id":"2926bf4c-63eb-433d-a2b4-274fd05d2f1c","_rev":"1-c15f26fe064f8357c19d1124286bf4c4","name":"Chu","PARENT":"Chepalungu","code":"123456","type":"contact","contact_type":"c_community_health_unit","parent":{"_id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","parent":{"_id":"95d9abd1-7c17-41b1-af98-595509f96631"}},"contact":{"_id":"bb9ebc4c6af161ee0f53b42339001fb1"},"reported_date":1701631255451}}, + ]; + res.json({ + "total_rows":2, + "offset":0, + "rows": DATA.filter(r => r.key[0] === startkey[0]) + }); +}); + app.use(timeoutMiddleware); -// Route for handling POST requests app.all('*', (req, res) => { - // Simulate long processing time to trigger timeout setTimeout(() => { - res.send('POST request handled successfully'); - }, 6000); // This will exceed the timeout and trigger a timeout response + res.status(200).send('OK'); + }, 2000); }); // Start the server diff --git a/src/lib/axios-retry-config.ts b/src/lib/axios-retry-config.ts index a59cb090..b5858fc7 100644 --- a/src/lib/axios-retry-config.ts +++ b/src/lib/axios-retry-config.ts @@ -1,12 +1,15 @@ -import { AxiosError } from 'axios'; +import { AxiosError, AxiosRequestConfig } from 'axios'; import isRetryAllowed from 'is-retry-allowed'; const axiosRetryConfig = { - retries: 5, + retries: 3, retryDelay: () => 1000, - retryCondition: isRetryAllowed, - onRetry: (retryCount: number, error: AxiosError) => { - console.log(`${retryCount} retry for ${error.request.url}`); + retryCondition: (err: AxiosError) => { + const status = err.response?.status; + return (!status || status === 409 || status >= 500) && isRetryAllowed(err); + }, + onRetry: (retryCount: number, error: AxiosError, requestConfig: AxiosRequestConfig) => { + console.log(`${requestConfig.url} failure. Retrying (${retryCount})`); }, }; diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index e57f0642..48c70848 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -11,6 +11,7 @@ import { UploadReplacementPlace } from './upload.replacement'; import { UserPayload } from './user-payload'; const UPLOAD_BATCH_SIZE = 10; +const RETRY_COUNT = 5; export interface Uploader { handleContact (payload: PlacePayload): Promise; @@ -74,7 +75,7 @@ export class UploadManager extends EventEmitter { console.log(`successfully created ${JSON.stringify(place.creationDetails)}`); this.eventedPlaceStateChange(place, PlaceUploadState.SUCCESS); } catch (err: any) { - const errorDetails = err.response?.data.toString() || err.toString(); + const errorDetails = err.response?.data ? JSON.stringify(err.response?.data) : err.toString(); console.log('error when creating user', errorDetails); place.uploadError = errorDetails; this.eventedPlaceStateChange(place, PlaceUploadState.FAILURE); @@ -103,27 +104,28 @@ export class UploadManager extends EventEmitter { } async function tryCreateUser (userPayload: UserPayload, chtApi: ChtApi): Promise<{ username: string; password: string }> { - for (let retryCount = 0; retryCount < 5; ++retryCount) { + for (let retryCount = 0; retryCount < RETRY_COUNT; ++retryCount) { try { await chtApi.createUser(userPayload); return userPayload; } catch (err: any) { - if (err?.response?.status !== 400) { - if (axiosRetryConfig.retryCondition(err)) { - continue; - } else { - throw err; - } + if (axiosRetryConfig.retryCondition(err)) { + continue; } - const msg = err.response?.data?.error?.message || err.response?.data; - console.error('createUser retry because', msg); - if (msg?.includes('already taken')) { + if (err.response?.status !== 400) { + throw err; + } + + const translationKey = err.response?.data?.error?.translationKey; + console.error('createUser retry because', translationKey); + if (translationKey === 'username.taken') { userPayload.makeUsernameMoreComplex(); continue; } - if (msg?.includes('password')) { // password too easy to guess + const RETRY_PASSWORD_TRANSLATIONS = ['password.length.minimum', 'password.weak']; + if (RETRY_PASSWORD_TRANSLATIONS.includes(translationKey)) { userPayload.regeneratePassword(); continue; } diff --git a/test/e2e-request.spec.ts b/test/e2e-request.spec.ts deleted file mode 100644 index 0f81fef9..00000000 --- a/test/e2e-request.spec.ts +++ /dev/null @@ -1,5 +0,0 @@ -describe('request e2e tests', () => { - it('given requests, assert the payload sent to CHT', async () => { - - }); -}); diff --git a/test/lib/axios-retry-config.spec.ts b/test/lib/axios-retry-config.spec.ts new file mode 100644 index 00000000..b2723206 --- /dev/null +++ b/test/lib/axios-retry-config.spec.ts @@ -0,0 +1,26 @@ +import { expect } from 'chai'; +import AxiosRetryConfig from '../../src/lib/axios-retry-config'; +import { UploadManagerRetryScenarios } from '../services/upload-manager.spec'; + +const scenarios = [ + { desc: '503', axiosError: { response: { status: 503 } } }, + { desc: 'axios timeout', axiosError: { code: 'ECONNABORTED' } }, + { desc: 'service timeout', axiosError: { code: 'ECONNRESET' } }, + { desc: 'update conflict', axiosError: { code: 'ERR_BAD_REQUEST', response: { status: 409 } } }, +]; + +describe('lib/axios-retry-config', () => { + for (const scenario of scenarios) { + it(scenario.desc, () => { + const doRetry = AxiosRetryConfig.retryCondition(scenario.axiosError as any); + expect(doRetry).to.eq(true); + }); + } + + for (const scenario of UploadManagerRetryScenarios) { + it(scenario.desc, () => { + const doRetry = AxiosRetryConfig.retryCondition(scenario.axiosError as any); + expect(doRetry).to.eq(false); + }); + } +}); diff --git a/test/services/upload-manager.spec.ts b/test/services/upload-manager.spec.ts index 4823fee2..4c5ffa59 100644 --- a/test/services/upload-manager.spec.ts +++ b/test/services/upload-manager.spec.ts @@ -11,6 +11,39 @@ import RemotePlaceCache from '../../src/lib/remote-place-cache'; import { Config } from '../../src/config'; import RemotePlaceResolver from '../../src/lib/remote-place-resolver'; +export const UploadManagerRetryScenarios = [ + { + desc: 'username taken is not retried', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { error: { message: 'Username "chu" already taken.', translationKey: 'username.taken' } }, + } + }, + }, + { + desc: 'password too short', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { error: { message: 'The password must be at least 8 characters long.', translationKey: 'password.length.minimum' } }, + } + }, + }, + { + desc: 'password too weak', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { error: { message: 'The password is too easy to guess. Include a range of types of characters to increase the score.', translationKey: 'password.weak' } }, + } + }, + }, +]; + describe('upload-manager.ts', () => { beforeEach(() => { RemotePlaceCache.clear({}); @@ -228,6 +261,23 @@ describe('upload-manager.ts', () => { expect(chtApi.updatePlace.called).to.be.false; expect(chtApi.disableUsersWithPlace.called).to.be.false; }); + + for (const scenario of UploadManagerRetryScenarios) { + it(`retry: ${scenario.desc}`, async() => { + const { remotePlace, sessionCache, chtApi } = await createMocks(); + + chtApi.createUser.throws(scenario.axiosError); + + const chu_name = 'new chu'; + const chu = await createChu(remotePlace, chu_name, sessionCache, chtApi); + + const uploadManager = new UploadManager(); + await uploadManager.doUpload(sessionCache.getPlaces(), chtApi); + expect(chu.isCreated).to.be.false; + expect(chtApi.createUser.callCount).to.be.gt(2); // retried + expect(chu.uploadError).to.include('could not create user'); + }); + } }); async function createChu(remotePlace: RemotePlace, chu_name: string, sessionCache: any, chtApi: ChtApi) { @@ -277,4 +327,3 @@ async function createMocks() { return { fakeFormData, contactType, sessionCache, chtApi, remotePlace }; } - From 2712be2517888d081138a520f83722bcfaa7a1b5 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Sun, 11 Feb 2024 11:19:07 +0700 Subject: [PATCH 09/15] Test for no-retry --- test/services/upload-manager.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/services/upload-manager.spec.ts b/test/services/upload-manager.spec.ts index 4c5ffa59..0585826a 100644 --- a/test/services/upload-manager.spec.ts +++ b/test/services/upload-manager.spec.ts @@ -228,7 +228,7 @@ describe('upload-manager.ts', () => { const { remotePlace, sessionCache, chtApi } = await createMocks(); chtApi.createUser - .throws({ code: 'CERT_REJECTED', toString: () => 'upload-error' }) + .throws({ response: { status: 404 }, toString: () => 'upload-error' }) .onSecondCall().resolves(); const chu_name = 'new chu'; @@ -237,6 +237,7 @@ describe('upload-manager.ts', () => { const uploadManager = new UploadManager(); await uploadManager.doUpload(sessionCache.getPlaces(), chtApi); expect(chu.isCreated).to.be.false; + expect(chtApi.createUser.calledOnce).to.be.true; expect(chu.uploadError).to.include('upload-error'); expect(chu.creationDetails).to.deep.eq({ contactId: 'created-contact-id', From ed50b0346644ea4765b85e2966b92787d9ccdb97 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Sun, 11 Feb 2024 11:26:53 +0700 Subject: [PATCH 10/15] Fix eslint --- scripts/retry-logic-server.js | 12 +++++++----- src/lib/axios-retry-config.ts | 2 +- test/services/upload-manager.spec.ts | 7 ++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/retry-logic-server.js b/scripts/retry-logic-server.js index 1c6ca1cb..dd53c029 100644 --- a/scripts/retry-logic-server.js +++ b/scripts/retry-logic-server.js @@ -23,13 +23,15 @@ app.get('/medic/_design/medic-client/_view/contacts_by_type_freetext', (req, res const startkey = JSON.parse(req.query.startkey); console.log('contacts_by_type_freetext', startkey); const DATA = [ - {"id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","key":["b_sub_county","name:malava"],"value":"false false b_sub_county malava","doc":{"_id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","_rev":"1-cd20b7095c20172237867233b0375eda","parent":{"_id":"95d9abd1-7c17-41b1-af98-595509f96631"},"type":"contact","is_name_generated":"false","name":"Malava","external_id":"","contact":{"_id":"1e3d8375-6ab4-4409-be3f-3324db7658e9"},"geolocation":"","meta":{"created_by":"medic","created_by_person_uuid":"","created_by_place_uuid":""},"contact_type":"b_sub_county","reported_date":1702573623984,"form_version":{"time":1700734737921,"sha256":"755eac18545f4282c2a3e04c6a8857a2c4d5142f36ed86f5c93b3228ec6fad7a"}}}, - {"id":"2926bf4c-63eb-433d-a2b4-274fd05d2f1c","key":["c_community_health_unit","name:chu"],"value":"false false c_community_health_unit chu","doc":{"_id":"2926bf4c-63eb-433d-a2b4-274fd05d2f1c","_rev":"1-c15f26fe064f8357c19d1124286bf4c4","name":"Chu","PARENT":"Chepalungu","code":"123456","type":"contact","contact_type":"c_community_health_unit","parent":{"_id":"e847f6e2-6dba-46dd-8128-5b153d0cd75f","parent":{"_id":"95d9abd1-7c17-41b1-af98-595509f96631"}},"contact":{"_id":"bb9ebc4c6af161ee0f53b42339001fb1"},"reported_date":1701631255451}}, + // eslint-disable-next-line max-len + { id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', key: ['b_sub_county', 'name:malava'], value: 'false false b_sub_county malava', doc: { _id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', _rev: '1-cd20b7095c20172237867233b0375eda', parent: { _id: '95d9abd1-7c17-41b1-af98-595509f96631' }, type: 'contact', is_name_generated: 'false', name: 'Malava', external_id: '', contact: { _id: '1e3d8375-6ab4-4409-be3f-3324db7658e9' }, contact_type: 'b_sub_county', reported_date: 1702573623984 } }, + // eslint-disable-next-line max-len + { id: '2926bf4c-63eb-433d-a2b4-274fd05d2f1c', key: ['c_community_health_unit', 'name:chu'], value: 'false false c_community_health_unit chu', doc: { _id: '2926bf4c-63eb-433d-a2b4-274fd05d2f1c', _rev: '1-c15f26fe064f8357c19d1124286bf4c4', name: 'Chu', PARENT: 'Chepalungu', code: '123456', type: 'contact', contact_type: 'c_community_health_unit', parent: { _id: 'e847f6e2-6dba-46dd-8128-5b153d0cd75f', parent: { _id: '95d9abd1-7c17-41b1-af98-595509f96631' } }, contact: { _id: 'bb9ebc4c6af161ee0f53b42339001fb1' }, reported_date: 1701631255451 } }, ]; res.json({ - "total_rows":2, - "offset":0, - "rows": DATA.filter(r => r.key[0] === startkey[0]) + total_rows: 2, + offset: 0, + rows: DATA.filter(r => r.key[0] === startkey[0]) }); }); diff --git a/src/lib/axios-retry-config.ts b/src/lib/axios-retry-config.ts index b5858fc7..bb60aee5 100644 --- a/src/lib/axios-retry-config.ts +++ b/src/lib/axios-retry-config.ts @@ -13,4 +13,4 @@ const axiosRetryConfig = { }, }; -export default axiosRetryConfig; \ No newline at end of file +export default axiosRetryConfig; diff --git a/test/services/upload-manager.spec.ts b/test/services/upload-manager.spec.ts index 0585826a..8d614527 100644 --- a/test/services/upload-manager.spec.ts +++ b/test/services/upload-manager.spec.ts @@ -38,7 +38,12 @@ export const UploadManagerRetryScenarios = [ code: 'ERR_BAD_REQUEST', response: { status: 400, - data: { error: { message: 'The password is too easy to guess. Include a range of types of characters to increase the score.', translationKey: 'password.weak' } }, + data: { + error: { + message: 'The password is too easy to guess. Include a range of types of characters to increase the score.', + translationKey: 'password.weak' + } + }, } }, }, From 974d5674ecd1aa04262c1f9d8bc3da6b1c606b8e Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Mon, 12 Feb 2024 11:55:33 +0700 Subject: [PATCH 11/15] Basically rewrite after testing update-conflict --- src/lib/axios-retry-config.ts | 16 ---- src/lib/cht-api.ts | 2 +- src/lib/retry-logic.ts | 70 +++++++++++++++ src/services/upload-manager.ts | 42 +-------- src/services/upload.replacement.ts | 3 +- test/lib/axios-retry-config.spec.ts | 26 ------ test/lib/retry-logic.spec.ts | 125 +++++++++++++++++++++++++++ test/services/upload-manager.spec.ts | 67 +++----------- 8 files changed, 216 insertions(+), 135 deletions(-) delete mode 100644 src/lib/axios-retry-config.ts create mode 100644 src/lib/retry-logic.ts delete mode 100644 test/lib/axios-retry-config.spec.ts create mode 100644 test/lib/retry-logic.spec.ts diff --git a/src/lib/axios-retry-config.ts b/src/lib/axios-retry-config.ts deleted file mode 100644 index bb60aee5..00000000 --- a/src/lib/axios-retry-config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { AxiosError, AxiosRequestConfig } from 'axios'; -import isRetryAllowed from 'is-retry-allowed'; - -const axiosRetryConfig = { - retries: 3, - retryDelay: () => 1000, - retryCondition: (err: AxiosError) => { - const status = err.response?.status; - return (!status || status === 409 || status >= 500) && isRetryAllowed(err); - }, - onRetry: (retryCount: number, error: AxiosError, requestConfig: AxiosRequestConfig) => { - console.log(`${requestConfig.url} failure. Retrying (${retryCount})`); - }, -}; - -export default axiosRetryConfig; diff --git a/src/lib/cht-api.ts b/src/lib/cht-api.ts index d6e18313..9762a4cd 100644 --- a/src/lib/cht-api.ts +++ b/src/lib/cht-api.ts @@ -1,7 +1,7 @@ import _ from 'lodash'; import axios, { AxiosHeaders } from 'axios'; import axiosRetry from 'axios-retry'; -import axiosRetryConfig from './axios-retry-config'; +import { axiosRetryConfig } from './retry-logic'; import { UserPayload } from '../services/user-payload'; import { AuthenticationInfo, Config, ContactType } from '../config'; diff --git a/src/lib/retry-logic.ts b/src/lib/retry-logic.ts new file mode 100644 index 00000000..d29afb8a --- /dev/null +++ b/src/lib/retry-logic.ts @@ -0,0 +1,70 @@ +import { AxiosError, AxiosRequestConfig } from 'axios'; +import isRetryAllowed from 'is-retry-allowed'; +import { UserPayload } from '../services/user-payload'; +import { ChtApi } from './cht-api'; + +const RETRY_COUNT = 4; + +export const axiosRetryConfig = { + retries: RETRY_COUNT, + retryDelay: () => 1000, + retryCondition: (err: AxiosError) => { + const status = err.response?.status; + return (!status || status >= 500) && isRetryAllowed(err); + }, + onRetry: (retryCount: number, error: AxiosError, requestConfig: AxiosRequestConfig) => { + console.log(`${requestConfig.url} failure. Retrying (${retryCount})`); + }, +}; + +export async function retryOnUpdateConflict(funcWithPut: () => Promise): Promise { + for (let retryCount = 0; retryCount < RETRY_COUNT; retryCount++) { + try { + return await funcWithPut(); + } catch (err : any) { + const statusCode = err.response?.status; + if (statusCode === 409) { + console.log(`Retrying on update-conflict (${retryCount})`); + continue; + } + + throw err; + } + } + + throw Error('update-conflict 409 persisted'); +} + +export async function createUserWithRetries(userPayload: UserPayload, chtApi: ChtApi): Promise<{ username: string; password: string }> { + for (let retryCount = 0; retryCount < RETRY_COUNT; ++retryCount) { + try { + await chtApi.createUser(userPayload); + return userPayload; + } catch (err: any) { + if (axiosRetryConfig.retryCondition(err)) { + continue; + } + + if (err.response?.status !== 400) { + throw err; + } + + const translationKey = err.response?.data?.error?.translationKey; + console.error('createUser retry because', translationKey); + if (translationKey === 'username.taken') { + userPayload.makeUsernameMoreComplex(); + continue; + } + + const RETRY_PASSWORD_TRANSLATIONS = ['password.length.minimum', 'password.weak']; + if (RETRY_PASSWORD_TRANSLATIONS.includes(translationKey)) { + userPayload.regeneratePassword(); + continue; + } + + throw err; + } + } + + throw new Error('could not create user ' + userPayload.contact); +} diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index 97ae19e6..2d108039 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -1,6 +1,6 @@ import EventEmitter from 'events'; -import axiosRetryConfig from '../lib/axios-retry-config'; +import * as RetryLogic from '../lib/retry-logic'; import { ChtApi, PlacePayload } from '../lib/cht-api'; import { Config } from '../config'; import Place, { PlaceUploadState } from './place'; @@ -11,7 +11,6 @@ import { UploadReplacementPlace } from './upload.replacement'; import { UserPayload } from './user-payload'; const UPLOAD_BATCH_SIZE = 10; -const RETRY_COUNT = 5; export interface Uploader { handleContact (payload: PlacePayload): Promise; @@ -56,7 +55,8 @@ export class UploadManager extends EventEmitter { place.creationDetails.placeId = placeId; } - await uploader.linkContactAndPlace(place, place.creationDetails?.placeId); + const createdPlaceId = place.creationDetails.placeId; // closure required for typescript + await RetryLogic.retryOnUpdateConflict(() => uploader.linkContactAndPlace(place, createdPlaceId)); if (!place.creationDetails.contactId) { throw Error('creationDetails.contactId not set'); @@ -64,7 +64,7 @@ export class UploadManager extends EventEmitter { if (!place.creationDetails.username) { const userPayload = new UserPayload(place, place.creationDetails.placeId, place.creationDetails.contactId); - const { username, password } = await tryCreateUser(userPayload, chtApi); + const { username, password } = await RetryLogic.createUserWithRetries(userPayload, chtApi); place.creationDetails.username = username; place.creationDetails.password = password; } @@ -102,37 +102,3 @@ export class UploadManager extends EventEmitter { this.emit('places_state_change', state); }; } - -async function tryCreateUser (userPayload: UserPayload, chtApi: ChtApi): Promise<{ username: string; password: string }> { - for (let retryCount = 0; retryCount < RETRY_COUNT; ++retryCount) { - try { - await chtApi.createUser(userPayload); - return userPayload; - } catch (err: any) { - if (axiosRetryConfig.retryCondition(err)) { - continue; - } - - if (err.response?.status !== 400) { - throw err; - } - - const translationKey = err.response?.data?.error?.translationKey; - console.error('createUser retry because', translationKey); - if (translationKey === 'username.taken') { - userPayload.makeUsernameMoreComplex(); - continue; - } - - const RETRY_PASSWORD_TRANSLATIONS = ['password.length.minimum', 'password.weak']; - if (RETRY_PASSWORD_TRANSLATIONS.includes(translationKey)) { - userPayload.regeneratePassword(); - continue; - } - - throw err; - } - } - - throw new Error('could not create user ' + userPayload.contact); -} diff --git a/src/services/upload.replacement.ts b/src/services/upload.replacement.ts index fb4e7dcd..524f64c6 100644 --- a/src/services/upload.replacement.ts +++ b/src/services/upload.replacement.ts @@ -1,5 +1,6 @@ import { ChtApi, PlacePayload } from '../lib/cht-api'; import Place from './place'; +import { retryOnUpdateConflict } from '../lib/retry-logic'; import { Uploader } from './upload-manager'; export class UploadReplacementPlace implements Uploader { @@ -21,7 +22,7 @@ export class UploadReplacementPlace implements Uploader { throw Error('contactId and placeId are required'); } - const updatedPlaceId = await this.chtApi.updatePlace(payload, contactId); + const updatedPlaceId = await retryOnUpdateConflict(() => this.chtApi.updatePlace(payload, contactId)); const disabledUsers = await this.chtApi.disableUsersWithPlace(placeId); place.creationDetails.disabledUsers = disabledUsers; diff --git a/test/lib/axios-retry-config.spec.ts b/test/lib/axios-retry-config.spec.ts deleted file mode 100644 index b2723206..00000000 --- a/test/lib/axios-retry-config.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { expect } from 'chai'; -import AxiosRetryConfig from '../../src/lib/axios-retry-config'; -import { UploadManagerRetryScenarios } from '../services/upload-manager.spec'; - -const scenarios = [ - { desc: '503', axiosError: { response: { status: 503 } } }, - { desc: 'axios timeout', axiosError: { code: 'ECONNABORTED' } }, - { desc: 'service timeout', axiosError: { code: 'ECONNRESET' } }, - { desc: 'update conflict', axiosError: { code: 'ERR_BAD_REQUEST', response: { status: 409 } } }, -]; - -describe('lib/axios-retry-config', () => { - for (const scenario of scenarios) { - it(scenario.desc, () => { - const doRetry = AxiosRetryConfig.retryCondition(scenario.axiosError as any); - expect(doRetry).to.eq(true); - }); - } - - for (const scenario of UploadManagerRetryScenarios) { - it(scenario.desc, () => { - const doRetry = AxiosRetryConfig.retryCondition(scenario.axiosError as any); - expect(doRetry).to.eq(false); - }); - } -}); diff --git a/test/lib/retry-logic.spec.ts b/test/lib/retry-logic.spec.ts new file mode 100644 index 00000000..2143fc51 --- /dev/null +++ b/test/lib/retry-logic.spec.ts @@ -0,0 +1,125 @@ +import Chai from 'chai'; +import sinon from 'sinon'; + +import * as RetryLogic from '../../src/lib/retry-logic'; +import { UserPayload } from '../../src/services/user-payload'; +import { ChtApi } from '../../src/lib/cht-api'; +import { mockSimpleContactType } from '../mocks'; + +import chaiAsPromised from 'chai-as-promised'; +Chai.use(chaiAsPromised); + +const { expect } = Chai; + +const RetryScenarios = [ + { desc: '503', axiosError: { response: { status: 503 } }, retry: 'axios' }, + { desc: 'axios timeout', axiosError: { code: 'ECONNABORTED' }, retry: 'axios' }, + { desc: 'service timeout', axiosError: { code: 'ECONNRESET' }, retry: 'axios' }, + + { desc: 'update conflict', axiosError: { code: 'ERR_BAD_REQUEST', response: { status: 409 } }, retry: 'update-conflict' }, + + { + desc: 'username taken is not retried', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { error: { message: 'Username "chu" already taken.', translationKey: 'username.taken' } }, + } + }, + retry: 'upload-manager' + }, + { + desc: 'password too short', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { error: { message: 'The password must be at least 8 characters long.', translationKey: 'password.length.minimum' } }, + } + }, + retry: 'upload-manager' + }, + { + desc: 'password too weak', + axiosError: { + code: 'ERR_BAD_REQUEST', + response: { + status: 400, + data: { + error: { + message: 'The password is too easy to guess. Include a range of types of characters to increase the score.', + translationKey: 'password.weak' + } + }, + } + }, + retry: 'upload-manager' + }, +]; + +export const UploadManagerRetryScenario = RetryScenarios[RetryScenarios.length - 1]; +const UpdateConflictScenario = RetryScenarios.find(s => s.retry === 'update-conflict'); + +describe('lib/retry-logic', () => { + describe('axiosRetryConfig', () => { + for (const scenario of RetryScenarios) { + it(scenario.desc, () => { + const doRetry = RetryLogic.axiosRetryConfig.retryCondition(scenario.axiosError as any); + expect(doRetry).to.eq(scenario.retry === 'axios'); + }); + } + }); + + describe('retryOnUpdateConflict', () => { + for (const scenario of RetryScenarios) { + it(scenario.desc, async () => { + const output = 'foo'; + const testFunction = sinon.stub() + .rejects(scenario.axiosError) + .onSecondCall().resolves(output); + const execute = RetryLogic.retryOnUpdateConflict(testFunction); + const expectRetry = scenario.retry === 'update-conflict'; + if (expectRetry) { + await expect(execute).to.eventually.eq(output); + expect(testFunction.callCount).to.eq(expectRetry ? 2 : 1); + } else { + await expect(execute).to.eventually.be.rejectedWith(scenario.axiosError); + } + }); + } + + it ('throws after persistent conflict', async () => { + const testFunction = sinon.stub().rejects(UpdateConflictScenario?.axiosError); + const execute = RetryLogic.retryOnUpdateConflict(testFunction); + await expect(execute).to.eventually.be.rejectedWith('persisted'); + expect(testFunction.callCount).to.eq(4); + }); + }); + + describe('createUserWithRetries', () => { + for (const scenario of RetryScenarios) { + it(scenario.desc, async() => { + const chtApi = { + createUser: sinon.stub().throws(scenario.axiosError), + }; + const place = { + generateUsername: sinon.stub().returns('username'), + type: mockSimpleContactType('string', 'bar'), + contact: { properties: {} }, + }; + const userPayload = new UserPayload(place as Place, 'place_id', 'contact_id'); + + const execute = RetryLogic.createUserWithRetries(userPayload as UserPayload, chtApi as ChtApi); + const expectRetry = ['upload-manager', 'axios'].includes(scenario.retry); + if (expectRetry) { + await expect(execute).to.eventually.be.rejectedWith('could not create user'); + expect(chtApi.createUser.callCount).to.eq(4); + } else { + await expect(execute).to.eventually.be.rejectedWith(scenario.axiosError); + expect(chtApi.createUser.callCount).to.eq(1); + } + }); + } + }); +}); diff --git a/test/services/upload-manager.spec.ts b/test/services/upload-manager.spec.ts index 8d614527..d8e94273 100644 --- a/test/services/upload-manager.spec.ts +++ b/test/services/upload-manager.spec.ts @@ -10,44 +10,7 @@ import { ChtApi, RemotePlace } from '../../src/lib/cht-api'; import RemotePlaceCache from '../../src/lib/remote-place-cache'; import { Config } from '../../src/config'; import RemotePlaceResolver from '../../src/lib/remote-place-resolver'; - -export const UploadManagerRetryScenarios = [ - { - desc: 'username taken is not retried', - axiosError: { - code: 'ERR_BAD_REQUEST', - response: { - status: 400, - data: { error: { message: 'Username "chu" already taken.', translationKey: 'username.taken' } }, - } - }, - }, - { - desc: 'password too short', - axiosError: { - code: 'ERR_BAD_REQUEST', - response: { - status: 400, - data: { error: { message: 'The password must be at least 8 characters long.', translationKey: 'password.length.minimum' } }, - } - }, - }, - { - desc: 'password too weak', - axiosError: { - code: 'ERR_BAD_REQUEST', - response: { - status: 400, - data: { - error: { - message: 'The password is too easy to guess. Include a range of types of characters to increase the score.', - translationKey: 'password.weak' - } - }, - } - }, - }, -]; +import { UploadManagerRetryScenario } from '../lib/retry-logic.spec'; describe('upload-manager.ts', () => { beforeEach(() => { @@ -268,22 +231,20 @@ describe('upload-manager.ts', () => { expect(chtApi.disableUsersWithPlace.called).to.be.false; }); - for (const scenario of UploadManagerRetryScenarios) { - it(`retry: ${scenario.desc}`, async() => { - const { remotePlace, sessionCache, chtApi } = await createMocks(); + it(`createUser is retried`, async() => { + const { remotePlace, sessionCache, chtApi } = await createMocks(); - chtApi.createUser.throws(scenario.axiosError); - - const chu_name = 'new chu'; - const chu = await createChu(remotePlace, chu_name, sessionCache, chtApi); - - const uploadManager = new UploadManager(); - await uploadManager.doUpload(sessionCache.getPlaces(), chtApi); - expect(chu.isCreated).to.be.false; - expect(chtApi.createUser.callCount).to.be.gt(2); // retried - expect(chu.uploadError).to.include('could not create user'); - }); - } + chtApi.createUser.throws(UploadManagerRetryScenario.axiosError); + + const chu_name = 'new chu'; + const chu = await createChu(remotePlace, chu_name, sessionCache, chtApi); + + const uploadManager = new UploadManager(); + await uploadManager.doUpload(sessionCache.getPlaces(), chtApi); + expect(chu.isCreated).to.be.false; + expect(chtApi.createUser.callCount).to.be.gt(2); // retried + expect(chu.uploadError).to.include('could not create user'); + }); }); async function createChu(remotePlace: RemotePlace, chu_name: string, sessionCache: any, chtApi: ChtApi) { From ba3103fb66e634a6f8cff45298c2275b6e0174b3 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Mon, 12 Feb 2024 11:56:42 +0700 Subject: [PATCH 12/15] Eslint --- test/lib/retry-logic.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib/retry-logic.spec.ts b/test/lib/retry-logic.spec.ts index 2143fc51..5325afa2 100644 --- a/test/lib/retry-logic.spec.ts +++ b/test/lib/retry-logic.spec.ts @@ -7,6 +7,7 @@ import { ChtApi } from '../../src/lib/cht-api'; import { mockSimpleContactType } from '../mocks'; import chaiAsPromised from 'chai-as-promised'; +import Place from '../../src/services/place'; Chai.use(chaiAsPromised); const { expect } = Chai; @@ -89,7 +90,7 @@ describe('lib/retry-logic', () => { }); } - it ('throws after persistent conflict', async () => { + it('throws after persistent conflict', async () => { const testFunction = sinon.stub().rejects(UpdateConflictScenario?.axiosError); const execute = RetryLogic.retryOnUpdateConflict(testFunction); await expect(execute).to.eventually.be.rejectedWith('persisted'); From 332a8ca6bbfa8262341bfcb356f31e85871dae59 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Tue, 13 Feb 2024 15:54:01 +0700 Subject: [PATCH 13/15] Code review feedback --- src/public/place/directive.html | 34 ++++++++++--------- src/public/place/directive_1_get_started.html | 2 +- .../place/directive_2_prompt_upload.html | 2 +- src/public/place/directive_3_in_progress.html | 2 +- src/public/place/directive_4_prompt_save.html | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/public/place/directive.html b/src/public/place/directive.html index 403fbe6e..87f3c9c3 100644 --- a/src/public/place/directive.html +++ b/src/public/place/directive.html @@ -1,29 +1,31 @@ -
-
-
+
+
+
{% if progress.inProgressCount > 0 %} {% include "place/directive_3_in_progress.html" %} - {% elsif progress.successCount > 0 %} - {% include "place/directive_4_prompt_save.html" %} {% elsif progress.stagedCount > 0 %} {% include "place/directive_2_prompt_upload.html" %} + {% elsif progress.successCount > 0 %} + {% include "place/directive_4_prompt_save.html" %} {% else %} {% include "place/directive_1_get_started.html" %} {% endif %}
-
+
-
-
- Summary: -
    -
  • {{ progress.stagedCount }} staged
    • -
  • {{ progress.validationErrorCount }} validation errors
    • -
  • {{ progress.completeCount }} uploaded
    • -
  • {{ progress.failureCount }} failures
    • -
+
+
+
+

Summary

+
    +
  • {{ progress.stagedCount }} staged
    • +
  • {{ progress.validationErrorCount }} validation errors
    • +
  • {{ progress.completeCount }} uploaded
    • +
  • {{ progress.failureCount }} failures
    • +
+
-
+
diff --git a/src/public/place/directive_1_get_started.html b/src/public/place/directive_1_get_started.html index 78f549ac..a926790a 100644 --- a/src/public/place/directive_1_get_started.html +++ b/src/public/place/directive_1_get_started.html @@ -1,4 +1,4 @@ -

+

Getting Started...

diff --git a/src/public/place/directive_2_prompt_upload.html b/src/public/place/directive_2_prompt_upload.html index 6287c6f5..da1efc62 100644 --- a/src/public/place/directive_2_prompt_upload.html +++ b/src/public/place/directive_2_prompt_upload.html @@ -1,4 +1,4 @@ -

+

Upload When Ready

diff --git a/src/public/place/directive_3_in_progress.html b/src/public/place/directive_3_in_progress.html index 97555635..6e9b3723 100644 --- a/src/public/place/directive_3_in_progress.html +++ b/src/public/place/directive_3_in_progress.html @@ -1,4 +1,4 @@ -

+

Upload In Progress...

diff --git a/src/public/place/directive_4_prompt_save.html b/src/public/place/directive_4_prompt_save.html index 75158ae0..94c921a1 100644 --- a/src/public/place/directive_4_prompt_save.html +++ b/src/public/place/directive_4_prompt_save.html @@ -1,4 +1,4 @@ -

+

Download Resulting User Credentials

From 543ec2544d84d24022aa11655f1cb3a718a0808d Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Fri, 23 Feb 2024 10:57:46 +0700 Subject: [PATCH 14/15] 1.0.15 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 747477d0..ace4b37e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cht-user-management", - "version": "1.0.14", + "version": "1.0.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cht-user-management", - "version": "1.0.14", + "version": "1.0.15", "license": "ISC", "dependencies": { "@fastify/autoload": "^5.8.0", diff --git a/package.json b/package.json index 8089d5ce..703b7eb5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cht-user-management", - "version": "1.0.14", + "version": "1.0.15", "main": "dist/index.js", "dependencies": { "@fastify/autoload": "^5.8.0", From 59989c8ffce5b6926233a7d3ab326752f442dca3 Mon Sep 17 00:00:00 2001 From: "kennsippell@gmail.com" Date: Fri, 23 Feb 2024 10:58:35 +0700 Subject: [PATCH 15/15] Fix lint --- src/services/upload-manager.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/upload-manager.ts b/src/services/upload-manager.ts index 9f82d9a7..dc0a0958 100644 --- a/src/services/upload-manager.ts +++ b/src/services/upload-manager.ts @@ -5,7 +5,6 @@ import { ChtApi, PlacePayload } from '../lib/cht-api'; import { Config } from '../config'; import Place, { PlaceUploadState } from './place'; import RemotePlaceCache from '../lib/remote-place-cache'; -import SessionCache, { SessionCacheUploadState } from './session-cache'; import { UploadNewPlace } from './upload.new'; import { UploadReplacementPlace } from './upload.replacement'; import { UserPayload } from './user-payload';