Skip to content

Commit

Permalink
config: move the maxParallelCalculators config to backend
Browse files Browse the repository at this point in the history
This value should be set only on the server side and send on request to
the frontend for validations (in the batch calculation forms). It is
always available in the server config and it defaults to the number of
available CPUs.

Update the unit tests to test the default value. The `config2_test.js`
does not initialize this value anymore.
  • Loading branch information
tahini committed Jan 28, 2025
1 parent 3c1543e commit db95249
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 15 deletions.
4 changes: 1 addition & 3 deletions examples/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ module.exports = {
}
// Transition does not support any other login method than localLogin
},
maxParallelCalculators: 2,

// Maximum number of parallel calculation. Used in tasks to start the calculator with this number of threads.
// Maximum number of parallel calculation. Used in tasks to start the calculator with this number of threads. By default, the number of calculators is set to the number of CPUs on the server
// maxParallelCalculators: 2,

// @deprecated: Use the cacheAllScenarios in the 'routing.transit.engines.trRouting' configuration instead
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,26 @@
* License text available at https://opensource.org/licenses/MIT
*/
process.env.PROJECT_CONFIG = `${__dirname}/../../../../../tests/config2_test.js`;
import os from 'os';
import config, { setProjectConfiguration } from '../server.config';
import path from 'path';

const availableCPUs = os.cpus();
jest.mock('os', () => ({
// Return 4 CPUs
cpus: jest.fn().mockReturnValue([
{ model: 'Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz', speed: 2800, times: { user: 0, nice: 0, sys: 0, idle: 0, irq: 0 } },
{ model: 'Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz', speed: 2800, times: { user: 0, nice: 0, sys: 0, idle: 0, irq: 0 } },
{ model: 'Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz', speed: 2800, times: { user: 0, nice: 0, sys: 0, idle: 0, irq: 0 } },
{ model: 'Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz', speed: 2800, times: { user: 0, nice: 0, sys: 0, idle: 0, irq: 0 } }
]),
}));

test('Expected default with env', () => {
expect(config.userDiskQuota).toEqual('1gb');
expect(config.maxFileUploadMB).toEqual(256);
expect(config.projectShortname).toEqual('unitTest');
expect(config.maxParallelCalculators).toEqual(1);
expect(config.maxParallelCalculators).toEqual(availableCPUs.length);
expect(config.projectDirectory).toEqual(path.normalize(`${__dirname}/../../../../../tests/dir`));
expect(config.routing.transit.engines.trRouting!.single).toEqual({ port: 4000, cacheAllScenarios: false, debug: false, logs: { nbFiles: 3, maxFileSizeKB: 5120 } });
expect(config.routing.transit.engines.trRouting!.batch).toEqual({ port: 14000, cacheAllScenarios: false, debug: false, logs: { nbFiles: 3, maxFileSizeKB: 5120 }});
Expand All @@ -36,13 +48,43 @@ test('setProjectConfiguration', () => {
osrmRouting: { port: 1234, enabled: false } as any
}
}
}
},
maxParallelCalculators: 3
});
expect(config.mapDefaultCenter).toEqual({ lon: -73, lat: 45 });
expect(config.separateAdminLoginPage).toEqual(false);
expect(config.maxParallelCalculators).toEqual(3);
expect(config.projectShortname).toEqual('newProject');
// Make sure the deep merge works for object configs
expect(config.routing.transit.engines.trRouting!.single).toEqual({ port: 5000, cacheAllScenarios: false, debug: false, logs: { nbFiles: 3, maxFileSizeKB: 5120 } });
expect(config.routing.transit.engines.trRouting!.batch).toEqual({ port: 14000, cacheAllScenarios: false, debug: false, logs: { nbFiles: 3, maxFileSizeKB: 10000 } });
expect(config.routing.driving!.engines.osrmRouting).toEqual({ port: 1234, host: null, autoStart: true, enabled: false });
});

describe('setProjectConfiguration, wrong values for max parallel calculators', () => {

const warnSpy = jest.spyOn(console, 'warn');
beforeEach(() => {
warnSpy.mockClear();
});

test('negative value', () => {
setProjectConfiguration({
maxParallelCalculators: -3
});
expect(config.maxParallelCalculators).toEqual(availableCPUs.length);
expect(warnSpy).toHaveBeenCalledTimes(1);
expect(warnSpy.mock.calls[0][0]).toBe(`maxParallelCalculators (-3) must be a positive number. Using the number of CPUs instead: ${availableCPUs.length}`);
});

test('more than CPU count', () => {
const parallelCalculators = 16;
setProjectConfiguration({
maxParallelCalculators: parallelCalculators
});
expect(config.maxParallelCalculators).toEqual(parallelCalculators);
expect(warnSpy).toHaveBeenCalledTimes(1);
expect(warnSpy.mock.calls[0][0]).toBe(`maxParallelCalculators (${parallelCalculators}) should not exceed the number of CPUs: ${availableCPUs.length}. This may cause performance issues.`);
});

});
23 changes: 22 additions & 1 deletion packages/chaire-lib-backend/src/config/server.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ export type ServerSideProjectConfiguration = {
* will expire
*/
tokenLifespanDays: number;
/**
* Maximum number of parallel calculations that can be run at the same time.
* It will default to the OS's number of available CPUs
*/
maxParallelCalculators: number;
};

const etcConfigFile = '/etc/transition/config.js';
Expand Down Expand Up @@ -328,7 +333,8 @@ setProjectConfigurationCommon<ServerSideProjectConfiguration>({
}
}
}
}
},
maxParallelCalculators: os.cpus().length
});

/**
Expand All @@ -354,6 +360,21 @@ export const setProjectConfiguration = (newConfig: Partial<ProjectConfiguration<
});
}

// Make sure the maximum number of parallel calculators is a positive value, but does not exceed the CPU count
if (newConfig.maxParallelCalculators !== undefined) {
const availableCPUs = os.cpus().length;
if (newConfig.maxParallelCalculators <= 0) {
console.warn(
`maxParallelCalculators (${newConfig.maxParallelCalculators}) must be a positive number. Using the number of CPUs instead: ${availableCPUs}`
);
newConfig.maxParallelCalculators = availableCPUs;
} else if (newConfig.maxParallelCalculators > availableCPUs) {
console.warn(
`maxParallelCalculators (${newConfig.maxParallelCalculators}) should not exceed the number of CPUs: ${availableCPUs}. This may cause performance issues.`
);
}
}

// If osrmRouting is configured using the old defaultPreferences format and not the new one in server.config, set it to the new format
// FIXME Remove once the old osrmRouting format is fully deprecated. See issue #1137
const osrmModesInDeprecatedFormat = (newConfig as any)?.defaultPreferences?.osrmRouting?.modes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
* This file is licensed under the MIT License.
* License text available at https://opensource.org/licenses/MIT
*/
import os from 'os';
import { directoryManager } from '../filesystem/directoryManager';
import ProcessManager from './ProcessManager';
import osrmService from '../osrm/OSRMService';
Expand Down Expand Up @@ -187,8 +186,7 @@ const startBatch = async function (
}: TrRoutingStartParameters = {}
) {
// Ensure we don't use more CPU than configured
// TODO The os.cpus().length should move to a "default config management class"
const maxThreadCount = config.maxParallelCalculators || os.cpus().length;
const maxThreadCount = config.maxParallelCalculators;
if (numberOfCpus === undefined) {
numberOfCpus = maxThreadCount;
} else if (numberOfCpus > maxThreadCount) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ export type ProjectConfiguration<AdditionalConfig> = {
* Absolute directory where project data will be stored (import files, osrm data, user data, caches, etc)
*/
projectDirectory: string;
maxParallelCalculators: number;
} & AdditionalConfig;

// Initialize default configuration
Expand All @@ -105,8 +104,7 @@ const projectConfig: ProjectConfiguration<any> = {
separateAdminLoginPage: false,
projectShortname: 'default',
userDiskQuota: '1gb',
maxFileUploadMB: 256,
maxParallelCalculators: 1
maxFileUploadMB: 256
};

/**
Expand Down
3 changes: 1 addition & 2 deletions packages/transition-backend/src/api/services.socketRoutes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
* This file is licensed under the MIT License.
* License text available at https://opensource.org/licenses/MIT
*/
import os from 'os';
import fs from 'fs';
import { EventEmitter } from 'events';
import osrm from 'osrm';
Expand Down Expand Up @@ -302,7 +301,7 @@ export default function (socket: EventEmitter, userId?: number) {

socket.on('service.parallelThreadCount', (callback: (response: { count: number }) => void) => {
callback({
count: serverConfig.maxParallelCalculators || os.cpus().length
count: serverConfig.maxParallelCalculators
});
});
}
1 change: 0 additions & 1 deletion tests/config2_test.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
module.exports = {

maxParallelCalculators: 1,
projectDirectory: `${__dirname}/dir`,
projectShortname: 'unitTest',

Expand Down

0 comments on commit db95249

Please sign in to comment.