diff --git a/lib/public/components/Filters/EnvironmentsFilter/KeyFilterModel.js b/lib/public/components/Filters/EnvironmentsFilter/KeyFilterModel.js new file mode 100644 index 0000000000..97d72bbf0e --- /dev/null +++ b/lib/public/components/Filters/EnvironmentsFilter/KeyFilterModel.js @@ -0,0 +1,29 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +import { ObservableBasedSelectionDropdownModel } from '../../detector/ObservableBasedSelectionDropdownModel.js'; +import { keyToOption } from './keyToOption.js'; + +/** + * Model storing state of a single key picked from the list of all the keys found in the environments raw configuration + */ +export class KeyFilterModel extends ObservableBasedSelectionDropdownModel { + /** + * Constructor + * + * @param {ObservableData>} keys$ observable remote data of keys list + */ + constructor(keys$) { + super(keys$, keyToOption, { multiple: false }); + } +} diff --git a/lib/public/components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js b/lib/public/components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js index 37632c5600..204c083529 100644 --- a/lib/public/components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js +++ b/lib/public/components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js @@ -12,6 +12,7 @@ */ import { FilterModel } from '../common/FilterModel.js'; +import { KeyFilterModel } from './KeyFilterModel.js'; import { RawTextFilterModel } from '../common/filters/RawTextFilterModel.js'; /** @@ -20,11 +21,13 @@ import { RawTextFilterModel } from '../common/filters/RawTextFilterModel.js'; export class RawConfigurationFilterModel extends FilterModel { /** * Constructor + * + * @param {ObservableData>} configurationKeys$ observable remote data of the existing configuration keys */ - constructor() { + constructor(configurationKeys$) { super(); - this._keyModel = new RawTextFilterModel(); + this._keyModel = new KeyFilterModel(configurationKeys$); this._valueModel = new RawTextFilterModel(); this._addSubmodel(this._keyModel); @@ -72,16 +75,16 @@ export class RawConfigurationFilterModel extends FilterModel { } /** - * Returns the key currently typed in the key input, trimmed and without surrounding quotes + * Returns the currently selected key, empty if none * * @return {string} the key */ get key() { - return this._keyModel.value.trim().replace(/^"(.*)"$/, '$1'); + return this._keyModel.selected[0] ?? ''; } /** - * Returns the value currently typed in the value input, trimmed + * Returns the currently typed in value * * @return {string} the value */ @@ -92,7 +95,7 @@ export class RawConfigurationFilterModel extends FilterModel { /** * Returns the key filter model * - * @return {RawTextFilterModel} the key filter model + * @return {KeyFilterModel} the key filter model */ get keyModel() { return this._keyModel; diff --git a/lib/public/components/Filters/EnvironmentsFilter/keyToOption.js b/lib/public/components/Filters/EnvironmentsFilter/keyToOption.js new file mode 100644 index 0000000000..bdbd9a1704 --- /dev/null +++ b/lib/public/components/Filters/EnvironmentsFilter/keyToOption.js @@ -0,0 +1,21 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +/** + * Convert a raw configuration key to a selection option + * The raw key value is best for mapping to the configuration + * + * @param {string} key the key to convert + * @return {SelectionOption} the key's option + */ +export const keyToOption = (key) => ({ label: key, value: key }); diff --git a/lib/public/components/Filters/EnvironmentsFilter/rawConfigurationFilter.js b/lib/public/components/Filters/EnvironmentsFilter/rawConfigurationFilter.js index 04193d8209..a5a2a31cb0 100644 --- a/lib/public/components/Filters/EnvironmentsFilter/rawConfigurationFilter.js +++ b/lib/public/components/Filters/EnvironmentsFilter/rawConfigurationFilter.js @@ -13,6 +13,7 @@ import { h } from '/js/src/index.js'; import { rawTextFilter } from '../common/filters/rawTextFilter.js'; +import { selectionDropdown } from '../../common/selection/dropdown/selectionDropdown.js'; /** * Renders a filter for the raw configuration of environments, allowing users to filter environments on a key, optionally with a value, @@ -25,21 +26,14 @@ export const rawConfigurationFilter = (rawConfigurationFilterModel) => h('.flex-row.g2', [ h( '.flex-column.flex-grow', - // Bubble to update the value placeholder when a key is typed - { oninput: () => rawConfigurationFilterModel.visualChange$.notify() }, - [ - rawTextFilter( - rawConfigurationFilterModel.keyModel, - { id: 'rawConfiguration-key', placeholder: 'e.g. hosts' }, - ), - ], + selectionDropdown(rawConfigurationFilterModel.keyModel, { selectorPrefix: 'rawConfiguration-key', placeholder: 'Any key' }), ), h('.flex-column.flex-grow', [ rawTextFilter( rawConfigurationFilterModel.valueModel, { id: 'rawConfiguration-value', - placeholder: rawConfigurationFilterModel.key ? 'e.g. some-host' : 'Search anywhere in the configuration', + placeholder: rawConfigurationFilterModel.key ? 'e.g. host.123@flp.ch' : 'Search anywhere in the configuration', }, ), ]), diff --git a/lib/public/components/common/selection/SelectionModel.js b/lib/public/components/common/selection/SelectionModel.js index b9926b4f32..0b629be9af 100644 --- a/lib/public/components/common/selection/SelectionModel.js +++ b/lib/public/components/common/selection/SelectionModel.js @@ -129,6 +129,7 @@ export class SelectionModel extends Observable { */ reset() { this._selectedOptions = [...this._defaultSelection]; + this._selectionBacklog = []; } /** @@ -245,7 +246,7 @@ export class SelectionModel extends Observable { const prepareOptions = (options) => { let actualOptions = addDefaultToOptions(options); if (filter) { - actualOptions = options.filter(filter); + actualOptions = actualOptions.filter(filter); } return actualOptions; }; @@ -267,8 +268,10 @@ export class SelectionModel extends Observable { this._availableOptions = availableOptions; this.visualChange$.notify(); - if (this._selectionBacklog.length) { + const optionsAvailable = availableOptions instanceof RemoteData ? availableOptions.isSuccess() : availableOptions.length > 0; + if (this._selectionBacklog.length && optionsAvailable) { this.selectedOptions = this._selectionBacklog; + this._selectionBacklog = []; this.notify(); } } diff --git a/lib/public/components/common/selection/dropdown/selectionDropdown.js b/lib/public/components/common/selection/dropdown/selectionDropdown.js index 5537fda333..d67dfde38a 100644 --- a/lib/public/components/common/selection/dropdown/selectionDropdown.js +++ b/lib/public/components/common/selection/dropdown/selectionDropdown.js @@ -118,7 +118,7 @@ const dropdownOptions = (dropdownModel, selectorPrefix, displayOption, openingOb NotAsked: () => null, Loading: () => spinner({ size: 2, absolute: false }), Success: (availableOptions) => optionsList(availableOptions, onSelectionChange), - Failure: () => null, + Failure: () => h('.danger.p2', 'Failed to load options'), }) : optionsList(dropdownModel.options, onSelectionChange)), ]; diff --git a/lib/public/services/environment/configurationKeysProvider.js b/lib/public/services/environment/configurationKeysProvider.js new file mode 100644 index 0000000000..95465cf5ba --- /dev/null +++ b/lib/public/services/environment/configurationKeysProvider.js @@ -0,0 +1,30 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +import { getRemoteData } from '../../utilities/fetch/getRemoteData.js'; +import { RemoteDataProvider } from '../RemoteDataProvider.js'; + +/** + * Service class to fetch the keys found in the raw configuration of the environments + */ +export class ConfigurationKeysProvider extends RemoteDataProvider { + /** + * @inheritDoc + */ + async getRemoteData() { + const { data } = await getRemoteData('/api/environments/configurationKeys'); + return data; + } +} + +export const configurationKeysProvider = new ConfigurationKeysProvider(); diff --git a/lib/public/views/Environments/Overview/EnvironmentOverviewModel.js b/lib/public/views/Environments/Overview/EnvironmentOverviewModel.js index 31b291b699..df7f83dfb5 100644 --- a/lib/public/views/Environments/Overview/EnvironmentOverviewModel.js +++ b/lib/public/views/Environments/Overview/EnvironmentOverviewModel.js @@ -14,6 +14,7 @@ import { TimeRangeInputModel } from '../../../components/Filters/common/filters/TimeRangeInputModel.js'; import { RawTextFilterModel } from '../../../components/Filters/common/filters/RawTextFilterModel.js'; import { RawConfigurationFilterModel } from '../../../components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js'; +import { configurationKeysProvider } from '../../../services/environment/configurationKeysProvider.js'; import { coloredEnvironmentStatusComponent } from '../ColoredEnvironmentStatusComponent.js'; import { StatusAcronym } from '../../../domain/enums/statusAcronym.mjs'; import { SelectionModel } from '../../../components/common/selection/SelectionModel.js'; @@ -44,7 +45,7 @@ export class EnvironmentOverviewModel extends FilterableOverviewPageModel { })), }), ids: new RawTextFilterModel(), - rawConfiguration: new RawConfigurationFilterModel(), + rawConfiguration: new RawConfigurationFilterModel(configurationKeysProvider.items$), }, ); } diff --git a/lib/server/controllers/environments.controller.js b/lib/server/controllers/environments.controller.js index 0cf907e47b..2b86f923be 100644 --- a/lib/server/controllers/environments.controller.js +++ b/lib/server/controllers/environments.controller.js @@ -144,6 +144,18 @@ const getEnvironmentById = async (request, response) => { } }; +// eslint-disable-next-line jsdoc/require-param +/** + * Route to fetch all the distinct top-level keys found in the raw configuration JSON of the environments + */ +const getConfigurationKeysHandler = async (_request, response) => { + try { + response.status(200).json({ data: await environmentService.getConfigurationKeys() }); + } catch (error) { + updateExpressResponseFromNativeError(response, error); + } +}; + // eslint-disable-next-line jsdoc/require-param /** * Route to fetch all the logs for a given environment @@ -171,4 +183,5 @@ module.exports = { getAllEnvironments, getEnvironmentById, getAllEnvironmentLogsHandler, + getConfigurationKeysHandler, }; diff --git a/lib/server/routers/environments.router.js b/lib/server/routers/environments.router.js index 56d4066e73..bbf59cec67 100644 --- a/lib/server/routers/environments.router.js +++ b/lib/server/routers/environments.router.js @@ -22,6 +22,11 @@ module.exports = { method: 'post', controller: EnvironmentsController.createEnvironment, }, + { + path: 'configurationKeys', + method: 'get', + controller: EnvironmentsController.getConfigurationKeysHandler, + }, { path: ':envId', children: [ diff --git a/lib/server/services/environment/EnvironmentService.js b/lib/server/services/environment/EnvironmentService.js index f2879b5a01..c1ab75469f 100644 --- a/lib/server/services/environment/EnvironmentService.js +++ b/lib/server/services/environment/EnvironmentService.js @@ -20,6 +20,7 @@ const { isAnEmptyShell } = require('../../utilities/isAnEmptyShell.js'); const { validateAgainstSchema } = require('../../utilities/validateAgainstSchema'); const { CreateEnvironmentHistoryItemDto } = require('../../../domain/dtos/CreateEnvironmentHistoryItemDto'); const { getEnvironmentOrFail } = require('./getEnvironmentOrFail.js'); +const { getEnvironmentsConfigurationKeys } = require('./getEnvironmentsConfigurationKeys.js'); const { dataSource } = require('../../../database/DataSource.js'); const { EnvironmentHistoryItemRepository } = require('../../../database/repositories/index.js'); @@ -65,6 +66,15 @@ class EnvironmentService { return environment ? environmentAdapter.toEntity(environment, false) : null; } + /** + * Returns a list of the distinct top-level keys found in the raw configuration JSON of the environments + * + * @return {Promise} the configuration keys + */ + async getConfigurationKeys() { + return getEnvironmentsConfigurationKeys(); + } + /** * Create an environment in the database with its first history item if current state is provided, and return the created instance * diff --git a/lib/server/services/environment/getEnvironmentsConfigurationKeys.js b/lib/server/services/environment/getEnvironmentsConfigurationKeys.js new file mode 100644 index 0000000000..32f7d97579 --- /dev/null +++ b/lib/server/services/environment/getEnvironmentsConfigurationKeys.js @@ -0,0 +1,47 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { sequelize } = require('../../../database'); +const { cacheAsyncFunction } = require('../../../utilities/cacheAsyncFunction.js'); + +/** + * Query the database for the distinct top-level keys of the environments' raw configurations + * + * @return {Promise} the configuration keys + */ +const fetchConfigurationKeys = async () => { + const [rows] = await sequelize.query(` + SELECT DISTINCT JSON_KEYS(raw_configuration) AS \`keys\` + FROM environments + WHERE JSON_VALID(raw_configuration) + `); + + const keys = new Set(); + for (const { keys: serializedKeys } of rows) { + for (const key of JSON.parse(serializedKeys) ?? []) { + keys.add(key); + } + } + + return [...keys]; +}; + +/** + * Returns the distinct top-level keys of the environments raw configuration + * + * The database is queried only once and then the same result is returned until server restart. + * If the query fails, the next call retries it. + * + * @return {Promise} the configuration keys + */ +exports.getEnvironmentsConfigurationKeys = cacheAsyncFunction(fetchConfigurationKeys, { cacheRejections: false }); diff --git a/lib/utilities/cacheAsyncFunction.js b/lib/utilities/cacheAsyncFunction.js index 765107f807..31cfc4fd55 100644 --- a/lib/utilities/cacheAsyncFunction.js +++ b/lib/utilities/cacheAsyncFunction.js @@ -25,12 +25,14 @@ const DEFAULT_FUNCTION_CACHE_SIZE = 1000; * @param {Object} [configuration={}] the caching configuration * @param {Function} [configuration.argsToCacheKey] method applied on arguments to obtain a unique cache key * @param {Function} [configuration.cacheSize] maximum amount of functions called to cache + * @param {boolean} [configuration.cacheRejections=true] if false, a call that rejects is removed from the cache so the next call retries * @return {Function} the cached function */ exports.cacheAsyncFunction = (toCache, configuration) => { const { argsToCacheKey = (...args) => JSON.stringify({ ...args }), cacheSize = DEFAULT_FUNCTION_CACHE_SIZE, + cacheRejections = true, } = configuration || {}; const cache = new Map(); @@ -43,6 +45,15 @@ exports.cacheAsyncFunction = (toCache, configuration) => { cache.set(cacheKey, resultPromise); keysQueue.push(cacheKey); + if (!cacheRejections) { + resultPromise.catch(() => { + if (cache.get(cacheKey) === resultPromise) { + cache.delete(cacheKey); + keysQueue.splice(keysQueue.indexOf(cacheKey), 1); + } + }); + } + if (keysQueue.length > cacheSize) { const oldestKey = keysQueue.shift(); cache.delete(oldestKey); diff --git a/test/api/environments.test.js b/test/api/environments.test.js index 05d4535342..4872488141 100644 --- a/test/api/environments.test.js +++ b/test/api/environments.test.js @@ -351,6 +351,17 @@ module.exports = () => { expect(response.status).to.equal(400); }); }); + describe('GET /api/environments/configurationKeys', () => { + it('should successfully return a 200 response containing the distinct keys of the environments\' raw configurations', async () => { + const response = await request(server).get('/api/environments/configurationKeys'); + expect(response.status).to.equal(200); + + const { data } = response.body; + expect(data).to.include.members(['ccdb_enabled', 'dcs_enabled', 'detectors', 'hosts', 'run_type']); + expect(data).to.eql([...new Set(data)]); + }); + }); + describe('POST /api/environments', () => { it('should return 201 if valid data is provided', async () => { const response = await request(server).post('/api/environments').send({ diff --git a/test/lib/utilities/cacheAsyncFunction.test.js b/test/lib/utilities/cacheAsyncFunction.test.js index 9b18fdbd46..3715989aa7 100644 --- a/test/lib/utilities/cacheAsyncFunction.test.js +++ b/test/lib/utilities/cacheAsyncFunction.test.js @@ -73,6 +73,38 @@ module.exports = () => { expect(fake.callCount).to.equal(2); }); + it('should not cache a rejecting function when cacheRejections is false', async () => { + const fake = sinon.fake(); + let shouldReject = true; + + const sometimesRejects = async (value) => { + fake(); + if (shouldReject) { + throw new Error(value); + } + return value; + }; + const cachedSometimesRejects = cacheAsyncFunction(sometimesRejects, { cacheRejections: false }); + + await assert.rejects( + () => cachedSometimesRejects(0), + new Error(0), + ); + await assert.rejects( + () => cachedSometimesRejects(0), + new Error(0), + ); + + expect(fake.callCount).to.equal(2); + + shouldReject = false; + + expect(await cachedSometimesRejects(0)).to.equal(0); + expect(await cachedSometimesRejects(0)).to.equal(0); + + expect(fake.callCount).to.equal(3); + }); + it('should successfully delete function call when cache overflows', async () => { const fake = sinon.fake(); // eslint-disable-next-line jsdoc/require-param diff --git a/test/public/envs/overview.test.js b/test/public/envs/overview.test.js index 6bb17c580c..7e70114ecc 100644 --- a/test/public/envs/overview.test.js +++ b/test/public/envs/overview.test.js @@ -303,10 +303,8 @@ module.exports = () => { await expectAttributeValue(page, '.id-filter input', 'placeholder', 'e.g. CmCvjNbg, TDI59So3d...'); await expectAttributeValue(page, '.runs-filter input', 'placeholder', 'e.g. 553203, 553221, ...'); await expectAttributeValue(page, '.historyItems-filter input', 'placeholder', 'e.g. D-R-X'); - await expectAttributeValue(page, '#rawConfiguration-key', 'placeholder', 'e.g. hosts'); + await expectInnerText(page, '.rawConfiguration-filter .dropdown-trigger', 'Any key'); await expectAttributeValue(page, '#rawConfiguration-value', 'placeholder', 'Search anywhere in the configuration'); - // The raw configuration column is only displayed when filtering on a key - expect(await page.$('#rowTDI59So3d-rawConfiguration')).to.be.null; // range of runNumbers await fillInput(page, '.runs-filter input', '103-104', ['change']); @@ -324,11 +322,12 @@ module.exports = () => { await fillInput(page, fromDateTimeSelector, '2019-08-09T00:00', ['change']); await fillInput(page, toDateTimeSelector, '2019-08-10T23:59', ['change']); - // Typing a key switches the value placeholder from the free-text search, without applying the filter - await fillInput(page, '#rawConfiguration-key', 'hosts', ['input']); - await page.waitForSelector('#rawConfiguration-value[placeholder="e.g. some-host"]'); - - await fillInput(page, '#rawConfiguration-key', 'hosts', ['change']); + // The keys are searchable, and selecting one changes the value placeholder to make more sense + await pressElement(page, '.rawConfiguration-filter .dropdown-trigger'); + await fillInput(page, '#rawConfiguration-key-dropdown-search-input', 'HOST'); + await page.waitForSelector('#rawConfiguration-key-dropdown-option-run_type', { hidden: true }); + await pressElement(page, '#rawConfiguration-key-dropdown-option-hosts', true); + await page.waitForSelector('#rawConfiguration-value[placeholder="e.g. host.123@flp.ch"]'); // Go through an empty result first, otherwise the expected single row is already displayed before the filter applies await fillInput(page, '#rawConfiguration-value', 'fake-flp1', ['change']); @@ -345,22 +344,33 @@ module.exports = () => { await openFilteringPanel(page); // No host contains ITS, but the detectors of CmCvjNbg do - await fillInput(page, '#rawConfiguration-key', 'hosts', ['change']); + await pressElement(page, '.rawConfiguration-filter .dropdown-trigger'); + await pressElement(page, '#rawConfiguration-key-dropdown-option-hosts', true); await fillInput(page, '#rawConfiguration-value', 'ITS', ['change']); await waitForEmptyTable(page, 5000); - await fillInput(page, '#rawConfiguration-key', '', ['change']); + // Deselecting the key so should search the value anywhere + await page.waitForSelector('#rawConfiguration-key-dropdown-option-hosts:checked'); + await pressElement(page, '#rawConfiguration-key-dropdown-option-hosts', true); await page.waitForSelector('tbody tr#rowCmCvjNbg'); - await expectInputValue(page, '#rawConfiguration-value', 'ITS'); }); it('should search a raw configuration value without key from the URL anywhere in the configuration', async () => { await goToPage(page, 'env-overview', { queryParameters: { filter: { rawConfiguration: { value: 'fake-flp3' } } } }); await openFilteringPanel(page); - await expectInputValue(page, '#rawConfiguration-key', ''); + await expectInnerText(page, '.rawConfiguration-filter .dropdown-trigger', 'Any key'); await expectInputValue(page, '#rawConfiguration-value', 'fake-flp3'); await page.waitForSelector('tbody tr#rowTDI59So3d'); await waitForTableLength(page, 1); }); + + it('should select a raw configuration key from the URL once the keys are loaded', async () => { + await goToPage(page, 'env-overview', { queryParameters: { filter: { rawConfiguration: { key: 'hosts', value: 'fake-flp' } } } }); + await openFilteringPanel(page); + + await expectInnerText(page, '.rawConfiguration-filter .dropdown-trigger', 'hosts'); + await page.waitForSelector('tbody tr#rowTDI59So3d'); + await page.waitForSelector('tbody tr#rowCmCvjNbg'); + }); };