O2B-1646 - Make envs config filter a searchable select

lib/public/components/Filters/EnvironmentsFilter/KeyFilterModel.js

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+import { ObservableBasedSelectionDropdownModel } from '../../detector/ObservableBasedSelectionDropdownModel.js';
+import { keyToOption } from './keyToOption.js';
+
+/**
+ * Model storing state of a single key picked from the list of all the keys found in the environments raw configuration
+ */
+export class KeyFilterModel extends ObservableBasedSelectionDropdownModel {
+    /**
+     * Constructor
+     *
+     * @param {ObservableData<RemoteData<string[], ApiError>>} keys$ observable remote data of keys list
+     */
+    constructor(keys$) {
+        super(keys$, keyToOption, { multiple: false });
+    }
+}

lib/public/components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js

@@ -12,6 +12,7 @@
  */
 
 import { FilterModel } from '../common/FilterModel.js';
+import { KeyFilterModel } from './KeyFilterModel.js';
 import { RawTextFilterModel } from '../common/filters/RawTextFilterModel.js';
 
 /**
@@ -20,11 +21,13 @@ import { RawTextFilterModel } from '../common/filters/RawTextFilterModel.js';
 export class RawConfigurationFilterModel extends FilterModel {
     /**
      * Constructor
+     *
+     * @param {ObservableData<RemoteData<string[], ApiError>>} configurationKeys$ observable remote data of the existing configuration keys
      */
-    constructor() {
+    constructor(configurationKeys$) {
         super();
 
-        this._keyModel = new RawTextFilterModel();
+        this._keyModel = new KeyFilterModel(configurationKeys$);
         this._valueModel = new RawTextFilterModel();
 
         this._addSubmodel(this._keyModel);
@@ -72,16 +75,16 @@ export class RawConfigurationFilterModel extends FilterModel {
     }
 
     /**
-     * Returns the key currently typed in the key input, trimmed and without surrounding quotes
+     * Returns the currently selected key, empty if none
      *
      * @return {string} the key
      */
     get key() {
-        return this._keyModel.value.trim().replace(/^"(.*)"$/, '$1');
+        return this._keyModel.selected[0] ?? '';
     }
 
     /**
-     * Returns the value currently typed in the value input, trimmed
+     * Returns the currently typed in value
      *
      * @return {string} the value
      */
@@ -92,7 +95,7 @@ export class RawConfigurationFilterModel extends FilterModel {
     /**
      * Returns the key filter model
      *
-     * @return {RawTextFilterModel} the key filter model
+     * @return {KeyFilterModel} the key filter model
      */
     get keyModel() {
         return this._keyModel;

lib/public/components/Filters/EnvironmentsFilter/keyToOption.js

@@ -0,0 +1,21 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+/**
+ * Convert a raw configuration key to a selection option
+ * The raw key value is best for mapping to the configuration
+ *
+ * @param {string} key the key to convert
+ * @return {SelectionOption} the key's option
+ */
+export const keyToOption = (key) => ({ label: key, value: key });

lib/public/components/Filters/EnvironmentsFilter/rawConfigurationFilter.js

@@ -13,6 +13,7 @@
 
 import { h } from '/js/src/index.js';
 import { rawTextFilter } from '../common/filters/rawTextFilter.js';
+import { selectionDropdown } from '../../common/selection/dropdown/selectionDropdown.js';
 
 /**
  * Renders a filter for the raw configuration of environments, allowing users to filter environments on a key, optionally with a value,
@@ -25,21 +26,14 @@ export const rawConfigurationFilter = (rawConfigurationFilterModel) =>
     h('.flex-row.g2', [
         h(
             '.flex-column.flex-grow',
-            // Bubble to update the value placeholder when a key is typed
-            { oninput: () => rawConfigurationFilterModel.visualChange$.notify() },
-            [
-                rawTextFilter(
-                    rawConfigurationFilterModel.keyModel,
-                    { id: 'rawConfiguration-key', placeholder: 'e.g. hosts' },
-                ),
-            ],
+            selectionDropdown(rawConfigurationFilterModel.keyModel, { selectorPrefix: 'rawConfiguration-key', placeholder: 'Any key' }),
         ),
         h('.flex-column.flex-grow', [
             rawTextFilter(
                 rawConfigurationFilterModel.valueModel,
                 {
                     id: 'rawConfiguration-value',
-                    placeholder: rawConfigurationFilterModel.key ? 'e.g. some-host' : 'Search anywhere in the configuration',
+                    placeholder: rawConfigurationFilterModel.key ? 'e.g. host.123@flp.ch' : 'Search anywhere in the configuration',
                 },
             ),
         ]),

lib/public/components/common/selection/SelectionModel.js

@@ -129,6 +129,7 @@ export class SelectionModel extends Observable {
      */
     reset() {
         this._selectedOptions = [...this._defaultSelection];
+        this._selectionBacklog = [];
     }
 
     /**
@@ -245,7 +246,7 @@ export class SelectionModel extends Observable {
         const prepareOptions = (options) => {
             let actualOptions = addDefaultToOptions(options);
             if (filter) {
-                actualOptions = options.filter(filter);
+                actualOptions = actualOptions.filter(filter);
             }
             return actualOptions;
         };
@@ -267,8 +268,10 @@ export class SelectionModel extends Observable {
         this._availableOptions = availableOptions;
         this.visualChange$.notify();
 
-        if (this._selectionBacklog.length) {
+        const optionsAvailable = availableOptions instanceof RemoteData ? availableOptions.isSuccess() : availableOptions.length > 0;
+        if (this._selectionBacklog.length && optionsAvailable) {
             this.selectedOptions = this._selectionBacklog;
+            this._selectionBacklog = [];
             this.notify();
         }
     }

lib/public/components/common/selection/dropdown/selectionDropdown.js

@@ -118,7 +118,7 @@ const dropdownOptions = (dropdownModel, selectorPrefix, displayOption, openingOb
                 NotAsked: () => null,
                 Loading: () => spinner({ size: 2, absolute: false }),
                 Success: (availableOptions) => optionsList(availableOptions, onSelectionChange),
-                Failure: () => null,
+                Failure: () => h('.danger.p2', 'Failed to load options'),
             })
             : optionsList(dropdownModel.options, onSelectionChange)),
     ];

lib/public/services/environment/configurationKeysProvider.js

@@ -0,0 +1,30 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+import { getRemoteData } from '../../utilities/fetch/getRemoteData.js';
+import { RemoteDataProvider } from '../RemoteDataProvider.js';
+
+/**
+ * Service class to fetch the keys found in the raw configuration of the environments
+ */
+export class ConfigurationKeysProvider extends RemoteDataProvider {
+    /**
+     * @inheritDoc
+     */
+    async getRemoteData() {
+        const { data } = await getRemoteData('/api/environments/configurationKeys');
+        return data;
+    }
+}
+
+export const configurationKeysProvider = new ConfigurationKeysProvider();

lib/public/views/Environments/Overview/EnvironmentOverviewModel.js

@@ -14,6 +14,7 @@
 import { TimeRangeInputModel } from '../../../components/Filters/common/filters/TimeRangeInputModel.js';
 import { RawTextFilterModel } from '../../../components/Filters/common/filters/RawTextFilterModel.js';
 import { RawConfigurationFilterModel } from '../../../components/Filters/EnvironmentsFilter/RawConfigurationFilterModel.js';
+import { configurationKeysProvider } from '../../../services/environment/configurationKeysProvider.js';
 import { coloredEnvironmentStatusComponent } from '../ColoredEnvironmentStatusComponent.js';
 import { StatusAcronym } from '../../../domain/enums/statusAcronym.mjs';
 import { SelectionModel } from '../../../components/common/selection/SelectionModel.js';
@@ -44,7 +45,7 @@ export class EnvironmentOverviewModel extends FilterableOverviewPageModel {
                     })),
                 }),
                 ids: new RawTextFilterModel(),
-                rawConfiguration: new RawConfigurationFilterModel(),
+                rawConfiguration: new RawConfigurationFilterModel(configurationKeysProvider.items$),
             },
         );
     }

lib/server/controllers/environments.controller.js

@@ -144,6 +144,18 @@ const getEnvironmentById = async (request, response) => {
     }
 };
 
+// eslint-disable-next-line jsdoc/require-param
+/**
+ * Route to fetch all the distinct top-level keys found in the raw configuration JSON of the environments
+ */
+const getConfigurationKeysHandler = async (_request, response) => {
+    try {
+        response.status(200).json({ data: await environmentService.getConfigurationKeys() });
+    } catch (error) {
+        updateExpressResponseFromNativeError(response, error);
+    }
+};
+
 // eslint-disable-next-line jsdoc/require-param
 /**
  * Route to fetch all the logs for a given environment
@@ -171,4 +183,5 @@ module.exports = {
     getAllEnvironments,
     getEnvironmentById,
     getAllEnvironmentLogsHandler,
+    getConfigurationKeysHandler,
 };

lib/server/routers/environments.router.js

@@ -22,6 +22,11 @@ module.exports = {
             method: 'post',
             controller: EnvironmentsController.createEnvironment,
         },
+        {
+            path: 'configurationKeys',
+            method: 'get',
+            controller: EnvironmentsController.getConfigurationKeysHandler,
+        },
         {
             path: ':envId',
             children: [

lib/server/services/environment/EnvironmentService.js

@@ -20,6 +20,7 @@ const { isAnEmptyShell } = require('../../utilities/isAnEmptyShell.js');
 const { validateAgainstSchema } = require('../../utilities/validateAgainstSchema');
 const { CreateEnvironmentHistoryItemDto } = require('../../../domain/dtos/CreateEnvironmentHistoryItemDto');
 const { getEnvironmentOrFail } = require('./getEnvironmentOrFail.js');
+const { getEnvironmentsConfigurationKeys } = require('./getEnvironmentsConfigurationKeys.js');
 const { dataSource } = require('../../../database/DataSource.js');
 const { EnvironmentHistoryItemRepository } = require('../../../database/repositories/index.js');
 
@@ -65,6 +66,15 @@ class EnvironmentService {
         return environment ? environmentAdapter.toEntity(environment, false) : null;
     }
 
+    /**
+     * Returns a list of the distinct top-level keys found in the raw configuration JSON of the environments
+     *
+     * @return {Promise<string[]>} the configuration keys
+     */
+    async getConfigurationKeys() {
+        return getEnvironmentsConfigurationKeys();
+    }
+
     /**
      * Create an environment in the database with its first history item if current state is provided, and return the created instance
      *

lib/server/services/environment/getEnvironmentsConfigurationKeys.js

@@ -0,0 +1,47 @@
+/**
+ * @license
+ * Copyright CERN and copyright holders of ALICE O2. This software is
+ * distributed under the terms of the GNU General Public License v3 (GPL
+ * Version 3), copied verbatim in the file "COPYING".
+ *
+ * See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ * In applying this license CERN does not waive the privileges and immunities
+ * granted to it by virtue of its status as an Intergovernmental Organization
+ * or submit itself to any jurisdiction.
+ */
+
+const { sequelize } = require('../../../database');
+const { cacheAsyncFunction } = require('../../../utilities/cacheAsyncFunction.js');
+
+/**
+ * Query the database for the distinct top-level keys of the environments' raw configurations
+ *
+ * @return {Promise<string[]>} the configuration keys
+ */
+const fetchConfigurationKeys = async () => {
+    const [rows] = await sequelize.query(`
+        SELECT DISTINCT JSON_KEYS(raw_configuration) AS \`keys\`
+        FROM environments
+        WHERE JSON_VALID(raw_configuration)
+    `);
+
+    const keys = new Set();
+    for (const { keys: serializedKeys } of rows) {
+        for (const key of JSON.parse(serializedKeys) ?? []) {
+            keys.add(key);
+        }
+    }
+
+    return [...keys];
+};
+
+/**
+ * Returns the distinct top-level keys of the environments raw configuration
+ *
+ * The database is queried only once and then the same result is returned until server restart.
+ * If the query fails, the next call retries it.
+ *
+ * @return {Promise<string[]>} the configuration keys
+ */
+exports.getEnvironmentsConfigurationKeys = cacheAsyncFunction(fetchConfigurationKeys, { cacheRejections: false });

lib/utilities/cacheAsyncFunction.js

@@ -25,12 +25,14 @@ const DEFAULT_FUNCTION_CACHE_SIZE = 1000;
  * @param {Object} [configuration={}] the caching configuration
  * @param {Function} [configuration.argsToCacheKey] method applied on arguments to obtain a unique cache key
  * @param {Function} [configuration.cacheSize] maximum amount of functions called to cache
+ * @param {boolean} [configuration.cacheRejections=true] if false, a call that rejects is removed from the cache so the next call retries
  * @return {Function} the cached function
  */
 exports.cacheAsyncFunction = (toCache, configuration) => {
     const {
         argsToCacheKey = (...args) => JSON.stringify({ ...args }),
         cacheSize = DEFAULT_FUNCTION_CACHE_SIZE,
+        cacheRejections = true,
     } = configuration || {};
 
     const cache = new Map();
@@ -43,6 +45,15 @@ exports.cacheAsyncFunction = (toCache, configuration) => {
             cache.set(cacheKey, resultPromise);
             keysQueue.push(cacheKey);
 
+            if (!cacheRejections) {
+                resultPromise.catch(() => {
+                    if (cache.get(cacheKey) === resultPromise) {
+                        cache.delete(cacheKey);
+                        keysQueue.splice(keysQueue.indexOf(cacheKey), 1);
+                    }
+                });
+            }
+
             if (keysQueue.length > cacheSize) {
                 const oldestKey = keysQueue.shift();
                 cache.delete(oldestKey);

test/api/environments.test.js

@@ -351,6 +351,17 @@ module.exports = () => {
             expect(response.status).to.equal(400);
         });
     });
+    describe('GET /api/environments/configurationKeys', () => {
+        it('should successfully return a 200 response containing the distinct keys of the environments\' raw configurations', async () => {
+            const response = await request(server).get('/api/environments/configurationKeys');
+            expect(response.status).to.equal(200);
+
+            const { data } = response.body;
+            expect(data).to.include.members(['ccdb_enabled', 'dcs_enabled', 'detectors', 'hosts', 'run_type']);
+            expect(data).to.eql([...new Set(data)]);
+        });
+    });
+
     describe('POST /api/environments', () => {
         it('should return 201 if valid data is provided', async () => {
             const response = await request(server).post('/api/environments').send({