diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0e430d6af..1ad374fa8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -160,8 +160,6 @@ jobs: include: - testsuite: api:routes runner: run_server_tests.bash - env: - MANAGEMENT_BACKEND: operator - testsuite: api:retry runner: run_server_tests.bash - testsuite: replication diff --git a/docs/configuration.md b/docs/configuration.md index 2dbab5dc0..dfdee1a70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,8 +103,6 @@ is derived for them, and their value is not validated. passed by the deployment rather than configured in the file. - `S3AUTH_CONFIG`: path of the account credentials file, for the `account` auth type. -- `MANAGEMENT_BACKEND`, `REMOTE_MANAGEMENT_DISABLE`: management backend of the - Zenko deployment, and whether to run it. - `LIFECYCLE_OBJECT_PROCESSOR_TYPE`: the lifecycle object tasks this processor consumes, `expiration` (the default) or `transition`. - `LIFECYCLE_MAX_AUTO_INDEX_DOC_COUNT`, diff --git a/docs/run-oob-locally.md b/docs/run-oob-locally.md index d476c57a6..30a455f89 100644 --- a/docs/run-oob-locally.md +++ b/docs/run-oob-locally.md @@ -124,19 +124,19 @@ Add Ring access key and secret key in conf/locationConfig.json: Start backbeat API: ``` -MANAGEMENT_BACKEND=operator REMOTE_MANAGEMENT_DISABLE=true yarn start +REMOTE_MANAGEMENT_DISABLE=true yarn start ``` ## Backbeat - Ingestion producer ``` -MANAGEMENT_BACKEND=operator yarn ingestion_populator +yarn ingestion_populator ``` ## Backbeat - Ingestion processor ``` -MANAGEMENT_BACKEND=operator yarn mongo_queue_processor +yarn mongo_queue_processor ``` ## VaultClient diff --git a/extensions/ingestion/IngestionQueuePopulator.js b/extensions/ingestion/IngestionQueuePopulator.js index c5928ebac..c39935533 100644 --- a/extensions/ingestion/IngestionQueuePopulator.js +++ b/extensions/ingestion/IngestionQueuePopulator.js @@ -1,5 +1,3 @@ -const { zenkoIDHeader } = require('@scality/arsenal').constants; - const QueuePopulatorExtension = require('../../lib/queuePopulator/QueuePopulatorExtension'); const { isMasterKey } = require('@scality/arsenal').versioning; @@ -10,7 +8,6 @@ class IngestionQueuePopulator extends QueuePopulatorExtension { constructor(params) { super(params); this.config = params.config; - this._instanceId = params.instanceId; } // called by _processLogEntry in lib/queuePopulator/LogReader.js @@ -73,10 +70,6 @@ class IngestionQueuePopulator extends QueuePopulatorExtension { }); return true; } - if (this._isRetroPropagationEntry(queueEntry)) { - this.log.trace('skipping retro-propagated entry'); - return true; - } if (this._isMasterKeyEntry(queueEntry)) { this.log.trace('skipping master key entry'); return true; @@ -100,41 +93,6 @@ class IngestionQueuePopulator extends QueuePopulatorExtension { return false; } - /** - * Retro-propagation is where S3C ingestion will re-ingest an object whose - * request originated from Zenko. Filter these entries indicated by user - * metadata field defined by constants.zenkoIDHeader - * @param {ObjectQueueEntry} entry - object queue entry instance - * @return {Boolean} true if we should filter entry - */ - _isRetroPropagationEntry(entry) { - const userMD = entry.getUserMetadata(); - let existingIDHeader; - if (userMD) { - try { - const metaHeaders = JSON.parse(userMD); - existingIDHeader = metaHeaders[zenkoIDHeader]; - } catch { - this.log.trace('malformed user metadata', { - method: 'IngestionQueuePopulator.filter', - bucket: entry.bucket, - key: entry.key, - type: entry.type, - }); - return true; - } - // if user metadata field of `constants.zenkoIDHeader` - // exists and value is either 'zenko' (for delete markers) - // or matches given hashed instance id - if (existingIDHeader && (existingIDHeader === 'zenko' || - existingIDHeader === this._instanceId)) { - this.log.trace('skipping retro-propagated entry'); - return true; - } - } - return false; - } - /** * Filter if the entry is considered a master key entry. * There is a case where a single null entry looks like a master key and diff --git a/lib/Config.js b/lib/Config.js index fa783caf7..d67b6fdd1 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -6,7 +6,6 @@ const joi = require('joi'); const fs = require('fs'); const path = require('path'); -const crypto = require('crypto'); const extensions = require('../extensions'); const { backbeatConfigJoi } = require('./config.joi'); @@ -271,16 +270,6 @@ class Config extends EventEmitter { return this.transientLocations[locationName] || false; } - getPublicInstanceId() { - return this.publicInstanceId; - } - - setPublicInstanceId(instanceId) { - this.publicInstanceId = crypto.createHash('sha256') - .update(instanceId) - .digest('hex'); - } - /** * returns the queue processor's site specific destination configuration * @param {string} site - site name diff --git a/lib/management/convertServiceStateFormat.js b/lib/management/convertServiceStateFormat.js deleted file mode 100644 index 63d7d345b..000000000 --- a/lib/management/convertServiceStateFormat.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * convert legacy service state format into new format (for legacy - * replication state) - * - * @param {object} state - service state object - * @return {object} - converted service state object - * - * @note this function should be removed once the new format is - * adopted for replication - */ -function convertServiceStateFormat(state) { - if (!state.streams) { - return state; - } - const convState = Object.assign({}, state); - const workflows = {}; - Object.keys(state.streams).forEach(streamId => { - const stream = Object.assign({}, state.streams[streamId]); - stream.workflowId = stream.streamId; - delete stream.streamId; - const bucketName = stream.source.bucketName; - if (!workflows[bucketName]) { - workflows[bucketName] = {}; - } - workflows[bucketName][streamId] = stream; - }); - convState.workflows = workflows; - delete convState.streams; - return convState; -} - -module.exports = convertServiceStateFormat; diff --git a/lib/management/index.js b/lib/management/index.js index f19bb10cc..a5f8428d0 100644 --- a/lib/management/index.js +++ b/lib/management/index.js @@ -1,26 +1,3 @@ -const config = require('../Config'); -const werelogs = require('werelogs'); - -werelogs.configure({ - level: config.log.logLevel, - dump: config.log.dumpLevel, -}); - -const logger = new werelogs.Logger('management:facade'); - -function loadManagementBackend() { - const backendName = process.env.MANAGEMENT_BACKEND; - const backend = backendName === 'operator' ? - 'operatorBackend' : 'pensieveBackend'; - - const mod = require(`./${backend}`); - - if (!mod) { - logger.error('management backend not found', { backendName }); - process.exit(1); - } - - return mod; -} - -module.exports = loadManagementBackend(); +// The operator (zenko-operator) backend is the only management backend: +// MANAGEMENT_BACKEND is still set by deployments but no longer selects anything. +module.exports = require('./operatorBackend'); diff --git a/lib/management/patchConfiguration.js b/lib/management/patchConfiguration.js index f2729d341..3c901359f 100644 --- a/lib/management/patchConfiguration.js +++ b/lib/management/patchConfiguration.js @@ -1,14 +1,9 @@ 'use strict'; -const async = require('async'); const config = require('../Config'); const bucketclient = require('@scality/bucketclient'); -const { patchLocations } = require('@scality/arsenal').patches.locationConstraints; const Metadata = require('@scality/arsenal').storage.metadata.MetadataWrapper; -const managementDatabaseName = 'PENSIEVE'; -const tokenConfigurationKey = 'auth/zenko/remote-management-token'; const refreshInterval = 5000; -let patchedLocations = {}; function updateIngestionBuckets(locations, metadata, logger, cb) { metadata.getIngestionBuckets(logger, (err, buckets) => { @@ -24,61 +19,6 @@ function updateIngestionBuckets(locations, metadata, logger, cb) { }); } -function updateLocations(locations) { - const locationsWithReplicationBackend = Object.keys(locations) - // NOTE: In Orbit, we don't need to have Scality location in our - // destination bootstrapList config, since we do not replicate to - // any Scality Instance yet. - .filter(key => locations[key].locationType !== 'location-file-v1') - .reduce((obj, key) => { - /* eslint no-param-reassign:0 */ - obj[key] = locations[key]; - return obj; - }, {}); - config.setBootstrapList(locationsWithReplicationBackend); - - Object.keys(locations).forEach(locName => { - config.setIsTransientLocation( - locName, locations[locName].isTransient); - }); -} - -function patchConfiguration( - overlayVersion, patchConfig, metadata, ingestionEnabled, logger, cb -) { - if (patchConfig === undefined || - patchConfig.version === undefined || - patchConfig.locations === undefined - ) { - return process.nextTick(cb, null); - } - - const conf = Object.assign({}, patchConfig); - return async.waterfall([ - next => metadata.getObjectMD( - managementDatabaseName, - tokenConfigurationKey, - {}, - logger, - next, - ), - (creds, next) => { - // only update locations on new overlay version - if (overlayVersion === undefined || conf.version > overlayVersion) { - patchedLocations = patchLocations(conf.locations, creds, logger); - config.setPublicInstanceId(conf.instanceId); - updateLocations(patchedLocations); - } - - // only update ingestion buckets list when ingestionEnabled - if (ingestionEnabled) { - return updateIngestionBuckets(patchedLocations, metadata, logger, next); - } - return process.nextTick(next); - }, - ], cb); -} - function buildMetadataParams(c) { const groupId = c.extensions.replication.replicationStatusProcessor.groupId; const mongo = c.queuePopulator.mongo; @@ -148,8 +88,6 @@ function periodicallyUpdateIngestionBuckets(locations, logger, cb) { } module.exports = { - patchConfiguration, - updateLocations, updateIngestionBuckets, buildMetadataParams, periodicallyUpdateIngestionBuckets, diff --git a/lib/management/pensieveBackend.js b/lib/management/pensieveBackend.js deleted file mode 100644 index a0b94c11c..000000000 --- a/lib/management/pensieveBackend.js +++ /dev/null @@ -1,196 +0,0 @@ -const async = require('async'); -const arsenal = require('@scality/arsenal'); -const { decryptSecret } = arsenal.pensieve.credentialUtils; -const config = require('../Config'); -const werelogs = require('werelogs'); -const bucketclient = require('@scality/bucketclient'); - -const { BaseServiceState } = require('./serviceState'); -const convertOverlayFormat = require('./convertOverlayFormat'); -const convertServiceStateFormat = require('./convertServiceStateFormat'); -const { patchConfiguration, buildMetadataParams, refreshInterval } = require('./patchConfiguration'); -const { reshapeExceptionError } = arsenal.errorUtils; - -werelogs.configure({ - level: config.log.logLevel, - dump: config.log.dumpLevel, -}); -const logger = new werelogs.Logger('management:pensieveBackend'); - -const serviceBucket = 'PENSIEVE'; - -const Metadata = arsenal.storage.metadata.MetadataWrapper; -let metadata; - -let serviceName; -let overlayVersion; - -// let initialized = false; -let serviceCredentials = { - accounts: [], -}; - -const tokenKey = 'auth/zenko/remote-management-token'; - -function saveServiceCredentials(conf, params, instanceAuth) { - // TODO use a proper account id in arn - serviceCredentials = { - accounts: (conf.users || []) - .filter(u => u.accountType === params.serviceAccount) - .map(u => ({ - name: u.accountType, - accountType: u.accountType, - arn: 'aws::iam:234456789012:root', - canonicalID: u.canonicalId, - displayName: u.userName, - keys: { - access: u.accessKey, - secret: decryptSecret(instanceAuth, u.secretKey), - }, - })), - }; - // initialized = true; -} - -function loadOverlayVersion(metadata, version, cb) { - metadata.getObjectMD(serviceBucket, `configuration/overlay/${version}`, {}, - logger, (err, val) => { - if (err) { - return cb(err); - } - const convConf = convertOverlayFormat(val); - logger.debug('converted overlay config to newest format'); - return cb(null, convConf); - }); -} - -class PensieveServiceState extends BaseServiceState { - constructor(serviceName) { - super(serviceName); - this.key = `configuration/state/${this.serviceName}`; - } - - load(cb) { - logger.debug(`loading ${this.serviceName} state`); - metadata.getObjectMD( - serviceBucket, this.key, - {}, logger, (err, currentStateSerialized) => { - if (err && err.NoSuchKey) { - return cb(null, this.getInitialState()); - } - if (err) { - return cb(err); - } - const currentState = JSON.parse(currentStateSerialized); - const convState = convertServiceStateFormat(currentState); - logger.debug(`converted ${this.serviceName} state`, - { convState }); - return cb(null, convState); - }); - } - - save(newState, cb) { - logger.debug(`saving ${this.serviceName} state`, { newState }); - return metadata.putObjectMD(serviceBucket, this.key, - JSON.stringify(newState), {}, logger, cb); - } -} - -/** - * Initialize Orbit management layer - * - * @param {object} params - params object - * @param {string} params.serviceName - name of service to manage - * @param {string} [params.serviceAccount] - name of managed service - * account, if any - * @param {function} [params.applyBucketWorkflows] - called when a - * bucket has a changed set of workflows that needs to be applied by - * the service: applyBucketWorkflows(bucketName, bucketWorkflows, - * workflowUpdates, cb) - * @param {function} done - callback function when init is complete - * @return {undefined} - */ -function initManagement(params, done) { - if (process.env.REMOTE_MANAGEMENT_DISABLE && - process.env.REMOTE_MANAGEMENT_DISABLE !== '0') { - logger.info('remote management disabled'); - return process.nextTick(done); - } - const ingestionEnabled = params.enableIngestionUpdates; - serviceName = params.serviceName; - const mdParams = buildMetadataParams(config); - let setup = false; - function iterate(done) { - const serviceState = new PensieveServiceState(serviceName); - return async.waterfall([ - cb => { - if (!setup) { - setup = true; - metadata = new Metadata('mongodb', mdParams, bucketclient, - logger); - return metadata.setup(() => cb()); - } - return process.nextTick(cb); - }, - cb => { - metadata.getObjectMD(serviceBucket, - 'configuration/overlay-version', {}, logger, (err, res) => - cb(err, res)); - }, - (version, cb) => loadOverlayVersion(metadata, version, cb), - (conf, cb) => patchConfiguration(overlayVersion, conf, metadata, - ingestionEnabled, logger, err => { - if (err) { - return cb(err); - } - if (conf.version) { - overlayVersion = conf.version; - } - return cb(null, conf); - }), - (conf, cb) => metadata.getObjectMD(serviceBucket, tokenKey, {}, - logger, (err, instanceAuth) => { - if (err) { - return cb(err); - } - saveServiceCredentials(conf, params, instanceAuth); - return cb(null, conf); - }), - (conf, cb) => serviceState.apply(conf, params, cb), - ], done); - } - - iterate(err => { - if (err) { - return done(reshapeExceptionError(err)); - } - setInterval(iterate, refreshInterval, err => { - if (err) { - logger.error('error refreshing mgdb', { error: - reshapeExceptionError(err) }); - } - }); - return done(); - }); - return undefined; -} - -/** - * - * @param {String} serviceName - Name of the service to persist state for - * @param {Object} zkClient - Zookeeper client - * @returns {BaseServiceState} - the service state applier - */ -function createServiceState(serviceName, zkClient) { // eslint-disable-line no-unused-vars - return new PensieveServiceState(serviceName); -} - -function getLatestServiceAccountCredentials() { - return serviceCredentials; -} - -module.exports = { - initManagement, - getLatestServiceAccountCredentials, - createServiceState, -}; diff --git a/lib/management/serviceState.js b/lib/management/serviceState.js index 07da676f9..105e79053 100644 --- a/lib/management/serviceState.js +++ b/lib/management/serviceState.js @@ -9,7 +9,7 @@ const getWorkflowUpdates = require('./getWorkflowUpdates'); const logger = new werelogs.Logger('mdManagementServiceState'); const notImplementedErr = errors.NotImplemented.customizeDescription( - `not implemented for management backend '${process.env.MANAGEMENT_BACKEND}'`); + 'not implemented by the service state backend'); function getServiceWorkflows(serviceName) { const workflows = [serviceName]; diff --git a/lib/queuePopulator/IngestionPopulator.js b/lib/queuePopulator/IngestionPopulator.js index 98a0fef8b..0f42ce649 100644 --- a/lib/queuePopulator/IngestionPopulator.js +++ b/lib/queuePopulator/IngestionPopulator.js @@ -623,7 +623,6 @@ class IngestionPopulator { const ext = new index.queuePopulatorExtension({ config: this.ingestionConfig, logger: this.log, - instanceId: config.getPublicInstanceId(), }); ext.setZkConfig(this.zkConfig); this._extension = ext; diff --git a/tests/functional/ingestion/DummyPensieveCredentials.json b/tests/functional/ingestion/DummyPensieveCredentials.json deleted file mode 100644 index 9eb09a49d..000000000 --- a/tests/functional/ingestion/DummyPensieveCredentials.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "_id" : "configuration/overlay/6", - "value" : { - "browserAccess" : { - "enabled" : true - }, - "creator" : "google-118404886697410081890", - "endpoints" : [ - { - "hostname" : "zenko-cloudserver-replicator", - "isBuiltin" : true, - "locationName" : "us-east-1" - }, - { - "hostname" : "dummy.orbit.ui.io", - "isBuiltin" : true, - "locationName" : "us-east-1" - } - ], - "instanceId" : "729d16d8-53d4-4ba6-8371-f3ed2ca4724d", - "locations" : { - "my-zenko-location" : { - "details" : { - "bootstrapList" : [] - }, - "locationType" : "location-file-v1", - "name" : "my-zenko-location", - "objectId" : "ebfe5e6d-2bc8-11e9-b1ef-3a1c209383de" - }, - "nonginx" : { - "details" : { - "accessKey" : "accessKey", - "bootstrapList" : [], - "bucketMatch" : true, - "bucketName" : "myringbucket", - "endpoint" : "testbucket1.s3-us-east-1.amazonaws.com", - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==" - }, - "locationType" : "location-scality-ring-s3-v1", - "name" : "nonginx", - "objectId" : "5a6ac062-2a60-11e9-90c1-0242ac100002" - }, - "s3clocation" : { - "details" : { - "accessKey" : "accessKey", - "bootstrapList" : [], - "bucketMatch" : true, - "bucketName" : "myringbucket", - "endpoint" : "testbucket2.s3-us-east-2.amazonaws.com", - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==" - }, - "locationType" : "location-scality-ring-s3-v1", - "name" : "s3clocation", - "objectId" : "80bcfe9d-2a59-11e9-90c1-0242ac100002" - }, - "test-local" : { - "details" : { - "bootstrapList" : [] - }, - "locationType" : "location-file-v1", - "name" : "test-local", - "objectId" : "5b3de111-2b31-11e9-8047-0242ac100004" - }, - "us-east-1" : { - "isBuiltin" : true, - "locationType" : "location-file-v1", - "name" : "us-east-1", - "objectId" : "d4afe66f-2a54-11e9-ba1c-068e630ke77e" - } - }, - "replicationStreams" : [], - "updatedAt" : "2019-02-08T17:42:36Z", - "users" : [ - { - "accessKey" : "accessKey", - "accountType" : "service-replication", - "canonicalId" : "1c83d73385bbfecbb0f2fbfa7f9a8fd004a5658cbae94ea8b247a55bdaffbb4b/replication", - "isBuiltin" : true, - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "Replication Service Account" - }, - { - "accessKey" : "accessKey", - "accountType" : "service-clueso", - "canonicalId" : "101a653542549820437ecb2abb348741d8b2f7b2892cdd40c59580b4d67fb36/search", - "isBuiltin" : true, - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "Search Service Account" - }, - { - "accessKey" : "accessKey", - "accountType" : "service-lifecycle", - "canonicalId" : "e374579d49585e0f34dda6628047b99d5857d52d360c9f49e5a0b9f8c5513b8b/lifecycle", - "isBuiltin" : true, - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "Lifecycle Service Account" - }, - { - "accessKey" : "accessKey", - "accountType" : "service-gc", - "canonicalId" : "5b02d0d95187dea9fb20885795da10268f1853e109de49e9f3b776f8950bd453/gc", - "isBuiltin" : true, - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "Garbage Collection Service Account" - }, - { - "accessKey" : "accessKey", - "accountType" : "service-md-ingestion", - "canonicalId" : "8079de005d88220f1e8d81d2888a081198fedcd05a068d2a5d067cf20928c131/md-ingestion", - "isBuiltin" : true, - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "Metadata Ingestion Service Account" - }, - { - "accessKey" : "accessKey", - "accountType" : "storage-api", - "canonicalId" : "3ca2b0e1961b304b1cb25b8c877d318d0a13972c2a92f0d2a951b4f991abed47", - "secretKey": "o7Q9X25qRv9KNYUewzlXAmfUmnOycFT9yTdgfk5IMntV8kEg4+mqEYl3QhAXyrCw22vbxvSzgdtjh+YhcZBIC6BL/AWurIh5MZyktbaSQabM3ZobTGuEet+qjog0I6Dr9tjHhxM1tfcOdN5Hy2lQk9LTW5uj2/7rtF6jLn5E1HLEn25sAAy60qPqMjBt+pQ0l6Y4JQ6dymZFCv/lZluEQ2mCdH0WlfDh4ZLcNC0KslwjQVJA4kPS5ydE88bB5m5BMscvzMeeRWkObHwdxnu6xN/YJqXvsx05NIC/G4ioeKdoQrBfuRvhlU1MhoCB/+yESdbVRLoM1MppEKkxQbQLTw==", - "userName" : "testaccount" - } - ], - "version" : 6 - } -} diff --git a/tests/functional/ingestion/DummySSHKey.json b/tests/functional/ingestion/DummySSHKey.json deleted file mode 100644 index ec6a860eb..000000000 --- a/tests/functional/ingestion/DummySSHKey.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "_id" : "auth/zenko/remote-management-token", - "value" : { - "instanceId" : "729d16d8-53d4-4ba6-8371-f3ed2ca4724d", - "issueDate" : "2019-02-06T21:19:02Z", - "publicKey" : "-----BEGIN PUBLIC KEY-----\r\nMIIBIjANBgkqhaiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArgv6xLmCC/YJgYdV3Dzc\r\nnXjoyyQnhParVTm8xZMe0eFaPrVj8wMJACr6yIKBJB7y3bNViieKAe7NA3XrtXzh\r\n1tV9GlcALsyuLmJo5nm1HgfUplsVa0NvAqkrSARJKOFA+lzLXCgZMEISkMU2LY+B\r\nTNXZ8Sx/7wMUvXD21sIzB8ZCMY/xfY8ZWBlcYwMaW2qJuVJniaUQhWaUjlxN8csp\r\nENF4sjz2ZoD56fCpCT8Y/7RnoKddE/NtOGDPaUVh0Jahe3qqXM4H3mnxuX35w+NS\r\nPPPIYBt5wDjgoiNqr6nt1beQp+/kF9RDskp1pbAUOO/IEfa4ThaKgVc0qm0ICGha\r\nnQIDAQAB\r\n-----END PUBLIC KEY-----\r\n", - "token" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOiIyMDE5LTAyLTEzVDIxOjE5OjByWiIsImlhdCI6IjIwMTktMDItMDZUMjE6MTk6MDJaIiwiaW5zdGFuY2VJZCI6IjcyOWQxNmQ4LTUzZDQtNGJhNi04MzcxLWYzZWQyY2E0NzE1ZCIsInJvbGUiOiJpbnN0YW5jZSJ9.zgsgukUAjlDUkkbzJyhlzUPK9ATAR0xdFgIAP4CHJ7Y", - "privateKey" : "-----BEGIN RSA PRIVATE KEY-----\r\nMIIEowIBAAKCAQEArgv6xLmCC/YJgYdV3DzcnXjoyyQnhParVTm8xZMe0eFaPrVj\r\n8wMJACr6yIKBJB7y3bNViieKAe7NA3XrtXzh1tV9GlcALsyuLmJo5nm1HgfUplsV\r\na0NvAqkrSARJKOFA+lzLXCgZMEISkMU2LY+BTNXZ8Sx/7wMUvXD21sIzB8ZCMY/x\r\nfY8ZWBlcYwMaW2qJuVJniaUQhWaUjlxN8cspENF4sjz2ZoD56fCpCT8Y/7RnoKdd\r\nE/NtOGDPaUVh0Jahe3qqXM4H3mnxuX35w+NSPPPIYBt5wDjgoiNqr6nt1beQp+/k\r\nF9RDskp1pbAUOO/IEfa4ThaKgVc0qm0ICGhanQIDAQABAoIBACvLkedssp+l4pB4\r\naI+JtQ3e/d0YFOWtWEGSzMB+m7O+XAIJjMP6kIo9RmusAyj+rKhC1xRzjJOd7nKf\r\nepdpfD6d6+Pz3nToRjAfzTuphnyvBcByjPo3trhZry1o/qvNupcFQJrI8EWnapqJ\r\nNW/Te3rZGRlJ1EtlYBOnWEWECmk6/PAR7Lpd/c0IU6EjU4RtMlSaVlaGp2Zf7w9J\r\nM/mna4rQol/UtBrL9b0KWADUuX1zoAKQ27xFRKGTG7FaOSB05VCsfQHrmdF6VrGM\r\nitVdnyJiORKS0n4XIuCoCR5h1BauHUNZYZLXWUZrrX7qbeDTc7mc9OPGKBVCJrSS\r\naKv2/VUCgYEA1rTZi1rRVNSQ4BRw5muApyreW5eCf5Pg+pQNwGNsiuCAiztHkEBX\r\nMzjbb9MmJuRSDSFewfT2oVZ9NEvYwmVses6mz8Z7WFLrV6NTBBVx2Kn4sA5sGoaE\r\nUNAurg8FCzS/JtC1O2e0NtkUhgF2Ap9NjQ6tJwisbYLczUXHTES+488CgYEAz4U8\r\nsmoLrZXFe63DO1iEJZ8hfjOpDGb/29dQAhQhZ75mrFn1kh6mFLqc9hmZrZk3hESv\r\nGo1sJhusCvCf29dtD7jsasuXMgz9eUl0RZU50KFpidiyEnJrOo1Xy+lrbM/pH6Fy\r\nAHh5Zv9qQIKmgSIkrrWYuujgo2gY4pPObr7uudMCgYEAy5StlaomRJZNTHt1VbZn\r\nRZqrPhOaJFfWgiJyK5Lab5BRsjMiNxAa3rGxuAVw7n2do1+Z1rTsSayViUnFnQXI\r\nZ2Wc2/QqLmRpTC7L31pqGXOJH359WGVl/q53JImPUcHQ4gARZ+N8CGmrQSkPeYsN\r\nYIui+RkLTcdAj87zG0kv6xcCgYB4F1zGywOzEqdUKXx/zXu74dkaFX9QxBJxAKs2\r\naLvkM5VrTCwGwWHPQScwbCQ5CB/OlYjXztu1OfJX6mQ4bvSYUqXOL79xoX7WP6Hf\r\n2rm4eJCAyzxR3JEdM0qJoK0gLTl1qpI0n4NyDYFJj9IXjNC5ojGeiMHnDojIJk0i\r\nXa/9LQKBgH3S3PKsZleQTn/fwfpDvcTxb0wzNtoa1l9sfHzm1u5MnGdJ3wNQD1dx\r\nhxT0/5Ys0QlsOMBCYDA44fAzvkp6Oefnci9D0kLsPY6wjEubypqEpLlvqYg50x94\r\nwokU4HUbmA99DzGJxTzGkHZkZN5wUrezQYAdIxfNRrHcNBqGVi84\r\n-----END RSA PRIVATE KEY-----\r\n" - } -} diff --git a/tests/functional/ingestion/IngestionReader.js b/tests/functional/ingestion/IngestionReader.js index 017e6b045..56b8a17f8 100644 --- a/tests/functional/ingestion/IngestionReader.js +++ b/tests/functional/ingestion/IngestionReader.js @@ -3,13 +3,10 @@ const async = require('async'); const http = require('http'); const kafka = require('node-rdkafka'); const { MetadataMock, mockLogs } = require('../utils/MetadataMock'); -const MongoClient = require('mongodb').MongoClient; const { promisify } = require('util'); const timers = require('timers/promises'); const dummyLogger = require('../../utils/DummyLogger'); -const dummyPensieveCredentials = require('./DummyPensieveCredentials.json'); -const dummySSHKey = require('./DummySSHKey.json'); const { expectedNewIngestionEntry, expectedZeroByteObj, expectedUTF8Obj, expectedVersionIdObj, expectedTagsObj } = require('./expectedEntries'); const IngestionQueuePopulator = @@ -104,15 +101,9 @@ describe('ingestion reader tests with mock', function fD() { let httpServer; let producer; let zkClient; - const mongoUrl = - `mongodb://${testConfig.queuePopulator.mongo.replicaSetHosts}` + - '/db?replicaSet=rs0'; - const client = new MongoClient(mongoUrl, {}); - const db = client.db('metadata', { ignoreUndefined: true }); before(async () => { testConfig.s3.port = testPort; const topic = testConfig.extensions.ingestion.topic; - await client.connect(); try { const createTopic = promisify(kafkaAdminClient.createTopic).bind(kafkaAdminClient); await createTopic({ @@ -137,14 +128,6 @@ describe('ingestion reader tests with mock', function fD() { consumer.subscribe([testConfig.extensions.ingestion.topic]); await timers.setTimeout(2000); - await db.createCollection('PENSIEVE'); - const collection = db.collection('PENSIEVE'); - await collection.insertOne(dummyPensieveCredentials); - await collection.insertOne({ - _id: 'configuration/overlay-version', - value: 6, - }); - await collection.insertOne(dummySSHKey); zkClient = new ZookeeperManager('localhost:2181', { autoCreateNamespace: true }, dummyLogger); await new Promise((resolve, reject) => { zkClient.once('error', reject); @@ -159,8 +142,6 @@ describe('ingestion reader tests with mock', function fD() { after(async () => { await promisify(httpServer.close.bind(httpServer))(); consumer.unsubscribe(); - await db.collection('PENSIEVE').drop(); - await client.close(); }); describe('testing with `bucket1` configuration', () => { diff --git a/tests/unit/ingestion/IngestionQueuePopulator.spec.js b/tests/unit/ingestion/IngestionQueuePopulator.spec.js index a65705692..e1f44bcef 100644 --- a/tests/unit/ingestion/IngestionQueuePopulator.spec.js +++ b/tests/unit/ingestion/IngestionQueuePopulator.spec.js @@ -12,7 +12,6 @@ describe('ingestion queue populator', () => { const params = { config: { topic: 'test-topic' }, logger: fakeLogger, - instanceId: 'test-instance', }; iqp = new IngestionQueuePopulator(params); }); diff --git a/tests/unit/management/convertServiceStateFormat.spec.js b/tests/unit/management/convertServiceStateFormat.spec.js deleted file mode 100644 index 8e8be6b48..000000000 --- a/tests/unit/management/convertServiceStateFormat.spec.js +++ /dev/null @@ -1,94 +0,0 @@ -const assert = require('assert'); - -const convertServiceStateFormat = - require('../../../lib/management/convertServiceStateFormat'); - -describe('convertServiceStateFormat', () => { - it('should convert a legacy state format into the new format', () => { - const legacyState = { - overlayVersion: 42, - streams: { - s1: { - streamId: 's1', - source: { - bucketName: 'b1', - }, - }, - s2: { - streamId: 's2', - source: { - bucketName: 'b2', - }, - }, - s3: { - streamId: 's3', - source: { - bucketName: 'b1', - }, - }, - }, - }; - const expectedConvState = { - overlayVersion: 42, - workflows: { - b1: { - s1: { - workflowId: 's1', - source: { - bucketName: 'b1', - }, - }, - s3: { - workflowId: 's3', - source: { - bucketName: 'b1', - }, - }, - }, - b2: { - s2: { - workflowId: 's2', - source: { - bucketName: 'b2', - }, - }, - }, - }, - }; - - const convState = convertServiceStateFormat(legacyState); - assert.deepStrictEqual(convState, expectedConvState); - }); - - it('should keep the new state format as-is if nothing to convert', () => { - const okState = { - overlayVersion: 42, - workflows: { - b1: { - s1: { - workflowId: 's1', - source: { - bucketName: 'b1', - }, - }, - s3: { - workflowId: 's3', - source: { - bucketName: 'b1', - }, - }, - }, - b2: { - s2: { - workflowId: 's2', - source: { - bucketName: 'b2', - }, - }, - }, - }, - }; - const convState = convertServiceStateFormat(okState); - assert.deepStrictEqual(convState, okState); - }); -}); diff --git a/tests/unit/management/patchConfiguration.js b/tests/unit/management/patchConfiguration.js index e95b5034b..21110a2c3 100644 --- a/tests/unit/management/patchConfiguration.js +++ b/tests/unit/management/patchConfiguration.js @@ -3,32 +3,27 @@ const async = require('async'); const Metadata = require('@scality/arsenal').storage.metadata.MetadataWrapper; const BucketInfo = require('@scality/arsenal').models.BucketInfo; -const { patchConfiguration } = +const { updateIngestionBuckets } = require('../../../lib/management/patchConfiguration'); const Config = require('../../../lib/Config'); const testConfig = require('../../config.json'); const fakeLogger = require('../../utils/fakeLogger'); -const managementDatabaseName = 'PENSIEVE'; -const tokenConfigurationKey = 'auth/zenko/remote-management-token'; -const privateKey = '-----BEGIN RSA PRIVATE KEY-----\r\nMIIEowIBAAKCAQEAj13sSYE40lAX2qpBvfdGfcSVNtBf8i5FH+E8FAhORwwPu+2S\r\n3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12DtxqFRnMA08LfO4oO6oC4V8XfKeuHyJ\r\n1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD5p7D+G26Chbr/Oo0ZwHula9DxXy6\r\neH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2dbBIhovMgjjikf5p2oWqnRKXc+JK\r\nBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1t5V4wfRZea5vwl/HlyyKodvHdxng\r\nJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTDfwIDAQABAoIBAAuDYGlavkRteCzw\r\nRU1LIVcSRWVcgIgDXTu9K8T0Ec0008Kkxomyn6LmxmroJbZ1VwsDH8s4eRH73ckA\r\nxrZxt6Pr+0lplq6eBvKtl8MtGhq1VDe+kJczjHEF6SQHOFAu/TEaPZrn2XMcGvRX\r\nO1BnRL9tepFlxm3u/06VRFYNWqqchM+tFyzLu2AuiuKd5+slSX7KZvVgdkY1ErKH\r\ngB75lPyhPb77C/6ptqUisVMSO4JhLhsD0+ekDVY982Sb7KkI+szdWSbtMx9Ek2Wo\r\ntXwJz7I8T7IbODy9aW9G+ydyhMDFmaEYIaDVFKJj5+fluNza3oQ5PtFNVE50GQJA\r\nsisGqfECgYEAwpkwt0KpSamSEH6qknNYPOwxgEuXWoFVzibko7is2tFPvY+YJowb\r\n68MqHIYhf7gHLq2dc5Jg1TTbGqLECjVxp4xLU4c95KBy1J9CPAcuH4xQLDXmeLzP\r\nJ2YgznRocbzAMCDAwafCr3uY9FM7oGDHAi5bE5W11xWx+9MlFExL3JkCgYEAvJp5\r\nf+JGN1W037bQe2QLYUWGszewZsvplnNOeytGQa57w4YdF42lPhMz6Kc/zdzKZpN9\r\njrshiIDhAD5NCno6dwqafBAW9WZl0sn7EnlLhD4Lwm8E9bRHnC9H82yFuqmNrzww\r\nzxBCQogJISwHiVz4EkU48B283ecBn0wT/fAa19cCgYEApKWsnEHgrhy1IxOpCoRh\r\nUhqdv2k1xDPN/8DUjtnAFtwmVcLa/zJopU/Zn4y1ZzSzjwECSTi+iWZRQ/YXXHPf\r\nl92SFjhFW92Niuy8w8FnevXjF6T7PYiy1SkJ9OR1QlZrXc04iiGBDazLu115A7ce\r\nanACS03OLw+CKgl6Q/RR83ECgYBCUngDVoimkMcIHHt3yJiP3ikeAKlRnMdJlsa0\r\nXWVZV4hCG3lDfRXsnEgWuimftNKf+6GdfYSvQdLdiQsCcjT5A4uLsQTByv5nf4uA\r\n1ZKOsFrmRrARzxGXhLDikvj7yP//7USkq+0BBGFhfuAvl7fMhPceyPZPehqB7/jf\r\nxX1LBQKBgAn5GgSXzzS0e06ZlP/VrKxreOHa5Z8wOmqqYQ0QTeczAbNNmuITdwwB\r\nNkbRqpVXRIfuj0BQBegAiix8om1W4it0cwz54IXBwQULxJR1StWxj3jo4QtpMQ+z\r\npVPdB1Ilb9zPV1YvDwRfdS1xsobzznAx56ecsXduZjs9mF61db8Q\r\n-----END RSA PRIVATE KEY-----\r\n'; // eslint-disable-line -const encryptedSecretKey = 'K5FyqZo5uFKfw9QBtn95o6vuPuD0zH/1seIrqPKqGnz8AxALNS' + - 'x6EeRq7G1I6JJpS1XN13EhnwGn2ipsml3Uf2fQ00YgEmImG8wzGVZm8fWotpVO4ilN4JGyQCah' + - '81rNX4wZ9xHqDD7qYR5MyIERxR/osoXfctOwY7GGUjRKJfLOguNUlpaovejg6mZfTvYAiDF+PTO1' + - 'sKUYqHt1IfKQtsK3dov1EFMBB5pWM7sVfncq/CthKN5M+VHx9Y87qdoP3+7AW+RCBbSDOfQgxvqtS7' + - 'PIAf10mDl8k2kEURLz+RqChu4O4S0UzbEmtja7wa7WYhYKv/tM/QeW7kyNJMmnPg=='; - -const PATCH_VERSION = 2; -const locationTypeMatch = { - 'location-mem-v1': 'mem', - 'location-file-v1': 'file', - 'location-azure-v1': 'azure', - 'location-do-spaces-v1': 'aws_s3', - 'location-aws-s3-v1': 'aws_s3', - 'location-wasabi-v1': 'aws_s3', - 'location-gcp-v1': 'gcp', - 'location-scality-ring-s3-v1': 'aws_s3', - 'location-ceph-radosgw-s3-v1': 'aws_s3', +const locations = { + 'location-1': { + details: { + accessKey: 'myaccesskey', + secretKey: 'mysecretkey', + }, + locationType: 'location-scality-ring-s3-v1', + }, + 'location-2': { + details: { + accessKey: 'anotheraccesskey', + secretKey: 'anothersecretkey', + }, + locationType: 'location-file-v1', + }, }; const mongoConfig = { replicaSetHosts: testConfig.queuePopulator.mongo.replicaSetHosts, @@ -39,36 +34,6 @@ const mongoConfig = { logger: fakeLogger, }; -function createConfig() { - return { - version: PATCH_VERSION + 1, - locations: { - 'location-1': { - details: { - accessKey: 'myaccesskey', - secretKey: encryptedSecretKey, - }, - locationType: 'location-scality-ring-s3-v1', - }, - 'location-2': { - details: { - accessKey: 'anotheraccesskey', - secretKey: encryptedSecretKey, - }, - locationType: 'location-file-v1', - }, - 'location-3': { - details: { - accessKey: 'anotheraccesskey', - secretKey: encryptedSecretKey, - }, - locationType: 'location-azure-v1', - }, - }, - instanceId: 'hello-zenko', - }; -} - function createBucketMDObject(bucketName, locationName, ingestion) { const mockCreationDate = new Date().toString(); return new BucketInfo(bucketName, 'owner', 'ownerDisplayName', @@ -92,14 +57,6 @@ describe('patchConfiguration', () => { null, fakeLogger); this.md.setup(next); }, - next => this.md.putObjectMD( - managementDatabaseName, - tokenConfigurationKey, - { privateKey }, - {}, - fakeLogger, - err => next(err), - ), // populate mongo with buckets next => this.md.createBucket('bucket-1', bucket1, fakeLogger, next), next => this.md.createBucket('bucket-2', bucket2, fakeLogger, next), @@ -108,8 +65,6 @@ describe('patchConfiguration', () => { }); beforeEach(() => { - // empty configs - Config.setBootstrapList({}); Config.setIngestionBuckets({}, []); }); @@ -118,201 +73,29 @@ describe('patchConfiguration', () => { client.db.dropDatabase(); }); - it('should not patch any configurations when a version has not been set', - done => { - const conf = { - version: undefined, - }; - // regardless of overlay version, there should be no update to configs - const overlayVersion = PATCH_VERSION; - patchConfiguration(overlayVersion, conf, this.md, true, fakeLogger, - (err, version) => { - assert.ifError(err); - - const bootstrapList = Config.getBootstrapList(); - const ingestionBuckets = Config.getIngestionBuckets(); - assert(version === undefined); - assert.strictEqual(bootstrapList.length, 0); - assert.strictEqual(ingestionBuckets.length, 0); - done(); - }); - }); - - describe('bootstrap list', () => { - let configOverlay; - let expectedLocations; - beforeEach(() => { - configOverlay = createConfig(); - expectedLocations = [ - 'location-1', - 'location-3', - ]; - }); - - it('should filter Scality locations from bootstrapList (Orbit ' + - 'specific)', done => { - const overlayVersion = PATCH_VERSION; - patchConfiguration(overlayVersion, configOverlay, this.md, true, - fakeLogger, err => { - assert.ifError(err); - - const bootstrapList = Config.getBootstrapList(); - const scalityLocationName = Object.keys(configOverlay.locations) - .find(l => (configOverlay.locations[l].locationType === - 'location-file-v1')); - assert(!bootstrapList.find(l => - l.site === scalityLocationName)); - done(); - }); - }); - - it('should not patch bootstrap list with an old overlay version', done => { - const prevOverlayVersion = PATCH_VERSION + 1; - patchConfiguration( - prevOverlayVersion, - configOverlay, - this.md, - true, - fakeLogger, - (err, version) => { - assert.ifError(err); - - const bootstrapList = Config.getBootstrapList(); - assert.strictEqual(version, undefined); - assert.strictEqual(bootstrapList.length, 0); - done(); - } - ); - }); - - it('should patch bootstrap list with a new overlay version', done => { - const overlayVersion = PATCH_VERSION; - patchConfiguration(overlayVersion, configOverlay, this.md, true, - fakeLogger, err => { - assert.ifError(err); - - const bootstrapList = Config.getBootstrapList(); - assert.strictEqual(bootstrapList.length, expectedLocations.length); - expectedLocations.forEach(location => { - assert(bootstrapList.find(l => l.site === location)); - }); - done(); - }); - }); - - it('should correctly form bootstrap list config', done => { - const overlayVersion = PATCH_VERSION; - patchConfiguration(overlayVersion, configOverlay, this.md, true, - fakeLogger, err => { - assert.ifError(err); - - const bootstrapList = Config.getBootstrapList(); - bootstrapList.forEach(location => { - assert(location.site); - assert(location.type); - const locType = - configOverlay.locations[location.site].locationType; - assert.strictEqual( - location.type, locationTypeMatch[locType]); - }); - done(); - }); - }); - }); - describe('ingestion bucket list', () => { - const ingestionTypeMatch = { - 'location-scality-ring-s3-v1': 'scality_s3', - }; - const expectedIngestionBuckets = [bucket1]; - - let configOverlay; - beforeEach(() => { - configOverlay = createConfig(); - }); - - it('should patch ingestion buckets config with an old overlay version', - done => { - let ingestionBuckets = Config.getIngestionBuckets(); - assert.strictEqual(ingestionBuckets.length, 0); - - const prevOverlayVersion = PATCH_VERSION + 1; - patchConfiguration(prevOverlayVersion, configOverlay, this.md, true, - fakeLogger, (err, version) => { - assert.ifError(err); - - ingestionBuckets = Config.getIngestionBuckets(); - assert(version === undefined); - assert.strictEqual(ingestionBuckets.length, 1); - expectedIngestionBuckets.forEach(bucket => { - assert(ingestionBuckets.find( - b => b.zenkoBucket === bucket.getName())); - }); - done(); - }); - }); - - it('should patch ingestion buckets config with a new overlay version', - done => { - const overlayVersion = PATCH_VERSION; - patchConfiguration(overlayVersion, configOverlay, this.md, true, - fakeLogger, err => { + it('should only keep buckets on a backbeat ingestion location', done => { + updateIngestionBuckets(locations, this.md, fakeLogger, err => { assert.ifError(err); const ingestionBuckets = Config.getIngestionBuckets(); - expectedIngestionBuckets.forEach(bucket => { - assert( - ingestionBuckets.find(b => b.zenkoBucket === bucket.getName()) - ); - }); + assert.strictEqual(ingestionBuckets.length, 1); + assert.strictEqual(ingestionBuckets[0].zenkoBucket, + bucket1.getName()); done(); }); }); - it('should not update ingestion buckets config if flag disabled', done => { - const overlayVersion = PATCH_VERSION; - patchConfiguration( - overlayVersion, - configOverlay, - this.md, - undefined, - fakeLogger, - err => { - assert.ifError(err); - - const ingestionBuckets = Config.getIngestionBuckets(); - assert.strictEqual(ingestionBuckets.length, 0); - - // should update bootstrapList given overlay version has been - // updated - const bootstrapList = Config.getBootstrapList(); - assert(bootstrapList.length > 0); - - done(); - }); - }); - it('should correctly form ingestion bucket list config', done => { - const overlayVersion = PATCH_VERSION; - const conf = Object.assign({}, configOverlay); - patchConfiguration(overlayVersion, conf, this.md, true, - fakeLogger, err => { + updateIngestionBuckets(locations, this.md, fakeLogger, err => { assert.ifError(err); - - const ingestionBuckets = Config.getIngestionBuckets(); - - ingestionBuckets.forEach(bucket => { - assert(bucket.credentials.accessKey); - assert(bucket.credentials.secretKey); - assert(bucket.zenkoBucket); - assert(bucket.locationConstraint); - assert(bucket.locationType); - const locs = configOverlay.locations; - const locType = - locs[bucket.locationConstraint].locationType; - assert.strictEqual( - bucket.locationType, ingestionTypeMatch[locType]); - assert.deepStrictEqual( - bucket.ingestion, { status: 'enabled' }); + const [bucket] = Config.getIngestionBuckets(); + assert.deepStrictEqual(bucket, { + accessKey: 'myaccesskey', + secretKey: 'mysecretkey', + locationType: 'scality_s3', + zenkoBucket: 'bucket-1', + ingestion: { status: 'enabled' }, + locationConstraint: 'location-1', }); done(); });