feat: Create migration from old config to new (#424)

* fix: adapt old settings loader to the new schema (filter schema_version)

feat: migrate commissions globals

feat: migrate locales

refactor: generalize the old fields search

chore: created functions signatures for all config migrations

feat: created wallet migration

feat: migrate operator info

feat: migrate coin atm radar

feat: migrate terms and conditions

feat: migrate commissions overrides

fix: removed the wallet_COIN_active field (don't exist anymore)

chore: moved the config-migration lib to the lib folder

feat: migrate cashout configurations

feat: migrate notifications globals

feat: export migration function

feat: migrate most of notifications scoped configs

fix: added the missing text property to the terms and conditions
migration

feat: migrate compliance triggers

feat: migrate receipt printing

feat: migrate accounts

chore: remove test code form module

refactor: change some functions naming

fix: set default trigger type to 'volume'

feat: added threshold days (default 1) to triggers

fix: removed strike from the accounts importing

refactor: cleaner code on fixed properties

feat: avoid repeated crypto/machine pairs on the commissions overrides
migrations

refactor: make renameAccountFields function internal to the account
migration function

fix: migrate all crypto scoped commission overrides

* fix: return plain objects from functions to make the jsons more readable

fix: fix bitgo fields casing

fix: improve commissions migration function readability

refactor: standard styling

* feat: add fallback values to the migration

* feat: created db migration for the new config

* feat: create migration to move machine names from file to db

fix: updates machine names before the config migration

fix: load machineLoader

fix: create a param to ignore the schema version when loading the latest
config using the old loader

* refactor: remove unnecessary arguments on createTrigger function

fix: check if there's an smsVerificationThreshold configured prior to
migrating triggers

* fix: migrate triggers with the correct thresholds and verify if they're
valid
This commit is contained in:
Liordino Neto 2020-09-25 07:07:47 -03:00 committed by GitHub
parent 3c6f547349
commit ccf7eacfad
8 changed files with 546 additions and 42 deletions

View file

@ -1,13 +1,14 @@
const _ = require('lodash/fp')
const db = require('../db')
const configValidate = require('../config-validate')
const config = require('./config')
const ph = require('../plugin-helper')
const schemas = ph.loadSchemas()
function fetchAccounts () {
return db.oneOrNone('select data from user_config where type=$1', ['accounts'])
return db.oneOrNone('select data from user_config where type=$1 and schema_version=$2', ['accounts', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(row => {
// Hard code this for now
const accounts = [{
@ -91,7 +92,7 @@ function getAccount (accountCode) {
}
function save (accounts) {
return db.none('update user_config set data=$1 where type=$2', [{accounts: accounts}, 'accounts'])
return db.none('update user_config set data=$1 where type=$2 and schema_version=$3', [{accounts: accounts}, 'accounts', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
}
function updateAccounts (newAccount, accounts) {

View file

@ -17,10 +17,10 @@ function fetchSchema () {
}
function fetchConfig () {
const sql = `select data from user_config where type=$1
const sql = `select data from user_config where type=$1 and schema_version=$2
order by id desc limit 1`
return db.oneOrNone(sql, ['config'])
return db.oneOrNone(sql, ['config', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(row => row ? row.data.config : [])
}

458
lib/config-migration.js Normal file
View file

@ -0,0 +1,458 @@
const _ = require('lodash/fp')
const uuid = require('uuid')
const { COINS } = require('../lib/new-admin/config/coins')
const { scopedValue } = require('./config-manager')
const GLOBAL = 'global'
const ALL_CRYPTOS = _.values(COINS).sort()
const ALL_MACHINES = 'ALL_MACHINES'
const GLOBAL_SCOPE = {
crypto: ALL_CRYPTOS,
machine: GLOBAL
}
function getConfigFields (codes, config) {
const stringfiedGlobalScope = JSON.stringify(GLOBAL_SCOPE)
const fields = config
.filter(i => codes.includes(i.fieldLocator.code))
.map(f => {
const crypto = Array.isArray(f.fieldLocator.fieldScope.crypto)
? f.fieldLocator.fieldScope.crypto.sort()
: f.fieldLocator.fieldScope.crypto === GLOBAL
? ALL_CRYPTOS
: [f.fieldLocator.fieldScope.crypto]
const machine = f.fieldLocator.fieldScope.machine
return {
code: f.fieldLocator.code,
scope: {
crypto,
machine
},
value: f.fieldValue.value
}
})
.filter(f => f.value != null)
const grouped = _.chain(fields)
.groupBy(f => JSON.stringify(f.scope))
.value()
return {
global: grouped[stringfiedGlobalScope] || [],
scoped:
_.entries(
_.chain(grouped)
.omit([stringfiedGlobalScope])
.value()
).map(f => {
const fallbackValues =
_.difference(codes, f[1].map(v => v.code))
.map(v => ({
code: v,
scope: JSON.parse(f[0]),
value: scopedValue(f[0].crypto, f[0].machine, v, config)
}))
.filter(f => f.value != null)
return {
scope: JSON.parse(f[0]),
values: f[1].concat(fallbackValues)
}
}) || []
}
}
function migrateCommissions (config) {
const areArraysEquals = (arr1, arr2) => _.isEmpty(_.xor(arr1, arr2))
const getMachine = _.get('scope.machine')
const getCrypto = _.get('scope.crypto')
const flattenCoins = _.compose(_.flatten, _.map(getCrypto))
const diffAllCryptos = _.compose(_.difference(ALL_CRYPTOS))
const codes = {
minimumTx: 'minimumTx',
cashInFee: 'fixedFee',
cashInCommission: 'cashIn',
cashOutCommission: 'cashOut'
}
const { global, scoped } = getConfigFields(_.keys(codes), config)
const machineAndCryptoScoped = scoped.filter(
f => f.scope.machine !== GLOBAL_SCOPE.machine && f.scope.crypto.length === 1
)
const cryptoScoped = scoped.filter(
f =>
f.scope.machine === GLOBAL_SCOPE.machine &&
!areArraysEquals(f.scope.crypto, GLOBAL_SCOPE.crypto)
)
const machineScoped = scoped.filter(
f =>
f.scope.machine !== GLOBAL_SCOPE.machine &&
areArraysEquals(f.scope.crypto, GLOBAL_SCOPE.crypto)
)
const withCryptoScoped = machineAndCryptoScoped.concat(cryptoScoped)
const filteredMachineScoped = _.map(it => {
const filterByMachine = _.filter(_.includes(getMachine(it)))
const unrepeatedCryptos = _.compose(
diffAllCryptos,
flattenCoins,
filterByMachine
)(withCryptoScoped)
return _.set('scope.crypto', unrepeatedCryptos)(it)
})(machineScoped)
const allCommissionsOverrides = withCryptoScoped.concat(filteredMachineScoped)
return {
..._.fromPairs(global.map(f => [`commissions_${codes[f.code]}`, f.value])),
...(allCommissionsOverrides.length > 0 && {
commissions_overrides: allCommissionsOverrides.map(s => ({
..._.fromPairs(s.values.map(f => [codes[f.code], f.value])),
machine: s.scope.machine === GLOBAL ? ALL_MACHINES : s.scope.machine,
cryptoCurrencies: s.scope.crypto,
id: uuid.v4()
}))
})
}
}
function migrateLocales (config) {
const codes = {
country: 'country',
fiatCurrency: 'fiatCurrency',
machineLanguages: 'languages',
cryptoCurrencies: 'cryptoCurrencies'
}
const { global, scoped } = getConfigFields(_.keys(codes), config)
return {
..._.fromPairs(global.map(f => [`locale_${codes[f.code]}`, f.value])),
...(scoped.length > 0 && {
locale_overrides: scoped.map(s => ({
..._.fromPairs(s.values.map(f => [codes[f.code], f.value])),
machine: s.scope.machine,
id: uuid.v4()
}))
})
}
}
// TODO new-admin: virtualCashOutDenomination
function migrateCashOut (config) {
const globalCodes = {
fudgeFactorActive: 'fudgeFactorActive'
}
const scopedCodes = {
cashOutEnabled: 'active',
topCashOutDenomination: 'top',
bottomCashOutDenomination: 'bottom',
zeroConfLimit: 'zeroConfLimit'
}
const { global } = getConfigFields(_.keys(globalCodes), config)
const { scoped } = getConfigFields(_.keys(scopedCodes), config)
return {
..._.fromPairs(
global.map(f => [`cashout_${globalCodes[f.code]}`, f.value])
),
..._.fromPairs(
_.flatten(
scoped.map(s => {
const fields = s.values.map(f => [
`cashout_${f.scope.machine}_${scopedCodes[f.code]}`,
f.value
])
fields.push([`cashout_${s.scope.machine}_id`, s.scope.machine])
return fields
})
)
)
}
}
function migrateNotifications (config) {
const globalCodes = {
notificationsEmailEnabled: 'email_active',
notificationsSMSEnabled: 'sms_active',
cashOutCassette1AlertThreshold: 'fiatBalanceCassette1',
cashOutCassette2AlertThreshold: 'fiatBalanceCassette2',
cryptoAlertThreshold: 'cryptoLowBalance'
}
const machineScopedCodes = {
cashOutCassette1AlertThreshold: 'cassette1',
cashOutCassette2AlertThreshold: 'cassette2'
}
const cryptoScopedCodes = {
cryptoAlertThreshold: 'lowBalance'
}
const { global } = getConfigFields(_.keys(globalCodes), config)
const machineScoped = getConfigFields(
_.keys(machineScopedCodes),
config
).scoped.filter(f => f.scope.crypto === GLOBAL && f.scope.machine !== GLOBAL)
const cryptoScoped = getConfigFields(
_.keys(cryptoScopedCodes),
config
).scoped.filter(f => f.scope.crypto !== GLOBAL && f.scope.machine === GLOBAL)
return {
..._.fromPairs(
global.map(f => [`notifications_${globalCodes[f.code]}`, f.value])
),
notifications_email_balance: true,
notifications_email_transactions: true,
notifications_email_compliance: true,
notifications_email_errors: true,
notifications_sms_balance: true,
notifications_sms_transactions: true,
notifications_sms_compliance: true,
notifications_sms_errors: true,
...(machineScoped.length > 0 && {
notifications_fiatBalanceOverrides: machineScoped.map(s => ({
..._.fromPairs(
s.values.map(f => [machineScopedCodes[f.code], f.value])
),
machine: s.scope.machine,
id: uuid.v4()
}))
}),
...(cryptoScoped.length > 0 && {
notifications_cryptoBalanceOverrides: cryptoScoped.map(s => ({
..._.fromPairs(s.values.map(f => [cryptoScopedCodes[f.code], f.value])),
cryptoCurrency: s.scope.crypto,
id: uuid.v4()
}))
})
}
}
function migrateWallet (config) {
const codes = {
ticker: 'ticker',
wallet: 'wallet',
exchange: 'exchange',
zeroConf: 'zeroConf'
}
const { scoped } = getConfigFields(_.keys(codes), config)
return {
...(scoped.length > 0 &&
_.fromPairs(
_.flatten(
scoped.map(s =>
s.values.map(f => [
`wallets_${f.scope.crypto}_${codes[f.code]}`,
f.value
])
)
)
))
}
}
function migrateOperatorInfo (config) {
const codes = {
operatorInfoActive: 'active',
operatorInfoEmail: 'email',
operatorInfoName: 'name',
operatorInfoPhone: 'phone',
operatorInfoWebsite: 'website',
operatorInfoCompanyNumber: 'companyNumber'
}
const { global } = getConfigFields(_.keys(codes), config)
return {
..._.fromPairs(global.map(f => [`operatorInfo_${codes[f.code]}`, f.value]))
}
}
function migrateReceiptPrinting (config) {
const codes = {
receiptPrintingActive: 'active'
}
const { global } = getConfigFields(_.keys(codes), config)
return {
..._.fromPairs(global.map(f => [`receipt_${codes[f.code]}`, f.value])),
receipt_operatorWebsite: true,
receipt_operatorEmail: true,
receipt_operatorPhone: true,
receipt_companyRegistration: true,
receipt_machineLocation: true,
receipt_customerNameOrPhoneNumber: true,
receipt_exchangeRate: true,
receipt_addressQRCode: true
}
}
function migrateCoinATMRadar (config) {
const codes = ['coinAtmRadarActive', 'coinAtmRadarShowRates']
const { global } = getConfigFields(codes, config)
const coinAtmRadar = _.fromPairs(global.map(f => [f.code, f.value]))
return {
coinAtmRadar_active: coinAtmRadar.coinAtmRadarActive,
coinAtmRadar_commissions: coinAtmRadar.coinAtmRadarShowRates,
coinAtmRadar_limitsAndVerification: coinAtmRadar.coinAtmRadarShowRates
}
}
function migrateTermsAndConditions (config) {
const codes = {
termsScreenActive: 'active',
termsScreenTitle: 'title',
termsScreenText: 'text',
termsAcceptButtonText: 'acceptButtonText',
termsCancelButtonText: 'cancelButtonText'
}
const { global } = getConfigFields(_.keys(codes), config)
return {
..._.fromPairs(
global.map(f => [`termsConditions_${codes[f.code]}`, f.value])
)
}
}
// TODO new-admin: rejectAddressReuseActive
function migrateComplianceTriggers (config) {
const triggerTypes = {
amount: 'txAmount',
velocity: 'txVelocity',
volume: 'txVolume',
consecutiveDays: 'consecutiveDays'
}
const requirements = {
sms: 'sms',
idData: 'idData',
idPhoto: 'idPhoto',
facePhoto: 'facePhoto',
sanctions: 'sanctions'
}
function createTrigger (
requirement,
threshold
) {
return {
id: uuid.v4(),
cashDirection: 'both',
threshold,
thresholdDays: 1,
triggerType: triggerTypes.volume,
requirement
}
}
const codes = [
'smsVerificationActive',
'smsVerificationThreshold',
'idCardDataVerificationActive',
'idCardDataVerificationThreshold',
'idCardPhotoVerificationActive',
'idCardPhotoVerificationThreshold',
'frontCameraVerificationActive',
'frontCameraVerificationThreshold',
'sanctionsVerificationActive',
'sanctionsVerificationThreshold'
]
const global = _.fromPairs(
getConfigFields(codes, config).global.map(f => [f.code, f.value])
)
const triggers = []
if (global.smsVerificationActive && _.isNumber(global.smsVerificationThreshold)) {
triggers.push(
createTrigger(requirements.sms, global.smsVerificationThreshold)
)
}
if (global.idCardDataVerificationActive && _.isNumber(global.idCardDataVerificationThreshold)) {
triggers.push(
createTrigger(requirements.idData, global.idCardDataVerificationThreshold)
)
}
if (global.idCardPhotoVerificationActive && _.isNumber(global.idCardPhotoVerificationThreshold)) {
triggers.push(
createTrigger(requirements.idPhoto, global.idCardPhotoVerificationThreshold)
)
}
if (global.frontCameraVerificationActive && _.isNumber(global.frontCameraVerificationThreshold)) {
triggers.push(
createTrigger(requirements.facePhoto, global.frontCameraVerificationThreshold)
)
}
if (global.sanctionsVerificationActive && _.isNumber(global.sanctionsVerificationThreshold)) {
triggers.push(
createTrigger(requirements.sanctions, global.sanctionsVerificationThreshold)
)
}
return {
triggers
}
}
function migrateConfig (config) {
return {
...migrateCommissions(config),
...migrateLocales(config),
...migrateCashOut(config),
...migrateNotifications(config),
...migrateWallet(config),
...migrateOperatorInfo(config),
...migrateReceiptPrinting(config),
...migrateCoinATMRadar(config),
...migrateTermsAndConditions(config),
...migrateComplianceTriggers(config)
}
}
function migrateAccounts (accounts) {
const accountArray = [
'bitgo',
'bitstamp',
'blockcypher',
'infura',
'itbit',
'kraken',
'mailgun',
'twilio'
]
return _.pick(accountArray)(accounts)
}
function migrate (config, accounts) {
return {
config: migrateConfig(config),
accounts: migrateAccounts(accounts)
}
}
module.exports = {
migrateConfig,
migrateAccounts,
migrate
}

View file

@ -7,6 +7,8 @@ const schema = require('../lamassu-schema.json')
const REMOVED_FIELDS = ['crossRefVerificationActive', 'crossRefVerificationThreshold']
const SETTINGS_LOADER_SCHEMA_VERSION = 1
function allScopes (cryptoScopes, machineScopes) {
const scopes = []
cryptoScopes.forEach(c => {
@ -181,4 +183,9 @@ function validate (config) {
})
}
module.exports = {validate, ensureConstraints, validateRequires}
module.exports = {
SETTINGS_LOADER_SCHEMA_VERSION,
validate,
ensureConstraints,
validateRequires
}

View file

@ -54,8 +54,8 @@ function load (versionId) {
}))
}
function loadLatest () {
return Promise.all([loadLatestConfig(), loadAccounts()])
function loadLatest (filterSchemaVersion = true) {
return Promise.all([loadLatestConfig(filterSchemaVersion), loadAccounts(filterSchemaVersion)])
.then(([config, accounts]) => ({
config,
accounts
@ -67,10 +67,10 @@ function loadConfig (versionId) {
const sql = `select data
from user_config
where id=$1 and type=$2
where id=$1 and type=$2 and schema_version=$3
and valid`
return db.one(sql, [versionId, 'config'])
return db.one(sql, [versionId, 'config', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(row => row.data.config)
.then(configValidate.validate)
.catch(err => {
@ -82,17 +82,17 @@ function loadConfig (versionId) {
})
}
function loadLatestConfig () {
function loadLatestConfig (filterSchemaVersion = true) {
if (argv.fixture) return loadFixture()
const sql = `select id, valid, data
from user_config
where type=$1
where type=$1 ${filterSchemaVersion ? 'and schema_version=$2' : ''}
and valid
order by id desc
limit 1`
return db.one(sql, ['config'])
return db.one(sql, ['config', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(row => row.data.config)
.then(configValidate.validate)
.catch(err => {
@ -109,19 +109,19 @@ function loadRecentConfig () {
const sql = `select id, data
from user_config
where type=$1
where type=$1 and schema_version=$2
order by id desc
limit 1`
return db.one(sql, ['config'])
return db.one(sql, ['config', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(row => row.data.config)
}
function loadAccounts () {
function loadAccounts (filterSchemaVersion = true) {
const toFields = fieldArr => _.fromPairs(_.map(r => [r.code, r.value], fieldArr))
const toPairs = r => [r.code, toFields(r.fields)]
return db.oneOrNone('select data from user_config where type=$1', 'accounts')
return db.oneOrNone(`select data from user_config where type=$1 ${filterSchemaVersion ? 'and schema_version=$2' : ''}`, ['accounts', configValidate.SETTINGS_LOADER_SCHEMA_VERSION])
.then(function (data) {
if (!data) return {}
return _.fromPairs(_.map(toPairs, data.data.accounts))

View file

@ -0,0 +1,38 @@
const db = require('./db')
const settingsLoader = require('../lib/settings-loader')
const machineLoader = require('../lib/machine-loader')
const { saveConfig, saveAccounts } = require('../lib/new-settings-loader')
const { migrate } = require('../lib/config-migration')
module.exports.up = function (next) {
function migrateConfig(settings) {
return migrate(settings.config, settings.accounts)
.then(newSettings => Promise.all([
saveConfig(newSettings.config),
saveAccounts(newSettings.accounts)
]))
.then(() => next())
}
settingsLoader.loadLatest(false)
.then(async settings => ({
settings,
machines: await machineLoader.getMachineNames(settings.config)
}))
.then(({ settings, machines }) => {
const sql = machines
? machines.map(m => `update devices set name = '${m.name}' where device_id = '${m.deviceId}'`)
: []
return db.multi(sql, () => migrateConfig(settings))
})
.catch(err => {
if (err.message = 'lamassu-server is not configured')
next()
console.log(err.message)
})
}
module.exports.down = function (next) {
next()
}

View file

@ -9,7 +9,7 @@ module.exports = {migrateNames}
function migrateNames () {
const cs = new pgp.helpers.ColumnSet(['?device_id', 'name'], {table: 'devices'})
return settingsLoader.loadLatest()
return settingsLoader.loadLatest(false)
.then(r => machineLoader.getMachineNames(r.config))
.then(_.map(r => ({device_id: r.deviceId, name: r.name})))
.then(data => pgp.helpers.update(data, cs) + ' WHERE t.device_id=v.device_id')

View file

@ -30,52 +30,52 @@ export default {
face: true
},
{
code: 'btcWalletId',
code: 'BTCWalletId',
display: 'BTC Wallet ID',
component: TextInput
},
{
code: 'btcWalletPassphrase',
code: 'BTCWalletPassphrase',
display: 'BTC Wallet Passphrase',
component: SecretInput
},
{
code: 'ltcWalletId',
code: 'LTCWalletId',
display: 'LTC Wallet ID',
component: TextInput
},
{
code: 'ltcWalletPassphrase',
code: 'LTCWalletPassphrase',
display: 'LTC Wallet Passphrase',
component: SecretInput
},
{
code: 'zecWalletId',
code: 'ZECWalletId',
display: 'ZEC Wallet ID',
component: TextInput
},
{
code: 'zecWalletPassphrase',
code: 'ZECWalletPassphrase',
display: 'ZEC Wallet Passphrase',
component: SecretInput
},
{
code: 'bchWalletId',
code: 'BCHWalletId',
display: 'BCH Wallet ID',
component: TextInput
},
{
code: 'bchWalletPassphrase',
code: 'BCHWalletPassphrase',
display: 'BCH Wallet Passphrase',
component: SecretInput
},
{
code: 'dashWalletId',
code: 'DASHWalletId',
display: 'DASH Wallet ID',
component: TextInput
},
{
code: 'dashWalletPassphrase',
code: 'DASHWalletPassphrase',
display: 'DASH Wallet Passphrase',
component: SecretInput
}
@ -84,38 +84,38 @@ export default {
token: Yup.string()
.max(100, 'Too long')
.required('Required'),
btcWalletId: Yup.string().max(100, 'Too long'),
btcWalletPassphrase: Yup.string()
BTCWalletId: Yup.string().max(100, 'Too long'),
BTCWalletPassphrase: Yup.string()
.max(100, 'Too long')
.when('btcWalletId', {
.when('BTCWalletId', {
is: isDefined,
then: Yup.string().required()
}),
ltcWalletId: Yup.string().max(100, 'Too long'),
ltcWalletPassphrase: Yup.string()
LTCWalletId: Yup.string().max(100, 'Too long'),
LTCWalletPassphrase: Yup.string()
.max(100, 'Too long')
.when('ltcWalletId', {
.when('LTCWalletId', {
is: isDefined,
then: Yup.string().required()
}),
zecWalletId: Yup.string().max(100, 'Too long'),
zecWalletPassphrase: Yup.string()
ZECWalletId: Yup.string().max(100, 'Too long'),
ZECWalletPassphrase: Yup.string()
.max(100, 'Too long')
.when('zecWalletId', {
.when('ZECWalletId', {
is: isDefined,
then: Yup.string().required()
}),
bchWalletId: Yup.string().max(100, 'Too long'),
bchWalletPassphrase: Yup.string()
BCHWalletId: Yup.string().max(100, 'Too long'),
BCHWalletPassphrase: Yup.string()
.max(100, 'Too long')
.when('bchWalletId', {
.when('BCHWalletId', {
is: isDefined,
then: Yup.string().required()
}),
dashWalletId: Yup.string().max(100, 'Too long'),
dashWalletPassphrase: Yup.string()
DASHWalletId: Yup.string().max(100, 'Too long'),
DASHWalletPassphrase: Yup.string()
.max(100, 'Too long')
.when('dashWalletId', {
.when('DASHWalletId', {
is: isDefined,
then: Yup.string().required()
}),