actual/packages/loot-core/src/server/accounts/transaction-rules.js

import {
  currentDay,
  addDays,
  subDays,
  parseDate,
  dayFromDate
} from '../../shared/months';
import {
  FIELD_TYPES,
  sortNumbers,
  getApproxNumberThreshold
} from '../../shared/rules';
import { partitionByField, fastSetMerge } from '../../shared/util';
import { schemaConfig } from '../aql';
import * as db from '../db';
import { getMappings } from '../db/mappings';
import { RuleError } from '../errors';
import { requiredFields, toDateRepr } from '../models';
import { setSyncingMode, batchMessages } from '../sync';
import { addSyncListener } from '../sync/index';
import {
  Condition,
  Action,
  Rule,
  RuleIndexer,
  rankRules,
  migrateIds,
  iterateIds
} from './rules';

// TODO: Detect if it looks like the user is creating a rename rule
// and prompt to create it in the pre phase instead
// * We could also make the "create rule" button a dropdown that
//   provides different "templates" like "create renaming rule"

export { iterateIds } from './rules';

let allRules;
let unlistenSync;
let firstcharIndexer;
let payeeIndexer;

export function resetState() {
  allRules = new Map();
  firstcharIndexer = new RuleIndexer({
    field: 'imported_payee',
    method: 'firstchar'
  });
  payeeIndexer = new RuleIndexer({ field: 'payee' });
}

// Database functions

function invert(obj) {
  return Object.fromEntries(
    Object.entries(obj).map(entry => {
      return [entry[1], entry[0]];
    })
  );
}

let internalFields = schemaConfig.views.transactions.fields;
let publicFields = invert(schemaConfig.views.transactions.fields);

function fromInternalField(obj) {
  return {
    ...obj,
    field: publicFields[obj.field] || obj.field
  };
}

function toInternalField(obj) {
  return {
    ...obj,
    field: internalFields[obj.field] || obj.field
  };
}

export const ruleModel = {
  validate(rule, { update } = {}) {
    requiredFields('rules', rule, ['conditions', 'actions'], update);

    if (!update || 'stage' in rule) {
      if (
        rule.stage !== 'pre' &&
        rule.stage !== 'post' &&
        rule.stage !== null
      ) {
        throw new Error('Invalid rule stage: ' + rule.stage);
      }
    }

    return rule;
  },

  toJS(row) {
    function parseArray(str) {
      let value;
      try {
        value = typeof str === 'string' ? JSON.parse(str) : str;
      } catch (e) {
        throw new RuleError('internal', 'Cannot parse rule json');
      }

      if (!Array.isArray(value)) {
        throw new RuleError('internal', 'Rule json must be an array');
      }
      return value;
    }

    let rule = { ...row };
    rule.conditions = rule.conditions
      ? parseArray(rule.conditions).map(cond => fromInternalField(cond))
      : [];
    rule.actions = rule.actions
      ? parseArray(rule.actions).map(action => fromInternalField(action))
      : [];
    return rule;
  },

  fromJS(rule) {
    let row = { ...rule };
    if ('conditions' in row) {
      let conditions = row.conditions.map(cond => toInternalField(cond));
      row.conditions = JSON.stringify(conditions);
    }
    if ('actions' in row) {
      let actions = row.actions.map(action => toInternalField(action));
      row.actions = JSON.stringify(actions);
    }
    return row;
  }
};

export function makeRule(data) {
  let rule;
  try {
    rule = new Rule({
      ...ruleModel.toJS(data),
      fieldTypes: FIELD_TYPES
    });
  } catch (e) {
    console.warn('Invalid rule', e);
    if (e instanceof RuleError) {
      return null;
    }
    throw e;
  }

  // This is needed because we map ids on the fly, and they might
  // not be persisted into the db. Mappings allow items to
  // transparently merge with other items
  migrateIds(rule, getMappings());

  return rule;
}

export async function loadRules() {
  resetState();

  let rules = await db.all(`
    SELECT * FROM rules
      WHERE conditions IS NOT NULL AND actions IS NOT NULL AND tombstone = 0
  `);

  for (let i = 0; i < rules.length; i++) {
    let desc = rules[i];
    // These are old stages, can be removed before release
    if (desc.stage === 'cleanup' || desc.stage === 'modify') {
      desc.stage = 'pre';
    }

    let rule = makeRule(desc);
    if (rule) {
      allRules.set(rule.id, rule);
      firstcharIndexer.index(rule);
      payeeIndexer.index(rule);
    }
  }

  if (unlistenSync) {
    unlistenSync();
  }
  unlistenSync = addSyncListener(onApplySync);
}

export function getRules() {
  // This can simply return the in-memory data
  return [...allRules.values()];
}

export async function insertRule(rule) {
  rule = ruleModel.validate(rule);
  return db.insertWithUUID('rules', ruleModel.fromJS(rule));
}

export async function updateRule(rule) {
  rule = ruleModel.validate(rule, { update: true });
  return db.update('rules', ruleModel.fromJS(rule));
}

export async function deleteRule(rule) {
  let schedule = await db.first('SELECT id FROM schedules WHERE rule = ?', [
    rule.id
  ]);

  if (schedule) {
    return false;
  }

  return db.delete_('rules', rule.id);
}

// Sync projections

function onApplySync(oldValues, newValues) {
  newValues.forEach((items, table) => {
    if (table === 'rules') {
      items.forEach(newValue => {
        let oldRule = allRules.get(newValue.id);

        if (newValue.tombstone === 1) {
          // Deleted, need to remove it from in-memory
          let rule = allRules.get(newValue.id);
          if (rule) {
            allRules.delete(rule.getId());
            firstcharIndexer.remove(rule);
            payeeIndexer.remove(rule);
          }
        } else {
          // Inserted/updated
          let rule = makeRule(newValue);
          if (rule) {
            if (oldRule) {
              firstcharIndexer.remove(oldRule);
              payeeIndexer.remove(oldRule);
            }
            allRules.set(newValue.id, rule);
            firstcharIndexer.index(rule);
            payeeIndexer.index(rule);
          }
        }
      });
    }
  });

  // If any of the mapping tables have changed, we need to refresh the
  // ids
  let tables = [...newValues.keys()];
  if (tables.find(table => table.indexOf('mapping') !== -1)) {
    getRules().forEach(rule => {
      migrateIds(rule, getMappings());
    });
  }
}

// Runner
export function runRules(trans) {
  let finalTrans = { ...trans };
  let allChanges = {};

  let rules = rankRules(
    fastSetMerge(
      firstcharIndexer.getApplicableRules(trans),
      payeeIndexer.getApplicableRules(trans)
    )
  );

  for (let i = 0; i < rules.length; i++) {
    finalTrans = rules[i].apply(finalTrans);
  }

  return finalTrans;
}

// This does the inverse: finds all the transactions matching a rule
export function conditionsToAQL(conditions, { recurDateBounds = 100 } = {}) {
  let errors = [];

  conditions = conditions
    .map(cond => {
      if (cond instanceof Condition) {
        return cond;
      }

      try {
        return new Condition(
          cond.op,
          cond.field,
          cond.value,
          cond.options,
          FIELD_TYPES
        );
      } catch (e) {
        errors.push(e.type || 'internal');
        console.log('conditionsToAQL: invalid condition: ' + e.message);
        return null;
      }
    })
    .filter(Boolean);

  // rule -> actualql
  let filters = conditions.map(cond => {
    let { type, field, op, value, options } = cond;

    let getValue = value => {
      if (type === 'number') {
        return value.value;
      }
      return value;
    };

    let apply = (field, op, value) => {
      if (type === 'number') {
        if (options) {
          if (options.outflow) {
            return {
              $and: [
                { amount: { $lt: 0 } },
                { [field]: { $transform: '$neg', [op]: value } }
              ]
            };
          } else if (options.inflow) {
            return {
              $and: [{ amount: { $gt: 0 } }, { [field]: { [op]: value } }]
            };
          }
        }

        return { amount: { [op]: value } };
      } else if (type === 'string') {
        return { [field]: { $transform: '$lower', [op]: value } };
      } else if (type === 'date') {
        return { [field]: { [op]: value.date } };
      }
      return { [field]: { [op]: value } };
    };

    switch (op) {
      case 'isapprox':
      case 'is':
        if (type === 'date') {
          if (value.type === 'recur') {
            let dates = value.schedule
              .occurrences({ take: recurDateBounds })
              .toArray()
              .map(d => dayFromDate(d.date));

            let compare = d => ({ $eq: d });

            return {
              $or: dates.map(d => {
                if (op === 'isapprox') {
                  return {
                    $and: [
                      { date: { $gte: subDays(d, 2) } },
                      { date: { $lte: addDays(d, 2) } }
                    ]
                  };
                }
                return { date: d };
              })
            };
          } else {
            let { date } = value;

            if (op === 'isapprox') {
              let fullDate = parseDate(value.date);
              let high = addDays(fullDate, 2);
              let low = subDays(fullDate, 2);

              return {
                $and: [{ date: { $gte: low } }, { date: { $lte: high } }]
              };
            } else {
              switch (value.type) {
                case 'date':
                  return { date: value.date };
                case 'month': {
                  let low = value.date + '-00';
                  let high = value.date + '-99';
                  return {
                    $and: [{ date: { $gte: low } }, { date: { $lte: high } }]
                  };
                }
                case 'year': {
                  let low = value.date + '-00-00';
                  let high = value.date + '-99-99';
                  return {
                    $and: [{ date: { $gte: low } }, { date: { $lte: high } }]
                  };
                }
                default:
              }
            }
          }
        } else if (type === 'number') {
          let number = value.value;
          if (op === 'isapprox') {
            let threshold = getApproxNumberThreshold(number);

            return {
              $and: [
                apply(field, '$gte', number - threshold),
                apply(field, '$lte', number + threshold)
              ]
            };
          }
          return apply(field, '$eq', number);
        }

        return apply(field, '$eq', value);

      case 'isbetween':
        // This operator is only applicable to the specific `between`
        // number type so we don't use `apply`
        let [low, high] = sortNumbers(value.num1, value.num2);
        return {
          [field]: [{ $gte: low }, { $lte: high }]
        };
      case 'contains':
        // Running contains with id will automatically reach into
        // the `name` of the referenced table and do a string match
        return apply(
          type === 'id' ? field + '.name' : field,
          '$like',
          '%' + value + '%'
        );
      case 'oneOf':
        let values = value;
        if (values.length === 0) {
          // This forces it to match nothing
          return { id: null };
        }
        return { $or: values.map(v => apply(field, '$eq', v)) };
      case 'gt':
        return apply(field, '$gt', getValue(value));
      case 'gte':
        return apply(field, '$gte', getValue(value));
      case 'lt':
        return apply(field, '$lt', getValue(value));
      case 'lte':
        return apply(field, '$lte', getValue(value));
      case 'true':
        return apply(field, '$eq', true);
      case 'false':
        return apply(field, '$eq', false);
      default:
        throw new Error('Unhandled operator: ' + op);
    }
  });

  return { filters, errors };
}

export function applyActions(transactionIds, actions, handlers) {
  let parsedActions = actions
    .map(action => {
      if (action instanceof Action) {
        return action;
      }

      try {
        return new Action(
          action.op,
          action.field,
          action.value,
          action.options,
          FIELD_TYPES
        );
      } catch (e) {
        console.log('Action error', e);
        return null;
      }
    })
    .filter(Boolean);

  if (parsedActions.length !== actions.length) {
    // An error happened while parsing
    return null;
  }

  let updated = transactionIds.map(id => {
    let update = { id };
    for (let action of parsedActions) {
      action.exec(update);
    }
    return update;
  });

  return handlers['transactions-batch-update']({ updated });
}

export function getRulesForPayee(payeeId) {
  let rules = new Set();
  iterateIds(getRules(), 'payee', (rule, id) => {
    if (id === payeeId) {
      rules.add(rule);
    }
  });

  return rankRules([...rules]);
}

function* getIsSetterRules(
  stage,
  condField,
  actionField,
  { condValue, actionValue }
) {
  let rules = getRules();
  for (let i = 0; i < rules.length; i++) {
    let rule = rules[i];

    if (
      rule.stage === stage &&
      rule.actions.length === 1 &&
      rule.actions[0].op === 'set' &&
      rule.actions[0].field === actionField &&
      (actionValue === undefined || rule.actions[0].value === actionValue) &&
      rule.conditions.length === 1 &&
      rule.conditions[0].op === 'is' &&
      rule.conditions[0].field === condField &&
      (condValue === undefined || rule.conditions[0].value === condValue)
    ) {
      yield rule.serialize();
    }
  }

  return null;
}

function* getOneOfSetterRules(
  stage,
  condField,
  actionField,
  { condValue, actionValue }
) {
  let rules = getRules();
  for (let i = 0; i < rules.length; i++) {
    let rule = rules[i];

    if (
      rule.stage === stage &&
      rule.actions.length === 1 &&
      rule.actions[0].op === 'set' &&
      rule.actions[0].field === actionField &&
      (actionValue == null || rule.actions[0].value === actionValue) &&
      rule.conditions.length === 1 &&
      rule.conditions[0].op === 'oneOf' &&
      rule.conditions[0].field === condField &&
      (condValue == null || rule.conditions[0].value.indexOf(condValue) !== -1)
    ) {
      yield rule.serialize();
    }
  }

  return null;
}

export async function updatePayeeRenameRule(fromNames, to) {
  let renameRule = getOneOfSetterRules('pre', 'imported_payee', 'payee', {
    actionValue: to
  }).next().value;

  // Note that we don't check for existing rules that set this
  // `imported_payee` to something else. It's important to do
  // that for categories because categories will be changes frequently
  // for the same payee, but renames won't be changed much. It's a use
  // case we could improve in the future, but this is fine for now.

  if (renameRule) {
    let condition = renameRule.conditions[0];
    let newValue = [
      ...fastSetMerge(
        new Set(condition.value),
        new Set(fromNames.filter(name => name !== ''))
      )
    ];
    let rule = {
      ...renameRule,
      conditions: [{ ...condition, value: newValue }]
    };
    await updateRule(rule);
    return renameRule.id;
  } else {
    let rule = new Rule({
      stage: 'pre',
      conditions: [{ op: 'oneOf', field: 'imported_payee', value: fromNames }],
      actions: [{ op: 'set', field: 'payee', value: to }],
      fieldTypes: FIELD_TYPES
    });
    return insertRule(rule.serialize());
  }
}

export function getProbableCategory(transactions) {
  let scores = new Map();

  transactions.forEach(trans => {
    if (trans.category) {
      scores.set(trans.category, (scores.get(trans.category) || 0) + 1);
    }
  });

  let winner = transactions.reduce((winner, trans) => {
    let score = scores.get(trans.category);
    if (!winner || score > winner.score) {
      return { score, category: trans.category };
    }
    return winner;
  }, null);

  return winner.score >= 3 ? winner.category : null;
}

export async function updateCategoryRules(transactions) {
  if (transactions.length === 0) {
    return;
  }

  let payeeIds = new Set(transactions.map(trans => trans.payee));
  let transIds = new Set(transactions.map(trans => trans.id));

  // It's going to be quickest to get the oldest date and then query
  // all transactions since then so we can work in memory
  let oldestDate = null;
  for (let i = 0; i < transactions.length; i++) {
    if (oldestDate === null || transactions[i].date < oldestDate) {
      oldestDate = transactions[i].date;
    }
  }

  // We look 6 months behind to include any other transaction. This
  // makes it so we, 1. don't have to load in all transactions ever
  // and 2. "forget" really old transactions which might be nice and
  // 3. don't have to individually run a query for each payee
  oldestDate = subDays(oldestDate, 180);

  // Also look 180 days in the future to get any future transactions
  // (this might change when we think about scheduled transactions)
  let register = await db.all(
    `SELECT t.* FROM v_transactions t
     LEFT JOIN accounts a ON a.id = t.account
     WHERE date >= ? AND date <= ? AND is_parent = 0 AND a.closed = 0`,
    [toDateRepr(oldestDate), toDateRepr(addDays(currentDay(), 180))]
  );

  let allTransactions = partitionByField(register, 'payee');
  let categoriesToSet = new Map();

  for (let payeeId of payeeIds) {
    // Don't do anything if payee is null
    if (payeeId) {
      let latestTrans = (allTransactions.get(payeeId) || []).slice(0, 5);

      // Check if one of the latest transactions was one that was
      // updated. We only want to update anything if so.
      if (latestTrans.find(trans => transIds.has(trans.id))) {
        let category = getProbableCategory(latestTrans);
        if (category) {
          categoriesToSet.set(payeeId, category);
        }
      }
    }
  }

  await batchMessages(async () => {
    for (let [payeeId, category] of categoriesToSet.entries()) {
      let ruleSetters = [
        ...getIsSetterRules(null, 'payee', 'category', {
          condValue: payeeId
        })
      ];

      if (ruleSetters.length > 0) {
        // If there are existing rules, change all of them to the new
        // category (if they aren't already using it). We set all of
        // them because it's possible that multiple rules exist
        // because 2 clients made them independently. Not really a big
        // deal, but to make sure our update gets applied set it to
        // all of them
        for (let rule of ruleSetters) {
          let action = rule.actions[0];
          if (action.value !== category) {
            await updateRule({
              ...rule,
              actions: [{ ...action, value: category }]
            });
          }
        }
      } else {
        // No existing rules, so create one
        let newRule = new Rule({
          stage: null,
          conditions: [{ op: 'is', field: 'payee', value: payeeId }],
          actions: [{ op: 'set', field: 'category', value: category }],
          fieldTypes: FIELD_TYPES
        });
        await insertRule(newRule.serialize());
      }
    }
  });
}

// This can be removed in the future
export async function migrateOldRules() {
  let allPayees = await db.all(
    `SELECT p.*, c.id as category FROM payees p
    LEFT JOIN category_mapping cm ON cm.id = p.category
    LEFT JOIN categories c ON (c.id = cm.transferId AND c.tombstone = 0)
    WHERE p.tombstone = 0 AND transfer_acct IS NULL`
  );
  let allRules = await db.all(
    `SELECT pr.*, pm.targetId as payee_id FROM payee_rules pr
      LEFT JOIN payee_mapping pm ON pm.id = pr.payee_id
      WHERE pr.tombstone = 0`
  );

  let payeesById = new Map();
  for (let i = 0; i < allPayees.length; i++) {
    payeesById.set(allPayees[i].id, allPayees[i]);
  }

  let rulesByPayeeId = new Map();
  for (let i = 0; i < allRules.length; i++) {
    let item = allRules[i];
    let rules = rulesByPayeeId.get(item.payee_id) || [];
    rules.push(item);
    rulesByPayeeId.set(item.payee_id, rules);
  }

  let rules = [];

  // Convert payee name rules
  for (let [payeeId, payeeRules] of rulesByPayeeId.entries()) {
    let equals = payeeRules.filter(r => {
      let payee = payeesById.get(r.payee_id);

      return (
        (r.type === 'equals' || r.type == null) &&
        (!payee || r.value.toLowerCase() !== payee.name.toLowerCase())
      );
    });
    let contains = payeeRules.filter(r => r.type === 'contains');
    let actions = [{ op: 'set', field: 'payee', value: payeeId }];

    if (equals.length > 0) {
      rules.push({
        stage: null,
        conditions: [
          {
            op: 'oneOf',
            field: 'imported_payee',
            value: equals.map(payeeRule => payeeRule.value)
          }
        ],
        actions
      });
    }

    if (contains.length > 0) {
      rules = rules.concat(
        contains.map(payeeRule => ({
          stage: null,
          conditions: [
            {
              op: 'contains',
              field: 'imported_payee',
              value: payeeRule.value
            }
          ],
          actions
        }))
      );
    }
  }

  // Convert category rules
  let catRules = allPayees
    .filter(p => p.category)
    .reduce((map, payee) => {
      let ids = map.get(payee.category) || new Set();
      ids.add(payee.id);
      map.set(payee.category, ids);
      return map;
    }, new Map());

  for (let [catId, payeeIds] of catRules) {
    rules.push({
      stage: null,
      conditions: [
        {
          op: 'oneOf',
          field: 'payee',
          value: [...payeeIds]
        }
      ],
      actions: [
        {
          op: 'set',
          field: 'category',
          value: catId
        }
      ]
    });
  }

  // Very important: we never want to sync migration changes, but it
  // still has to run through the syncing layer to make sure
  // projections are correct. This is only OK because we require a
  // sync reset after this.
  let prevMode = setSyncingMode('disabled');
  await batchMessages(async () => {
    for (let rule of rules) {
      await insertRule({
        stage: rule.stage,
        conditions: rule.conditions,
        actions: rule.actions
      });
    }

    await db.runQuery('DELETE FROM payee_rules', []);
  });
  setSyncingMode(prevMode);
}