actual/packages/loot-core/src/server/accounts/sync.test.js

import * as monthUtils from '../../shared/months';
import * as db from '../db';
import { loadMappings } from '../db/mappings';
import { getServer } from '../server-config';
import {
  syncAccount,
  reconcileTransactions,
  addTransactions,
  fromPlaid
} from './sync';
import { loadRules, insertRule } from './transaction-rules';
import * as transfer from './transfer';

const snapshotDiff = require('snapshot-diff');

const { post } = require('../post');
const mockSyncServer = require('../tests/mockSyncServer');

beforeEach(async () => {
  mockSyncServer.reset();
  await global.emptyDatabase()();
  await loadMappings();
  await loadRules();
});

function getAllTransactions() {
  return db.all(
    `SELECT t.*, p.name as payee_name
       FROM v_transactions_internal t
       LEFT JOIN payees p ON p.id = t.payee
       ORDER BY date DESC, amount DESC, id
     `
  );
}

function expectSnapshotWithDiffer(initialValue) {
  let currentValue = initialValue;
  expect(initialValue).toMatchSnapshot();
  return {
    expectToMatchDiff: value => {
      expect(snapshotDiff(currentValue, value)).toMatchSnapshot();
      currentValue = value;
    }
  };
}

function prepMockTransactions() {
  let mockTransactions;
  mockSyncServer.filterMockData(data => {
    const account_id = data.accounts[0].account_id;
    const transactions = data.transactions[account_id].filter(t => !t.pending);

    mockTransactions = [
      ...transactions.filter(t => t.date <= '2017-10-15'),
      ...transactions.filter(t => t.date === '2017-10-16').slice(0, 1),
      ...transactions.filter(t => t.date === '2017-10-17').slice(0, 3)
    ];

    return {
      accounts: data.accounts,
      transactions: { [account_id]: mockTransactions }
    };
  });
  return mockTransactions;
}

async function prepareDatabase() {
  await db.insertCategoryGroup({ id: 'group1', name: 'group1', is_income: 1 });
  await db.insertCategory({
    name: 'income',
    cat_group: 'group1',
    is_income: 1
  });

  const { accounts } = await post(getServer().PLAID_SERVER + '/accounts', {
    client_id: '',
    group_id: '',
    item_id: '1'
  });
  const acct = accounts[0];

  const id = await db.insertAccount({
    id: 'one',
    account_id: acct.account_id,
    name: acct.official_name,
    balance_current: acct.balances.current
  });
  await db.insertPayee({
    id: 'transfer-' + id,
    name: '',
    transfer_acct: id
  });

  return { id, account_id: acct.account_id };
}

async function getAllPayees() {
  return (await db.getPayees()).filter(p => p.transfer_acct == null);
}

describe.skip('Account sync', () => {
  test('reconcile creates payees correctly', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    let { id, account_id } = await prepareDatabase();

    let payees = await getAllPayees();
    expect(payees.length).toBe(0);

    await reconcileTransactions(id, [
      { date: '2020-01-02', payee_name: 'bakkerij', amount: 4133 },
      { date: '2020-01-03', payee_name: 'kroger', amount: 5000 }
    ]);

    payees = await getAllPayees();
    expect(payees.length).toBe(2);

    let transactions = await getAllTransactions();
    expect(transactions.length).toBe(2);
    expect(transactions.find(t => t.amount === 4133).payee).toBe(
      payees.find(p => p.name === 'Bakkerij').id
    );
    expect(transactions.find(t => t.amount === 5000).payee).toBe(
      payees.find(p => p.name === 'Kroger').id
    );
  });

  test('reconcile matches single transaction', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    const { id, account_id } = await prepareDatabase();

    await syncAccount('userId', 'userKey', id, account_id, 'bank');

    // The payee can be anything, all that matters is the amount is the same
    let mockTransaction = mockTransactions.find(t => t.date === '2017-10-17');
    let payeeName = mockTransaction.name;
    mockTransaction.amount = 29.47;

    let payeeId = await db.insertPayee({ name: 'macy' });
    await db.insertTransaction({
      id: 'one',
      account: id,
      amount: -2947,
      date: '2017-10-15',
      payee: payeeId
    });

    let { added, updated } = await reconcileTransactions(
      id,
      mockTransactions.filter(t => t.date >= '2017-10-15').map(fromPlaid)
    );

    expect(added.length).toBe(3);
    expect(updated.length).toBe(1);

    let transactions = await getAllTransactions();
    let transaction = transactions.find(t => t.amount === -2947);
    expect(transaction.id).toBe(updated[0]);

    // The payee has not been updated - it's still the payee that the original transaction had
    let payees = await getAllPayees();
    expect(payees.length).toBe(18);
    expect(transaction.payee).toBe(payeeId);
  });

  test('reconcile matches multiple transactions', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    const { id, account_id } = await prepareDatabase();

    await syncAccount('userId', 'userKey', id, account_id, 'bank');

    // These should all match, but note that the one with the payee
    // `macy` should match with the imported one with the same payee
    // name. This should happen even though other transactions with
    // the same amount are imported first, i.e. high fidelity matches
    // always win
    let mocked = mockTransactions.filter(t => t.date === '2017-10-17');
    mocked[0].name = 'Papa Johns east side';
    mocked[0].amount = 29.47;
    mocked[1].name = "Lowe's Store";
    mocked[1].amount = 29.47;
    mocked[2].name = 'macy';
    mocked[2].amount = 29.47;

    // Make sure that it macy is correctly matched from a different
    // day first, and then the other two are matched based on amount.
    // And it should never match the same transactions twice
    await db.insertTransaction({
      id: 'one',
      account: id,
      amount: -2947,
      date: '2017-10-15',
      payee: await db.insertPayee({ name: 'papa johns' })
    });
    await db.insertTransaction({
      id: 'two',
      account: id,
      amount: -2947,
      date: '2017-10-17',
      payee: await db.insertPayee({ name: 'lowes' })
    });
    await db.insertTransaction({
      id: 'three',
      account: id,
      amount: -2947,
      date: '2017-10-17',
      payee: await db.insertPayee({ name: 'macy' })
    });

    let { added, updated } = await reconcileTransactions(
      id,
      mockTransactions.filter(t => t.date >= '2017-10-15').map(fromPlaid)
    );

    let transactions = await getAllTransactions();
    expect(updated.length).toBe(3);
    expect(added.length).toBe(1);

    expect(transactions.find(t => t.id === 'one').imported_id).toBe(
      mocked[1].transaction_id
    );
    expect(transactions.find(t => t.id === 'two').imported_id).toBe(
      mocked[0].transaction_id
    );
    expect(transactions.find(t => t.id === 'three').imported_id).toBe(
      mocked[2].transaction_id
    );
  });

  test('reconcile matches multiple transactions (imported_id wins)', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    const { id, account_id } = await prepareDatabase();

    await syncAccount('userId', 'userKey', id, account_id, 'bank');

    let mocked = mockTransactions.filter(t => t.date === '2017-10-17');
    mocked[0].name = 'Papa Johns east side';
    mocked[0].amount = 29.47;
    mocked[1].name = "Lowe's Store";
    mocked[1].amount = 29.47;
    mocked[1].transaction_id = 'imported1';

    // Technically, the amount doesn't even matter. The
    // imported_id will always match no matter what
    await db.insertTransaction({
      id: 'one',
      account: id,
      amount: -3000,
      date: '2017-10-15',
      imported_id: 'imported1',
      payee: await db.insertPayee({ name: 'papa johns' })
    });
    await db.insertTransaction({
      id: 'two',
      account: id,
      amount: -2947,
      date: '2017-10-17',
      payee: await db.insertPayee({ name: 'lowes' })
    });

    let { added, updated } = await reconcileTransactions(
      id,
      mockTransactions.filter(t => t.date >= '2017-10-15').map(fromPlaid)
    );

    let transactions = await getAllTransactions();
    expect(updated).toEqual(['two', 'one']);
    expect(added.length).toBe(2);

    // Make sure lowes, which has the imported_id, is the one that
    // got matched with the same imported_id
    expect(transactions.find(t => t.id === 'one').imported_payee).toBe(
      "Lowe's Store"
    );
  });

  test('import never matches existing with financial ids', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    const { id, account_id } = await prepareDatabase();

    await syncAccount('userId', 'userKey', id, account_id, 'bank');
    let differ = expectSnapshotWithDiffer(await getAllTransactions());

    mockTransactions = mockTransactions.filter(t => t.date === '2017-10-17');
    mockTransactions[0].name = 'foo';
    mockTransactions[0].amount = 29.47;
    mockTransactions[1].name = 'bar';
    mockTransactions[1].amount = 29.47;

    // Make sure, no matter what, it never tries to match with an
    // existing transaction that already has a financial id
    await db.insertTransaction({
      id: 'one',
      account: id,
      amount: -2947,
      date: '2017-10-15',
      payee: await db.insertPayee({ name: 'foo' }),
      imported_id: 'trans1'
    });

    await db.insertTransaction({
      id: 'two',
      account: id,
      amount: -2947,
      date: '2017-10-15',
      payee: await db.insertPayee({ name: 'bar' }),
      imported_id: 'trans2'
    });

    differ.expectToMatchDiff(await getAllTransactions());

    monthUtils.currentDay = () => '2017-10-17';
    await syncAccount('userId', 'userKey', id, account_id, 'bank');

    differ.expectToMatchDiff(await getAllTransactions());
  });

  test('import updates transfers when matched', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    let mockTransactions = prepMockTransactions();
    const { id, account_id } = await prepareDatabase();
    await db.insertAccount({ id: 'two', name: 'two' });
    await db.insertPayee({
      id: 'transfer-two',
      name: '',
      transfer_acct: 'two'
    });

    await syncAccount('userId', 'userKey', id, account_id, 'bank');
    let differ = expectSnapshotWithDiffer(await getAllTransactions());

    const mockTransaction = mockTransactions.find(t => t.date === '2017-10-17');
    mockTransaction.name = "#001 fenn st Macy's 33333 EMX";
    mockTransaction.amount = 29.48;

    const transactionId = await db.insertTransaction({
      id: 'one',
      account: 'two',
      amount: 2948,
      date: '2017-10-15',
      payee: 'transfer-' + id
    });
    await transfer.onInsert(await db.getTransaction(transactionId));

    differ.expectToMatchDiff(await getAllTransactions());

    monthUtils.currentDay = () => '2017-10-17';
    await syncAccount('userId', 'userKey', id, account_id, 'bank');

    // Don't use `differ.expectToMatchDiff` because there's too many
    // changes that look too confusing
    expect(await getAllTransactions()).toMatchSnapshot();
  });

  test('reconcile handles transactions with undefined fields', async () => {
    const { id: acctId } = await prepareDatabase();

    const transactionId = await db.insertTransaction({
      id: 'one',
      account: acctId,
      amount: 2948,
      date: '2020-01-01'
    });

    await reconcileTransactions(acctId, [
      { date: '2020-01-02' },
      { date: '2020-01-01', amount: 2948 }
    ]);

    let transactions = await getAllTransactions();
    expect(transactions.length).toBe(2);
    expect(transactions).toMatchSnapshot();

    // No payees should be created
    let payees = await getAllPayees();
    expect(payees.length).toBe(0);

    // Make _at least_ the date is required
    await expect(reconcileTransactions(acctId, [{}])).rejects.toThrow(
      /`date` is required/
    );
  });

  test('reconcile run rules with inferred payee', async () => {
    const { id: acctId } = await prepareDatabase();
    await db.insertCategoryGroup({
      id: 'group2',
      name: 'group2'
    });
    let catId = await db.insertCategory({
      name: 'Food',
      cat_group: 'group2'
    });

    let payeeId = await db.insertPayee({ name: 'bakkerij' });

    await insertRule({
      stage: null,
      conditions: [{ op: 'is', field: 'payee', value: payeeId }],
      actions: [{ op: 'set', field: 'category', value: catId }]
    });

    await reconcileTransactions(acctId, [
      { date: '2020-01-02', payee_name: 'Bakkerij', amount: 4133 }
    ]);

    let transactions = await getAllTransactions();
    // Even though the payee was inferred from the string name (no
    // renaming rules ran), it should match the above rule and set the
    // category
    expect(transactions.length).toBe(1);
    expect(transactions[0].payee).toBe(payeeId);
    expect(transactions[0].category).toBe(catId);

    // It also should not have created a payee
    let payees = await getAllPayees();
    expect(payees.length).toBe(1);
    expect(payees[0].id).toBe(payeeId);
  });

  test('reconcile avoids creating blank payees', async () => {
    const { id: acctId } = await prepareDatabase();

    await reconcileTransactions(acctId, [
      { date: '2020-01-02', payee_name: '     ', amount: 4133 }
    ]);

    let transactions = await getAllTransactions();
    // Even though the payee was inferred from the string name (no
    // renaming rules ran), it should match the above rule and set the
    // category
    expect(transactions.length).toBe(1);
    expect(transactions[0].payee).toBe(null);
    expect(transactions[0].amount).toBe(4133);
    expect(transactions[0].date).toBe(20200102);

    // It also should not have created a payee
    let payees = await getAllPayees();
    expect(payees.length).toBe(0);
  });

  test('reconcile run rules dont create unnecessary payees', async () => {
    const { id: acctId } = await prepareDatabase();

    let payeeId = await db.insertPayee({ name: 'bakkerij-renamed' });

    await insertRule({
      stage: null,
      conditions: [{ op: 'is', field: 'imported_payee', value: 'Bakkerij' }],
      actions: [{ op: 'set', field: 'payee', value: payeeId }]
    });

    await reconcileTransactions(acctId, [
      { date: '2020-01-02', payee_name: 'bakkerij', amount: 4133 }
    ]);

    let payees = await getAllPayees();
    expect(payees.length).toBe(1);
    expect(payees[0].id).toBe(payeeId);

    let transactions = await getAllTransactions();
    expect(transactions.length).toBe(1);
    expect(transactions[0].payee).toBe(payeeId);
  });

  let testMapped = version => {
    test.skip(`reconcile matches unmapped and mapped payees (${version})`, async () => {
      const { id: acctId } = await prepareDatabase();

      if (version === 'v1') {
        // This is quite complicated, but important to test. If a payee is
        // merged with another, a rule sets the payee of a transaction to
        // the updated one, make sure it still matches an existing
        // transaction that points to the old merged payee
      } else if (version === 'v2') {
        // This is similar to v1, but inverted: make sure that
        // if a rule sets the payee to an *old* payee, that it still
        // matches to a transaction with the new payee that it was merged
        // to
      }

      let payeeId1 = await db.insertPayee({ name: 'bakkerij2' });
      let payeeId2 = await db.insertPayee({ name: 'bakkerij-renamed' });

      // Insert a rule *before* payees are merged. Not that v2 would
      // fail if we inserted this rule after, because the rule would
      // set to an *old* payee but the matching would take place on a
      // *new* payee. But that's ok - it would fallback to matching
      // amount anyway, so while it loses some fidelity, it's an edge
      // case that we don't need to worry much about because the user
      // shouldn't be able able to create rules for a merged payee.
      // Unless they sync in a rule...
      await insertRule({
        stage: null,
        conditions: [{ op: 'is', field: 'imported_payee', value: 'Bakkerij' }],
        actions: [{ op: 'set', field: 'payee', value: payeeId2 }]
      });

      if (version === 'v1') {
        await db.mergePayees(payeeId2, [payeeId1]);
      } else if (version === 'v2') {
        await db.mergePayees(payeeId1, [payeeId2]);
      }

      await db.insertTransaction({
        id: 'one',
        account: acctId,
        amount: -2947,
        date: '2017-10-15',
        payee: payeeId1
      });
      // It will try to match to this one first, make sure it matches
      // the above transaction though
      await db.insertTransaction({
        id: 'two',
        account: acctId,
        amount: -2947,
        date: '2017-10-17',
        payee: null
      });

      let { added, updated } = await reconcileTransactions(acctId, [
        {
          date: '2017-10-17',
          payee_name: 'bakkerij',
          amount: -2947,
          imported_id: 'imported1'
        }
      ]);

      let payees = await getAllPayees();
      expect(payees.length).toBe(1);
      expect(payees[0].id).toBe(version === 'v1' ? payeeId2 : payeeId1);

      expect(updated.length).toBe(1);
      expect(updated[0]).toBe('one');

      let transactions = await getAllTransactions();
      expect(transactions.length).toBe(2);
      expect(transactions.find(t => t.id === 'one').imported_id).toBe(
        'imported1'
      );
    });
  };

  testMapped('v1');
  testMapped('v2');

  test('addTransactions simply adds transactions', async () => {
    const { id: acctId } = await prepareDatabase();

    let payeeId = await db.insertPayee({ name: 'bakkerij-renamed' });

    // Make sure it still runs rules
    await insertRule({
      stage: null,
      conditions: [{ op: 'is', field: 'imported_payee', value: 'Bakkerij' }],
      actions: [{ op: 'set', field: 'payee', value: payeeId }]
    });

    let transactions = [
      {
        date: '2017-10-17',
        payee_name: 'BAKKerij',
        amount: -2947
      },
      {
        date: '2017-10-18',
        payee_name: 'bakkERIj2',
        amount: -2947
      },
      {
        date: '2017-10-19',
        payee_name: 'bakkerij3',
        amount: -2947
      },
      {
        date: '2017-10-20',
        payee_name: 'BakkeriJ3',
        amount: -2947
      }
    ];

    let added = await addTransactions(acctId, transactions);
    expect(added.length).toBe(transactions.length);

    let payees = await getAllPayees();
    expect(payees.length).toBe(3);

    let getName = id => payees.find(p => p.id === id).name;

    let allTransactions = await getAllTransactions();
    expect(allTransactions.length).toBe(4);
    expect(allTransactions.map(t => getName(t.payee))).toEqual([
      'bakkerij3',
      'bakkerij3',
      'bakkERIj2',
      'bakkerij-renamed'
    ]);
  });

  test('imports transactions for current day and adds latest', async () => {
    monthUtils.currentDay = () => '2017-10-15';
    monthUtils.currentMonth = () => '2017-10';

    const { id, account_id } = await prepareDatabase();

    expect((await getAllTransactions()).length).toBe(0);
    await syncAccount('userId', 'userKey', id, account_id, 'bank');
    expect(await getAllTransactions()).toMatchSnapshot();

    monthUtils.currentDay = () => '2017-10-17';

    await syncAccount('userId', 'userKey', id, account_id, 'bank');
    expect(await getAllTransactions()).toMatchSnapshot();
  });
});