Fix reference to Timestamp; version 22.12.09

Build docker image on push to master or tag

.github/workflows/docker.yml

@@ -1,10 +1,31 @@
 name: Build Docker Image
 
-# Docker Images are only built when a new tag is created
+# Docker Images are built for every push to master or when a new tag is created
 on:
   push:
+    branches:
+      - master
     tags:
       - 'v*.*.*'
+    paths-ignore:
+      - README.md
+      - LICENSE.txt
+
+env:
+  IMAGES: |
+    jlongster/actual-server
+    ghcr.io/actualbudget/actual-server
+
+  # Creates the following tags:
+  # - actual-server:latest (see docker/metadata-action flavor inputs, below)
+  # - actual-server:edge (for master)
+  # - actual-server:1.3
+  # - actual-server:1.3.7
+  # - actual-server:sha-90dd603
+  TAGS: |
+    type=edge,value=edge
+    type=semver,pattern={{version}}
+    type=sha
 
 jobs:
   build:
@@ -24,35 +45,22 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           # Push to both Docker Hub and Github Container Registry
-          images: |
-            jlongster/actual-server
-            ghcr.io/actualbudget/actual-server
-          # Creates the following tags:
-          # - actual-server:latest
-          # - actual-server:1.3
-          # - actual-server:1.3.7
-          # - actual-server:sha-90dd603
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=sha
+          images: ${{ env.IMAGES }}
+          # Automatically update :latest for our semver tags
+          flavor: |
+            latest=auto
+          tags: ${{ env.TAGS }}
 
       - name: Docker meta for Alpine image
         id: alpine-meta
         uses: docker/metadata-action@v4
         with:
-          images: |
-            jlongster/actual-server
-            ghcr.io/actualbudget/actual-server
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=raw,value=latest,suffix=-alpine
-            type=semver,pattern={{version}},suffix=-alpine
-            type=semver,pattern={{major}}.{{minor}},suffix=-alpine
-            type=sha,suffix=-alpine
+          images: ${{ env.IMAGES }}
+          # Automatically update :latest for our semver tags and suffix all tags
+          flavor: |
+            latest=auto
+            suffix=-alpine,onlatest=true
+          tags: $${{ env.TAGS }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@v1

app-sync.js

@@ -1,14 +1,13 @@
+let fs = require('fs/promises');
 let { Buffer } = require('buffer');
-let { join } = require('path');
 let express = require('express');
 let uuid = require('uuid');
-let AdmZip = require('adm-zip');
 let { validateUser } = require('./util/validate-user');
 let errorMiddleware = require('./util/error-middleware');
-let config = require('./load-config');
 let { getAccountDb } = require('./account-db');
+let { getPathForUserFile, getPathForGroupFile } = require('./util/paths');
 
-let fullSync = require('./sync-full');
+let simpleSync = require('./sync-simple');
 
 let actual = require('@actual-app/api');
 let SyncPb = actual.internal.SyncProtoBuf;
@@ -16,17 +15,8 @@ let SyncPb = actual.internal.SyncProtoBuf;
 const app = express();
 app.use(errorMiddleware);
 
-async function init() {
-  let fileDir = join(process.env.ACTUAL_USER_FILES || config.userFiles);
-
-  console.log('Initializing Actual with user file dir:', fileDir);
-
-  await actual.init({
-    config: {
-      dataDir: fileDir
-    }
-  });
-}
+// eslint-disable-next-line
+async function init() {}
 
 // This is a version representing the internal format of sync
 // messages. When this changes, all sync files need to be reset. We
@@ -121,24 +111,15 @@ app.post('/sync', async (req, res) => {
     return false;
   }
 
-  // TODO: We also provide a "simple" sync method which currently isn't
-  // used. This method just stores the messages locally and doesn't
-  // load the whole app at all. If we want to support end-to-end
-  // encryption, this method is required because we can't read the
-  // messages. Using it looks like this:
-  //
-  // let simpleSync = require('./sync-simple');
-  // let {trie, newMessages } = simpleSync.sync(messages, since, file_id);
-
-  let { trie, newMessages } = await fullSync.sync(messages, since, file_id);
+  let { trie, newMessages } = simpleSync.sync(messages, since, group_id);
 
   // encode it back...
   let responsePb = new SyncPb.SyncResponse();
   responsePb.setMerkle(JSON.stringify(trie));
-
   newMessages.forEach((msg) => responsePb.addMessages(msg));
 
   res.set('Content-Type', 'application/actual-sync');
+  res.set('X-ACTUAL-SYNC-METHOD', 'simple');
   res.send(Buffer.from(responsePb.serializeBinary()));
 });
 
@@ -185,7 +166,7 @@ app.post('/user-create-key', (req, res) => {
   res.send(JSON.stringify({ status: 'ok' }));
 });
 
-app.post('/reset-user-file', (req, res) => {
+app.post('/reset-user-file', async (req, res) => {
   let user = validateUser(req, res);
   if (!user) {
     return;
@@ -205,10 +186,11 @@ app.post('/reset-user-file', (req, res) => {
   accountDb.mutate('UPDATE files SET group_id = NULL WHERE id = ?', [fileId]);
 
   if (group_id) {
-    // TODO: Instead of doing this, just delete the db file named
-    // after the group
-    // db.mutate('DELETE FROM messages_binary WHERE group_id = ?', [group_id]);
-    // db.mutate('DELETE FROM messages_merkles WHERE group_id = ?', [group_id]);
+    try {
+      await fs.unlink(getPathForGroupFile(group_id));
+    } catch (e) {
+      console.log(`Unable to delete sync data for group "${group_id}"`);
+    }
   }
 
   res.send(JSON.stringify({ status: 'ok' }));
@@ -265,21 +247,11 @@ app.post('/upload-user-file', async (req, res) => {
     }
   }
 
-  // TODO: If we want to support end-to-end encryption, we'd write the
-  // raw file down because it's an encrypted blob. This isn't
-  // supported yet in the self-hosted version because it's unclear if
-  // it's still needed, given that you own your server
-  //
-  // await fs.writeFile(join(config.userFiles, `${fileId}.blob`), req.body);
-
-  let zip = new AdmZip(req.body);
-
   try {
-    zip.extractAllTo(join(config.userFiles, fileId), true);
+    await fs.writeFile(getPathForUserFile(fileId), req.body);
   } catch (err) {
     console.log('Error writing file', err);
     res.send(JSON.stringify({ status: 'error' }));
-    return;
   }
 
   let rows = accountDb.all('SELECT id FROM files WHERE id = ?', [fileId]);
@@ -306,6 +278,7 @@ app.post('/upload-user-file', async (req, res) => {
       'UPDATE files SET sync_version = ?, encrypt_meta = ?, name = ? WHERE id = ?',
       [syncFormatVersion, encryptMeta, name, fileId]
     );
+
     res.send(JSON.stringify({ status: 'ok', groupId }));
   }
 });
@@ -328,15 +301,14 @@ app.get('/download-user-file', async (req, res) => {
     return;
   }
 
-  let zip = new AdmZip();
+  let buffer;
   try {
-    zip.addLocalFile(join(config.userFiles, fileId, 'db.sqlite'), '');
-    zip.addLocalFile(join(config.userFiles, fileId, 'metadata.json'), '');
+    buffer = await fs.readFile(getPathForUserFile(fileId));
   } catch (e) {
-    res.status(500).send('Error reading files');
+    console.log(`Error: file does not exist: ${getPathForUserFile(fileId)}`);
+    res.status(500).send('File does not exist on server');
     return;
   }
-  let buffer = zip.toBuffer();
 
   res.setHeader('Content-Disposition', `attachment;filename=${fileId}`);
   res.send(buffer);

load-config.js

@@ -16,4 +16,7 @@ try {
   };
 }
 
+// The env variable always takes precedence
+config.userFiles = process.env.ACTUAL_USER_FILES || config.userFiles;
+
 module.exports = config;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "actual-sync",
-  "version": "1.0.1",
+  "version": "22.12.09",
   "license": "MIT",
   "description": "actual syncing server",
   "main": "index.js",
@@ -12,9 +12,8 @@
     "verify": "yarn -s lint && yarn types"
   },
   "dependencies": {
-    "@actual-app/api": "4.1.0",
-    "@actual-app/web": "4.1.0",
-    "adm-zip": "^0.5.9",
+    "@actual-app/api": "4.1.5",
+    "@actual-app/web": "22.12.3",
     "bcrypt": "^5.0.1",
     "better-sqlite3": "^7.5.0",
     "body-parser": "^1.18.3",

sql/messages.sql

@@ -1,10 +1,9 @@
 
 CREATE TABLE messages_binary
-  (timestamp TEXT,
+  (timestamp TEXT PRIMARY KEY,
    is_encrypted BOOLEAN,
-   content bytea,
-   PRIMARY KEY(timestamp, group_id));
+   content bytea);
 
 CREATE TABLE messages_merkles
-  (id TEXT PRIMAREY KEY,
+  (id INTEGER PRIMARY KEY,
    merkle TEXT);

sync-simple.js

@@ -1,13 +1,15 @@
 let { existsSync, readFileSync } = require('fs');
 let { join } = require('path');
 let { openDatabase } = require('./db');
+let { getPathForGroupFile } = require('./util/paths');
 
 let actual = require('@actual-app/api');
 let merkle = actual.internal.merkle;
+let SyncPb = actual.internal.SyncProtoBuf;
 let Timestamp = actual.internal.timestamp.Timestamp;
 
 function getGroupDb(groupId) {
-  let path = join(__dirname, `user-files/${groupId}.sqlite`);
+  let path = getPathForGroupFile(groupId);
   let needsInit = !existsSync(path);
 
   let db = openDatabase(path);
@@ -56,8 +58,8 @@ function addMessages(db, messages) {
   return returnValue;
 }
 
-function getMerkle(db, group_id) {
-  let rows = db.all('SELECT * FROM messages_merkles', [group_id]);
+function getMerkle(db) {
+  let rows = db.all('SELECT * FROM messages_merkles');
 
   if (rows.length > 0) {
     return JSON.parse(rows[0].merkle);
@@ -68,8 +70,8 @@ function getMerkle(db, group_id) {
   }
 }
 
-function sync(messages, since, fileId) {
-  let db = getGroupDb(fileId);
+function sync(messages, since, groupId) {
+  let db = getGroupDb(groupId);
   let newMessages = db.all(
     `SELECT * FROM messages_binary
          WHERE timestamp > ?
@@ -79,7 +81,18 @@ function sync(messages, since, fileId) {
 
   let trie = addMessages(db, messages);
 
-  return { trie, newMessages };
+  db.close();
+
+  return {
+    trie,
+    newMessages: newMessages.map((msg) => {
+      const envelopePb = new SyncPb.MessageEnvelope();
+      envelopePb.setTimestamp(msg.timestamp);
+      envelopePb.setIsencrypted(msg.is_encrypted);
+      envelopePb.setContent(msg.content);
+      return envelopePb;
+    })
+  };
 }
 
 module.exports = { sync };

util/paths.js

@@ -0,0 +1,12 @@
+let { join } = require('path');
+let config = require('../load-config');
+
+function getPathForUserFile(fileId) {
+  return join(config.userFiles, `file-${fileId}.blob`);
+}
+
+function getPathForGroupFile(groupId) {
+  return join(config.userFiles, `group-${groupId}.sqlite`);
+}
+
+module.exports = { getPathForUserFile, getPathForGroupFile };

yarn.lock

@@ -2,19 +2,19 @@
 # yarn lockfile v1
 
 
-"@actual-app/api@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@actual-app/api/-/api-4.1.0.tgz#43580a56a370a12dae5668b8d56c540c1849c4c9"
-  integrity sha512-8tLuA6zDnN0NjD9wui5fB/cSe0hr/iYWOHGyW/ikJoGPrCu8N8zIITz2FhfplD2QDszgAsTRt48H2YtPmR9plg==
+"@actual-app/api@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@actual-app/api/-/api-4.1.5.tgz#727f7e2233dd29204d2b7e244a671b3b88d0683a"
+  integrity sha512-rI4xqaHt9UNxovSPo8tukjluuESlxscMXUkImyYV4gYzAETsSFET3k0708Zw0EZsXCVanqAtfv+v84SNEBqn0A==
   dependencies:
     better-sqlite3 "^7.5.0"
     node-fetch "^1.6.3"
     uuid "3.3.2"
 
-"@actual-app/web@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@actual-app/web/-/web-4.1.0.tgz#9a5c5d5fd3c6e5a1a34dcaf087fb1920a27f13eb"
-  integrity sha512-QgxrHBUoDXsUCRzQTD4PmDkTt27UUcPfFJMqXPHtk2lP0ey7iVsZIW7AUeYkEs1+ghmqVVqk+jw3OAD4XQzbRA==
+"@actual-app/web@22.12.3":
+  version "22.12.3"
+  resolved "https://registry.yarnpkg.com/@actual-app/web/-/web-22.12.3.tgz#e171fcd2bc3cdeea5cd87d879cc28986e3685a0f"
+  integrity sha512-Ii6xbISEfDlLP8X+ZUYwyINuiJF1r8PHGy83OZl9lx6IbBj+d81Z8l2Qv5fhizmqHin8JytfZt0dR7rmpduVRg==
 
 "@eslint/eslintrc@^1.2.3":
   version "1.2.3"
@@ -201,11 +201,6 @@ acorn@^8.7.1:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30"
   integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==
 
-adm-zip@^0.5.9:
-  version "0.5.9"
-  resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.9.tgz#b33691028333821c0cf95c31374c5462f2905a83"
-  integrity sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==
-
 agent-base@6:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"