Store user files as blobs instead of unzipping them

app-sync.js

@@ -1,12 +1,11 @@
+let fs = require('fs/promises');
 let { Buffer } = require('buffer');
-let { join } = require('path');
 let express = require('express');
 let uuid = require('uuid');
-let AdmZip = require('adm-zip');
 let { validateUser } = require('./util/validate-user');
 let errorMiddleware = require('./util/error-middleware');
-let config = require('./load-config');
 let { getAccountDb } = require('./account-db');
+let { getPathForUserFile, getPathForGroupFile } = require('./util/paths');
 
 let simpleSync = require('./sync-simple');
 
@@ -167,7 +166,7 @@ app.post('/user-create-key', (req, res) => {
   res.send(JSON.stringify({ status: 'ok' }));
 });
 
-app.post('/reset-user-file', (req, res) => {
+app.post('/reset-user-file', async (req, res) => {
   let user = validateUser(req, res);
   if (!user) {
     return;
@@ -187,10 +186,11 @@ app.post('/reset-user-file', (req, res) => {
   accountDb.mutate('UPDATE files SET group_id = NULL WHERE id = ?', [fileId]);
 
   if (group_id) {
-    // TODO: Instead of doing this, just delete the db file named
+    try {
-    // after the group
+      await fs.unlink(getPathForGroupFile(group_id));
-    // db.mutate('DELETE FROM messages_binary WHERE group_id = ?', [group_id]);
+    } catch (e) {
-    // db.mutate('DELETE FROM messages_merkles WHERE group_id = ?', [group_id]);
+      console.log(`Unable to delete sync data for group "${group_id}"`);
+    }
   }
 
   res.send(JSON.stringify({ status: 'ok' }));
@@ -247,21 +247,11 @@ app.post('/upload-user-file', async (req, res) => {
     }
   }
 
-  // TODO: If we want to support end-to-end encryption, we'd write the
-  // raw file down because it's an encrypted blob. This isn't
-  // supported yet in the self-hosted version because it's unclear if
-  // it's still needed, given that you own your server
-  //
-  // await fs.writeFile(join(config.userFiles, `${fileId}.blob`), req.body);
-
-  let zip = new AdmZip(req.body);
-
   try {
-    zip.extractAllTo(join(config.userFiles, fileId), true);
+    await fs.writeFile(getPathForUserFile(fileId), req.body);
   } catch (err) {
     console.log('Error writing file', err);
     res.send(JSON.stringify({ status: 'error' }));
-    return;
   }
 
   let rows = accountDb.all('SELECT id FROM files WHERE id = ?', [fileId]);
@@ -311,15 +301,14 @@ app.get('/download-user-file', async (req, res) => {
     return;
   }
 
-  let zip = new AdmZip();
+  let buffer;
   try {
-    zip.addLocalFile(join(config.userFiles, fileId, 'db.sqlite'), '');
+    buffer = await fs.readFile(getPathForUserFile(fileId));
-    zip.addLocalFile(join(config.userFiles, fileId, 'metadata.json'), '');
   } catch (e) {
-    res.status(500).send('Error reading files');
+    console.log(`Error: file does not exist: ${getPathForUserFile(fileId)}`);
+    res.status(500).send('File does not exist on server');
     return;
   }
-  let buffer = zip.toBuffer();
 
   res.setHeader('Content-Disposition', `attachment;filename=${fileId}`);
   res.send(buffer);

package.json

@@ -14,7 +14,6 @@
   "dependencies": {
     "@actual-app/api": "4.1.0",
     "@actual-app/web": "4.1.0",
-    "adm-zip": "^0.5.9",
     "bcrypt": "^5.0.1",
     "better-sqlite3": "^7.5.0",
     "body-parser": "^1.18.3",

sync-simple.js

@@ -1,7 +1,7 @@
 let { existsSync, readFileSync } = require('fs');
 let { join } = require('path');
 let { openDatabase } = require('./db');
-let config = require('./load-config');
+let { getPathForGroupFile } = require('./util/paths');
 
 let actual = require('@actual-app/api');
 let merkle = actual.internal.merkle;
@@ -9,7 +9,7 @@ let SyncPb = actual.internal.SyncProtoBuf;
 let Timestamp = actual.internal.timestamp.default;
 
 function getGroupDb(groupId) {
-  let path = join(config.userFiles, `${groupId}.sqlite`);
+  let path = getPathForGroupFile(groupId);
   let needsInit = !existsSync(path);
 
   let db = openDatabase(path);

util/paths.js

@@ -0,0 +1,12 @@
+let { join } = require('path');
+let config = require('../load-config');
+
+function getPathForUserFile(fileId) {
+  return join(config.userFiles, `file-${fileId}.blob`);
+}
+
+function getPathForGroupFile(groupId) {
+  return join(config.userFiles, `group-${groupId}.sqlite`);
+}
+
+module.exports = { getPathForUserFile, getPathForGroupFile };

yarn.lock

@@ -201,11 +201,6 @@ acorn@^8.7.1:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30"
   integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==
 
-adm-zip@^0.5.9:
-  version "0.5.9"
-  resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.9.tgz#b33691028333821c0cf95c31374c5462f2905a83"
-  integrity sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==
-
 agent-base@6:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"