Compare commits

..

65 commits

Author SHA1 Message Date
James Long
97081d46c4 Fix reference to Timestamp; version 22.12.09 2022-12-08 17:38:26 -05:00
James Long
60f83b334e 22.12.08 2022-12-08 16:00:20 -05:00
James Long
340ac869ce Bump API to working version 2022-12-08 16:00:01 -05:00
Trevor Farlow
b2b6ba4921 Docker workflow: Drop major.minor tagging 2022-12-08 14:53:48 -05:00
James Long
3274a06e34 Bump to 22.12.03 2022-12-08 12:11:21 -05:00
James Long
e9850bfc56 Bump @actual-app/web and @actual-app/api 2022-12-08 11:30:07 -05:00
James Long
529c42cccf Version 22.10.25 of actual-server 2022-10-25 12:19:40 -04:00
Rich In SQL
2a00227486 Update Actual to 22.10.25 2022-10-25 11:28:16 -04:00
Trevor Farlow
5252edbf70
Merge pull request #82 from trevdor/patch-1
Build docker image on push to master or tag
2022-10-19 21:32:16 -06:00
Trevor Farlow
74c15b4f42
fixup! Build docker image on push to master or tag 2022-10-13 00:19:59 -06:00
Trevor Farlow
d482b9baf6
Build docker image on push to master or tag 2022-10-13 00:17:12 -06:00
James Long
cde216523e Store user files as blobs instead of unzipping them 2022-10-05 21:47:14 -04:00
James Long
8aeb815b5a Respect configuration for user-files 2022-09-15 10:19:22 -04:00
James Long
3c602268e3 Log sync method in response 2022-09-14 23:43:12 -04:00
James Long
9177fb4d77 Fix lint 2022-09-14 23:43:12 -04:00
James Long
e3f1fafad9 Switch syncing to simple sync method 2022-09-14 23:43:12 -04:00
Arthur E. Jones
32bf923c1a build: add node GC argument to fly template
Fly deployments on the free tier have ~256mb of memory available. Users
with large transaction histories were encountering out of memory errors
when attempting to export their data. This commit adds a node argument
to (more or less) run the garbage collector at a smaller memory usage,
helping keep users on flyio within their available limit.
2022-08-31 23:51:05 -04:00
Arthur E. Jones
d3a0e8067e build: add tini subreaper arg to fly template
Fly deployments with the previous template setting are running without
tini's subreaper capabilities. This change enables tini as a subreaper
in that environment.
2022-08-31 23:50:26 -04:00
Arthur E. Jones
105d5007cf fix: zip only necessary files in budget download
While investigating #54 it was noted that the previous implementation
zips the entire budget folder in the download endpoint. Once received on
the client side, only the most recent db and metadata are actually used.
This means up to 10 backups are being zipped in memory and transferred
to the client (in addition to the two necessary files) despite none of
that data being used. While this inefficiency isn't a major concern in
some environments, it may be problematic in memory constrained
environments.

This change transfers only the files that are actually utilized.

issue #54
2022-08-31 23:49:52 -04:00
Tom French
bafa486668 style: ignore unused variables which begin with an underscore 2022-08-29 23:11:51 -04:00
Tom French
80a2b34d43 style: apply prettier fixes 2022-08-29 23:11:51 -04:00
Tom French
a5e1e38e74 fix: activate prettier rules 2022-08-29 23:11:51 -04:00
Tom French
0486e9e37b ci: check that the server builds correctly on PRs/master 2022-08-29 23:11:02 -04:00
Tom French
09722d8678 ci: rename github workflow file from build to docker 2022-08-29 23:11:02 -04:00
Rich In SQL
cd22e38660 Express version update
Update: Updated express version, this resolves #69
2022-08-09 10:41:29 -04:00
James Long
f83fe76280 1.0.1 2022-06-26 17:33:32 -04:00
James Long
b9e1e6030f Remove changes to messages transform 2022-06-26 17:31:55 -04:00
brend
4874b53c7c move messages generation into sync-full.js 2022-06-26 17:31:55 -04:00
brend
b1a48f4f27 fix 'Out of sync' error 2022-06-26 17:31:55 -04:00
Jared Gharib
3fee9cbb42 fix: error handling middleware signature
Error handling middleware functions must have four arguments to
identify it as such.

Fixes #40
2022-05-28 23:16:46 -04:00
James Long
a2a460a883 Fix CORS issues 2022-05-28 23:15:14 -04:00
James Long
6bcd67a906 Bump Actual to 4.1.0 2022-05-28 23:14:36 -04:00
Arthur E. Jones
9e2d253fb6 fix: correct tsconfig for node.js
- specify module type, resolution, and interop. this package runs under
  nodejs, not a browser environment, and needs ts to be configured
  accordingly.
2022-05-20 13:58:48 -04:00
Arthur E. Jones
a7efc82944 fix: suppress missing module error
- code as written expects the file may be absent and has a fallback
  implemented, so the error can be safely ignored. There may be a better
  strategy for dealing with this, however.
2022-05-20 13:58:48 -04:00
Arthur E. Jones
25f4bb5557 fix: remove unused variable in fn call
- the WrappedDatabase.all() function only has two arguments
2022-05-20 13:58:48 -04:00
Arthur E. Jones
74d6b7edc5 build: add ts build script 2022-05-20 13:58:48 -04:00
Arthur E. Jones
1204b5b1a6 chore: add build dir to gitignore 2022-05-20 13:58:48 -04:00
Arthur E. Jones
59ddc965ec build: skip typechecking app-plaid.js
Doesn't appear to be used; there isn't any plaid depedency in the
package file.
2022-05-20 13:58:48 -04:00
Arthur E. Jones
9cc4ffaf33 build: add typecheck script 2022-05-20 13:58:48 -04:00
Arthur E. Jones
11ba63d086 chore: add better-sqlite3 type definitions 2022-05-20 13:58:48 -04:00
Arthur E. Jones
06d2aba57c chore: add node type definitions 2022-05-20 13:58:48 -04:00
Tom French
7ecaad529f ci: rename linting workflow 2022-05-20 12:19:22 -04:00
Tom French
5ef3aa4153 ci: check linting in CI 2022-05-20 12:19:22 -04:00
Tom French
5e83e14637 style: silence error for old-style require statements
This will be removed once we migrate to typescript
2022-05-20 09:24:19 -04:00
Tom French
8dbc10efd7 style: silence warning for empty function 2022-05-20 09:24:19 -04:00
Tom French
592f0540f9 style: fix linting errors 2022-05-20 09:24:19 -04:00
Tom French
0e28f77a1f build: add prettier plugin 2022-05-20 09:24:19 -04:00
Tom French
618609dbfa build: migrate to use typescript compatible linter setup 2022-05-20 09:24:19 -04:00
James Long
c86f1f5546 Add pikapods under one-click hosted (closes #20) 2022-05-19 22:52:32 -04:00
Manu
a09a028dc9 Move to deployment options 2 2022-05-19 22:51:42 -04:00
Manu
44da71bcdd Move to deployment options 2022-05-19 22:51:38 -04:00
Manu
343ea0c306 Add one-click hosting option. 2022-05-19 22:50:40 -04:00
James Long
a60f22ef5c Update README.md 2022-05-19 12:43:55 -04:00
Kovah
d614070f44 Update readme with details about the Docker images 2022-05-19 08:16:40 -04:00
Kovah
fd5d81e399 Adjust Docker hub image name 2022-05-19 08:16:40 -04:00
Kovah
841d3ac115 Fix tag handling for standard and alpine image 2022-05-19 08:16:40 -04:00
Kovah
3df101a91d Update build process with requested changes
- push to both Docker Hub and Github registry
- use git tags for versioning
- update Docker tags to reflect the git tags and commit SHA
2022-05-19 08:16:40 -04:00
Kovah
3124c29052 Add additional build step for the Alpine image 2022-05-19 08:16:40 -04:00
Kovah
a4a4eda0eb Fix typo 2022-05-19 08:16:40 -04:00
Kovah
7d418b91b4 Update checkout action to v3 2022-05-19 08:16:40 -04:00
Kovah
9ef7771adc Add Github Actions workflow to automatically build a Docker image on pushes to master 2022-05-19 08:16:40 -04:00
Chris
d0a8b678d3 Correct fly template port
Deployments to fly.io were failing with error:
> Failed due to unhealthy allocations
Fixes #16
2022-05-19 07:25:50 -04:00
Kk-ships
6d52cc7c73 Update docker-compose.yml
Co-authored-by: Nathan Isaac <nathanjisaac@users.noreply.github.com>
2022-05-19 07:19:33 -04:00
Kaustubh Shirpurkar
2acc034fc1 added a basic docker-compose file 2022-05-19 07:19:33 -04:00
James Long
51a2cb91f2
Update README.md 2022-05-05 14:25:03 -04:00
26 changed files with 1011 additions and 1290 deletions

5
.eslintignore Normal file
View file

@ -0,0 +1,5 @@
**/node_modules/*
**/log/*
**/shared/*
supervise

22
.eslintrc.js Normal file
View file

@ -0,0 +1,22 @@
module.exports = {
root: true,
env: {
browser: true,
amd: true,
node: true
},
parser: '@typescript-eslint/parser',
plugins: ['@typescript-eslint', 'prettier'],
extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'],
rules: {
'prettier/prettier': 'error',
'@typescript-eslint/no-unused-vars': [
'warn',
{
argsIgnorePattern: '^_'
}
],
'@typescript-eslint/no-var-requires': 'off'
}
};

29
.github/workflows/build.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Build
on:
push:
branches:
- master
pull_request:
branches: '*'
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install node
uses: actions/setup-node@v1
with:
node-version: 16
- name: Cache
uses: actions/cache@v2
id: cache
with:
path: '**/node_modules'
key: yarn-v1-${{ hashFiles('**/yarn.lock') }}
- name: Install
run: yarn --immutable
if: steps.cache.outputs.cache-hit != 'true'
- name: Build
run: yarn build

93
.github/workflows/docker.yml vendored Normal file
View file

@ -0,0 +1,93 @@
name: Build Docker Image
# Docker Images are built for every push to master or when a new tag is created
on:
push:
branches:
- master
tags:
- 'v*.*.*'
paths-ignore:
- README.md
- LICENSE.txt
env:
IMAGES: |
jlongster/actual-server
ghcr.io/actualbudget/actual-server
# Creates the following tags:
# - actual-server:latest (see docker/metadata-action flavor inputs, below)
# - actual-server:edge (for master)
# - actual-server:1.3
# - actual-server:1.3.7
# - actual-server:sha-90dd603
TAGS: |
type=edge,value=edge
type=semver,pattern={{version}}
type=sha
jobs:
build:
name: Build Docker image
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
# Push to both Docker Hub and Github Container Registry
images: ${{ env.IMAGES }}
# Automatically update :latest for our semver tags
flavor: |
latest=auto
tags: ${{ env.TAGS }}
- name: Docker meta for Alpine image
id: alpine-meta
uses: docker/metadata-action@v4
with:
images: ${{ env.IMAGES }}
# Automatically update :latest for our semver tags and suffix all tags
flavor: |
latest=auto
suffix=-alpine,onlatest=true
tags: $${{ env.TAGS }}
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push standard image
uses: docker/build-push-action@v2
with:
context: .
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
- name: Build and push Alpine image
uses: docker/build-push-action@v2
with:
context: .
push: true
file: Dockerfile.alpine
platforms: linux/amd64,linux/arm64
tags: ${{ steps.alpine-meta.outputs.tags }}

29
.github/workflows/lint.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Linter
on:
push:
branches:
- master
pull_request:
branches: '*'
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install node
uses: actions/setup-node@v1
with:
node-version: 16
- name: Cache
uses: actions/cache@v2
id: cache
with:
path: '**/node_modules'
key: yarn-v1-${{ hashFiles('**/yarn.lock') }}
- name: Install
run: yarn --immutable
if: steps.cache.outputs.cache-hit != 'true'
- name: Lint
run: yarn lint

1
.gitignore vendored
View file

@ -8,3 +8,4 @@ bin/large-sync-data.txt
user-files user-files
server-files server-files
fly.toml fly.toml
build/

4
.prettierrc.json Normal file
View file

@ -0,0 +1,4 @@
{
"singleQuote": true,
"trailingComma": "none"
}

View file

@ -1,11 +1,10 @@
This is the main project to run [Actual](https://github.com/actualbudget/actual), a local-first personal finance tool. It comes with the latest version of Actual, and a server to persist changes and make data available across all devices. This is the main project to run [Actual](https://github.com/actualbudget/actual), a local-first personal finance tool. It comes with the latest version of Actual, and a server to persist changes and make data available across all devices.
Join the [discord](https://discord.gg/pRYNYr4W5A)! Join the [discord](https://discord.gg/pRYNYr4W5A)!
## Non-technical users ## Non-technical users
We are looking into a feature for one-button click click deployment of Actual. This will reduce the friction for people not as comfortable with the command line. We are working on simpler one-button click deployments of Actual. This will reduce the friction for people not as comfortable with the command line. Some non-official options are listed at the bottom.
## Running ## Running
@ -30,11 +29,12 @@ docker build -t actual-server .
docker run -p 5006:5006 actual-server docker run -p 5006:5006 actual-server
``` ```
## Deploying ## Deploying
You should deploy your server so it's always running. We recommend [fly.io](https://fly.io) which makes it incredibly easy and provides a free plan. You should deploy your server so it's always running. We recommend [fly.io](https://fly.io) which makes it incredibly easy and provides a free plan.
[fly.io](https://fly.io) allows running the application directly and provides a free tier. You should be comfortable with using the command line to set it up though.
[Create an account](https://fly.io/app/sign-in). Although you are required to enter payment details, everything we do here will work on the free tier and you won't be charged. [Create an account](https://fly.io/app/sign-in). Although you are required to enter payment details, everything we do here will work on the free tier and you won't be charged.
Next, [install the `flyctl`](https://fly.io/docs/flyctl/installing/) utility. Run `flyctl auth login` to sign into your account. Next, [install the `flyctl`](https://fly.io/docs/flyctl/installing/) utility. Run `flyctl auth login` to sign into your account.
@ -45,7 +45,12 @@ Now, run `flyctl launch` from `actual-server`. You should have a running app now
Whenever you want to update Actual, update the versions of `@actual-app/api` and `@actual-app/web` in `package.json` and run `flyctl deploy`. Whenever you want to update Actual, update the versions of `@actual-app/api` and `@actual-app/web` in `package.json` and run `flyctl deploy`.
**Note:** if you don't want to use fly, we still provide a `Dockerfile` to build the app so it should work anywhere that can compile a docker image. ### Using a custom Docker setup
Actual is also available as a Docker image ready to be run in your own custom environment.
- Docker Hub: `jlongster/actual-server`
- Github Registry: `ghcr.io/actualbudget/actual-server`
### Persisting server data ### Persisting server data
@ -69,6 +74,13 @@ That's it! Actual will automatically check if the `/data` directory exists and u
_You can also configure the data dir with the `ACTUAL_USER_FILES` environment variable._ _You can also configure the data dir with the `ACTUAL_USER_FILES` environment variable._
### One-click hosting solutions
These are non-official methods of one-click solutions for running Actual. If you provide a service like this, feel free to open a PR and add it to this list. These run Actual via a Docker image.
* PikaPods: [Run on PikaPods](https://www.pikapods.com/pods?run=actual)
## Configuring the server URL ## Configuring the server URL
The Actual app is totally separate from the server. In this project, they happen to both be served by the same server, but the app doesn't know where the server lives. The Actual app is totally separate from the server. In this project, they happen to both be served by the same server, but the app doesn't know where the server lives.
@ -76,3 +88,4 @@ The Actual app is totally separate from the server. In this project, they happen
The server could live on a completely different domain. You might setup Actual so that the app and server are running in completely separate places. The server could live on a completely different domain. You might setup Actual so that the app and server are running in completely separate places.
Since Actual doesn't know what server to use, the first thing it does is asks you for the server URL. If you are running this project, simply click "Use this domain" and it will automatically fill it in with the current domain. This works because we are serving the app and server in the same place. Since Actual doesn't know what server to use, the first thing it does is asks you for the server URL. If you are running this project, simply click "Use this domain" and it will automatically fill it in with the current domain. This works because we are serving the app and server in the same place.

View file

@ -8,7 +8,9 @@ let { getAccountDb } = require('./account-db');
let app = express(); let app = express();
app.use(errorMiddleware); app.use(errorMiddleware);
function init() {} function init() {
// eslint-disable-previous-line @typescript-eslint/no-empty-function
}
function hashPassword(password) { function hashPassword(password) {
return bcrypt.hashSync(password, 12); return bcrypt.hashSync(password, 12);

View file

@ -190,7 +190,7 @@ app.post(
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'User-Agent': 'Actual Budget' 'User-Agent': 'Actual Budget'
} }
}).then(res => res.json()); }).then((res) => res.json());
await req.runQuery( await req.runQuery(
'INSERT INTO access_tokens (item_id, user_id, access_token) VALUES ($1, $2, $3)', 'INSERT INTO access_tokens (item_id, user_id, access_token) VALUES ($1, $2, $3)',
@ -233,7 +233,7 @@ app.post(
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'User-Agent': 'Actual Budget' 'User-Agent': 'Actual Budget'
} }
}).then(res => res.json()); }).then((res) => res.json());
if (resData.removed !== true) { if (resData.removed !== true) {
console.log('[Error] Item not removed: ' + access_token.slice(0, 3)); console.log('[Error] Item not removed: ' + access_token.slice(0, 3));
@ -286,7 +286,7 @@ app.post(
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'User-Agent': 'Actual Budget' 'User-Agent': 'Actual Budget'
} }
}).then(res => res.json()); }).then((res) => res.json());
res.send( res.send(
JSON.stringify({ JSON.stringify({
@ -342,7 +342,7 @@ app.post(
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'User-Agent': 'Actual Budget' 'User-Agent': 'Actual Budget'
} }
}).then(res => res.json()); }).then((res) => res.json());
res.send( res.send(
JSON.stringify({ JSON.stringify({

View file

@ -1,16 +1,13 @@
let { Buffer } = require('buffer');
let fs = require('fs/promises'); let fs = require('fs/promises');
let { join } = require('path'); let { Buffer } = require('buffer');
let express = require('express'); let express = require('express');
let uuid = require('uuid'); let uuid = require('uuid');
let AdmZip = require('adm-zip');
let { validateUser } = require('./util/validate-user'); let { validateUser } = require('./util/validate-user');
let errorMiddleware = require('./util/error-middleware'); let errorMiddleware = require('./util/error-middleware');
let config = require('./load-config');
let { getAccountDb } = require('./account-db'); let { getAccountDb } = require('./account-db');
let { getPathForUserFile, getPathForGroupFile } = require('./util/paths');
let simpleSync = require('./sync-simple'); let simpleSync = require('./sync-simple');
let fullSync = require('./sync-full');
let actual = require('@actual-app/api'); let actual = require('@actual-app/api');
let SyncPb = actual.internal.SyncProtoBuf; let SyncPb = actual.internal.SyncProtoBuf;
@ -18,17 +15,8 @@ let SyncPb = actual.internal.SyncProtoBuf;
const app = express(); const app = express();
app.use(errorMiddleware); app.use(errorMiddleware);
async function init() { // eslint-disable-next-line
let fileDir = join(process.env.ACTUAL_USER_FILES || config.userFiles); async function init() {}
console.log('Initializing Actual with user file dir:', fileDir);
await actual.init({
config: {
dataDir: fileDir
}
});
}
// This is a version representing the internal format of sync // This is a version representing the internal format of sync
// messages. When this changes, all sync files need to be reset. We // messages. When this changes, all sync files need to be reset. We
@ -123,31 +111,15 @@ app.post('/sync', async (req, res) => {
return false; return false;
} }
// TODO: We also provide a "simple" sync method which currently isn't let { trie, newMessages } = simpleSync.sync(messages, since, group_id);
// used. This method just stores the messages locally and doesn't
// load the whole app at all. If we want to support end-to-end
// encryption, this method is required because we can't read the
// messages. Using it looks like this:
//
// let simpleSync = require('./sync-simple');
// let {trie, newMessages } = simpleSync.sync(messages, since, file_id);
let { trie, newMessages } = await fullSync.sync(messages, since, file_id);
// encode it back... // encode it back...
let responsePb = new SyncPb.SyncResponse(); let responsePb = new SyncPb.SyncResponse();
responsePb.setMerkle(JSON.stringify(trie)); responsePb.setMerkle(JSON.stringify(trie));
newMessages.forEach((msg) => responsePb.addMessages(msg));
for (let i = 0; i < newMessages.length; i++) {
let msg = newMessages[i];
let envelopePb = new SyncPb.MessageEnvelope();
envelopePb.setTimestamp(msg.timestamp);
envelopePb.setIsencrypted(msg.is_encrypted === 1);
envelopePb.setContent(msg.content);
responsePb.addMessages(envelopePb);
}
res.set('Content-Type', 'application/actual-sync'); res.set('Content-Type', 'application/actual-sync');
res.set('X-ACTUAL-SYNC-METHOD', 'simple');
res.send(Buffer.from(responsePb.serializeBinary())); res.send(Buffer.from(responsePb.serializeBinary()));
}); });
@ -194,7 +166,7 @@ app.post('/user-create-key', (req, res) => {
res.send(JSON.stringify({ status: 'ok' })); res.send(JSON.stringify({ status: 'ok' }));
}); });
app.post('/reset-user-file', (req, res) => { app.post('/reset-user-file', async (req, res) => {
let user = validateUser(req, res); let user = validateUser(req, res);
if (!user) { if (!user) {
return; return;
@ -214,10 +186,11 @@ app.post('/reset-user-file', (req, res) => {
accountDb.mutate('UPDATE files SET group_id = NULL WHERE id = ?', [fileId]); accountDb.mutate('UPDATE files SET group_id = NULL WHERE id = ?', [fileId]);
if (group_id) { if (group_id) {
// TODO: Instead of doing this, just delete the db file named try {
// after the group await fs.unlink(getPathForGroupFile(group_id));
// db.mutate('DELETE FROM messages_binary WHERE group_id = ?', [group_id]); } catch (e) {
// db.mutate('DELETE FROM messages_merkles WHERE group_id = ?', [group_id]); console.log(`Unable to delete sync data for group "${group_id}"`);
}
} }
res.send(JSON.stringify({ status: 'ok' })); res.send(JSON.stringify({ status: 'ok' }));
@ -274,21 +247,11 @@ app.post('/upload-user-file', async (req, res) => {
} }
} }
// TODO: If we want to support end-to-end encryption, we'd write the
// raw file down because it's an encrypted blob. This isn't
// supported yet in the self-hosted version because it's unclear if
// it's still needed, given that you own your server
//
// await fs.writeFile(join(config.userFiles, `${fileId}.blob`), req.body);
let zip = new AdmZip(req.body);
try { try {
zip.extractAllTo(join(config.userFiles, fileId), true); await fs.writeFile(getPathForUserFile(fileId), req.body);
} catch (err) { } catch (err) {
console.log('Error writing file', err); console.log('Error writing file', err);
res.send(JSON.stringify({ status: 'error' })); res.send(JSON.stringify({ status: 'error' }));
return;
} }
let rows = accountDb.all('SELECT id FROM files WHERE id = ?', [fileId]); let rows = accountDb.all('SELECT id FROM files WHERE id = ?', [fileId]);
@ -315,6 +278,7 @@ app.post('/upload-user-file', async (req, res) => {
'UPDATE files SET sync_version = ?, encrypt_meta = ?, name = ? WHERE id = ?', 'UPDATE files SET sync_version = ?, encrypt_meta = ?, name = ? WHERE id = ?',
[syncFormatVersion, encryptMeta, name, fileId] [syncFormatVersion, encryptMeta, name, fileId]
); );
res.send(JSON.stringify({ status: 'ok', groupId })); res.send(JSON.stringify({ status: 'ok', groupId }));
} }
}); });
@ -337,14 +301,14 @@ app.get('/download-user-file', async (req, res) => {
return; return;
} }
let zip = new AdmZip(); let buffer;
try { try {
zip.addLocalFolder(join(config.userFiles, fileId), '/'); buffer = await fs.readFile(getPathForUserFile(fileId));
} catch (e) { } catch (e) {
res.status(500).send('Error reading files'); console.log(`Error: file does not exist: ${getPathForUserFile(fileId)}`);
res.status(500).send('File does not exist on server');
return; return;
} }
let buffer = zip.toBuffer();
res.setHeader('Content-Disposition', `attachment;filename=${fileId}`); res.setHeader('Content-Disposition', `attachment;filename=${fileId}`);
res.send(buffer); res.send(buffer);
@ -385,7 +349,7 @@ app.get('/list-user-files', (req, res) => {
res.send( res.send(
JSON.stringify({ JSON.stringify({
status: 'ok', status: 'ok',
data: rows.map(row => ({ data: rows.map((row) => ({
deleted: row.deleted, deleted: row.deleted,
fileId: row.id, fileId: row.id,
groupId: row.group_id, groupId: row.group_id,

4
app.js
View file

@ -10,7 +10,7 @@ const syncApp = require('./app-sync');
const app = express(); const app = express();
process.on('unhandledRejection', reason => { process.on('unhandledRejection', (reason) => {
console.log('Rejection:', reason); console.log('Rejection:', reason);
}); });
@ -59,7 +59,7 @@ async function run() {
app.listen(config.port, config.hostname); app.listen(config.port, config.hostname);
} }
run().catch(err => { run().catch((err) => {
console.log('Error starting app:', err); console.log('Error starting app:', err);
process.exit(1); process.exit(1);
}); });

13
docker-compose.yml Normal file
View file

@ -0,0 +1,13 @@
version: "3"
services:
actual_server:
container_name: actual_server
build:
context: ./
dockerfile: Dockerfile
ports:
- "5006:5006"
volumes:
- ./server-files:/app/server-files
- ./user-files:/app/user-files
restart: unless-stopped

View file

@ -5,11 +5,13 @@ kill_timeout = 5
processes = [] processes = []
[env] [env]
PORT = "8080" PORT = "5006"
TINI_SUBREAPER = 1
[experimental] [experimental]
allowed_public_ports = [] allowed_public_ports = []
auto_rollback = true auto_rollback = true
cmd = ["node", "--max-old-space-size=180", "app.js"]
[[services]] [[services]]
http_checks = [] http_checks = []

View file

@ -1,5 +1,6 @@
let config; let config;
try { try {
// @ts-expect-error TS2307: we expect this file may not exist
config = require('./config'); config = require('./config');
} catch (e) { } catch (e) {
let fs = require('fs'); let fs = require('fs');
@ -15,4 +16,7 @@ try {
}; };
} }
// The env variable always takes precedence
config.userFiles = process.env.ACTUAL_USER_FILES || config.userFiles;
module.exports = config; module.exports = config;

View file

@ -1,4 +1,4 @@
export default async function runMigration(db, uuid) { export default async function runMigration(db) {
function getValue(node) { function getValue(node) {
return node.expr != null ? node.expr : node.cachedValue; return node.expr != null ? node.expr : node.cachedValue;
} }
@ -37,7 +37,7 @@ CREATE TABLE kvcache_key (id INTEGER PRIMARY KEY, key REAL);
true true
); );
db.transaction(() => { db.transaction(() => {
budget.map(monthBudget => { budget.map((monthBudget) => {
let match = monthBudget.name.match( let match = monthBudget.name.match(
/^(budget-report|budget)(\d+)!budget-(.+)$/ /^(budget-report|budget)(\d+)!budget-(.+)$/
); );
@ -84,7 +84,7 @@ CREATE TABLE kvcache_key (id INTEGER PRIMARY KEY, key REAL);
true true
); );
db.transaction(() => { db.transaction(() => {
buffers.map(buffer => { buffers.map((buffer) => {
let match = buffer.name.match(/^budget(\d+)!buffered$/); let match = buffer.name.match(/^budget(\d+)!buffered$/);
if (match) { if (match) {
let month = match[1].slice(0, 4) + '-' + match[1].slice(4); let month = match[1].slice(0, 4) + '-' + match[1].slice(4);
@ -108,7 +108,7 @@ CREATE TABLE kvcache_key (id INTEGER PRIMARY KEY, key REAL);
true true
); );
let parseNote = str => { let parseNote = (str) => {
try { try {
let value = JSON.parse(str); let value = JSON.parse(str);
return value && value !== '' ? value : null; return value && value !== '' ? value : null;
@ -118,7 +118,7 @@ CREATE TABLE kvcache_key (id INTEGER PRIMARY KEY, key REAL);
}; };
db.transaction(() => { db.transaction(() => {
notes.forEach(note => { notes.forEach((note) => {
let parsed = parseNote(getValue(note)); let parsed = parseNote(getValue(note));
if (parsed) { if (parsed) {
let [, id] = note.name.split('!'); let [, id] = note.name.split('!');

View file

@ -1,41 +1,37 @@
{ {
"name": "actual-sync", "name": "actual-sync",
"version": "1.0.0", "version": "22.12.09",
"license": "MIT", "license": "MIT",
"description": "actual syncing server", "description": "actual syncing server",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"start": "node app", "start": "node app",
"lint": "eslint --ignore-pattern '**/node_modules/*' --ignore-pattern '**/log/*' --ignore-pattern 'supervise' --ignore-pattern '**/shared/*' ." "lint": "eslint .",
"build": "tsc",
"types": "tsc --noEmit --incremental",
"verify": "yarn -s lint && yarn types"
}, },
"dependencies": { "dependencies": {
"@actual-app/api": "^4.0.1", "@actual-app/api": "4.1.5",
"@actual-app/web": "^4.0.2", "@actual-app/web": "22.12.3",
"adm-zip": "^0.5.9",
"bcrypt": "^5.0.1", "bcrypt": "^5.0.1",
"better-sqlite3": "^7.5.0", "better-sqlite3": "^7.5.0",
"body-parser": "^1.18.3", "body-parser": "^1.18.3",
"cors": "^2.8.5", "cors": "^2.8.5",
"express": "^4.16.3", "express": "4.17",
"express-actuator": "^1.8.1", "express-actuator": "^1.8.1",
"express-response-size": "^0.0.3", "express-response-size": "^0.0.3",
"node-fetch": "^2.2.0", "node-fetch": "^2.2.0",
"uuid": "^3.3.2" "uuid": "^3.3.2"
}, },
"eslintConfig": {
"extends": "react-app"
},
"devDependencies": { "devDependencies": {
"babel-eslint": "^10.0.1", "@types/better-sqlite3": "^7.5.0",
"eslint": "^5.12.1", "@types/node": "^17.0.31",
"eslint-config-react-app": "^3.0.6", "@typescript-eslint/eslint-plugin": "^5.23.0",
"eslint-plugin-flowtype": "^3.2.1", "@typescript-eslint/parser": "^5.23.0",
"eslint-plugin-import": "^2.14.0", "eslint": "^8.15.0",
"eslint-plugin-jsx-a11y": "^6.1.2", "eslint-plugin-prettier": "^4.0.0",
"eslint-plugin-react": "^7.12.4" "prettier": "^2.6.2",
}, "typescript": "^4.6.4"
"prettier": {
"singleQuote": true,
"trailingComma": "none"
} }
} }

View file

@ -1,10 +1,9 @@
CREATE TABLE messages_binary CREATE TABLE messages_binary
(timestamp TEXT, (timestamp TEXT PRIMARY KEY,
is_encrypted BOOLEAN, is_encrypted BOOLEAN,
content bytea, content bytea);
PRIMARY KEY(timestamp, group_id));
CREATE TABLE messages_merkles CREATE TABLE messages_merkles
(id TEXT PRIMAREY KEY, (id INTEGER PRIMARY KEY,
merkle TEXT); merkle TEXT);

View file

@ -15,7 +15,7 @@ const sync = sequential(async function syncAPI(messages, since, fileId) {
await actual.internal.send('load-budget', { id: fileId }); await actual.internal.send('load-budget', { id: fileId });
} }
messages = messages.map(envPb => { messages = messages.map((envPb) => {
let timestamp = envPb.getTimestamp(); let timestamp = envPb.getTimestamp();
let msg = SyncPb.Message.deserializeBinary(envPb.getContent()); let msg = SyncPb.Message.deserializeBinary(envPb.getContent());
return { return {
@ -27,11 +27,26 @@ const sync = sequential(async function syncAPI(messages, since, fileId) {
}; };
}); });
let newMessages = actual.internal.syncAndReceiveMessages(messages, since); const newMessages = await actual.internal.syncAndReceiveMessages(
messages,
since
);
return { return {
trie: actual.internal.timestamp.getClock().merkle, trie: actual.internal.timestamp.getClock().merkle,
newMessages: newMessages newMessages: newMessages.map((msg) => {
const envelopePb = new SyncPb.MessageEnvelope();
const messagePb = new SyncPb.Message();
messagePb.setDataset(msg.dataset);
messagePb.setRow(msg.row);
messagePb.setColumn(msg.column);
messagePb.setValue(msg.value);
envelopePb.setTimestamp(msg.timestamp);
envelopePb.setContent(messagePb.serializeBinary());
return envelopePb;
})
}; };
}); });

View file

@ -1,13 +1,15 @@
let { existsSync, readFileSync } = require('fs'); let { existsSync, readFileSync } = require('fs');
let { join } = require('path'); let { join } = require('path');
let { openDatabase } = require('./db'); let { openDatabase } = require('./db');
let { getPathForGroupFile } = require('./util/paths');
let actual = require('@actual-app/api'); let actual = require('@actual-app/api');
let merkle = actual.internal.merkle; let merkle = actual.internal.merkle;
let SyncPb = actual.internal.SyncProtoBuf;
let Timestamp = actual.internal.timestamp.Timestamp; let Timestamp = actual.internal.timestamp.Timestamp;
function getGroupDb(groupId) { function getGroupDb(groupId) {
let path = join(__dirname, `user-files/${groupId}.sqlite`); let path = getPathForGroupFile(groupId);
let needsInit = !existsSync(path); let needsInit = !existsSync(path);
let db = openDatabase(path); let db = openDatabase(path);
@ -56,8 +58,8 @@ function addMessages(db, messages) {
return returnValue; return returnValue;
} }
function getMerkle(db, group_id) { function getMerkle(db) {
let rows = db.all('SELECT * FROM messages_merkles', [group_id]); let rows = db.all('SELECT * FROM messages_merkles');
if (rows.length > 0) { if (rows.length > 0) {
return JSON.parse(rows[0].merkle); return JSON.parse(rows[0].merkle);
@ -68,19 +70,29 @@ function getMerkle(db, group_id) {
} }
} }
function sync(messages, since, fileId) { function sync(messages, since, groupId) {
let db = getGroupDb(fileId); let db = getGroupDb(groupId);
let newMessages = db.all( let newMessages = db.all(
`SELECT * FROM messages_binary `SELECT * FROM messages_binary
WHERE timestamp > ? WHERE timestamp > ?
ORDER BY timestamp`, ORDER BY timestamp`,
[since], [since]
true
); );
let trie = addMessages(db, messages); let trie = addMessages(db, messages);
return { trie, newMessages }; db.close();
return {
trie,
newMessages: newMessages.map((msg) => {
const envelopePb = new SyncPb.MessageEnvelope();
envelopePb.setTimestamp(msg.timestamp);
envelopePb.setIsencrypted(msg.is_encrypted);
envelopePb.setContent(msg.content);
return envelopePb;
})
};
} }
module.exports = { sync }; module.exports = { sync };

View file

@ -4,6 +4,7 @@
// DOM for URL global in Node 16+ // DOM for URL global in Node 16+
"lib": ["ES2021", "DOM"], "lib": ["ES2021", "DOM"],
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"experimentalDecorators": true, "experimentalDecorators": true,
"resolveJsonModule": true, "resolveJsonModule": true,
"downlevelIteration": true, "downlevelIteration": true,
@ -11,10 +12,9 @@
// Check JS files too // Check JS files too
"allowJs": true, "allowJs": true,
"checkJs": true, "checkJs": true,
// Used for temp builds "moduleResolution": "node",
"outDir": "build", "module": "commonjs",
"moduleResolution": "Node", "outDir": "build"
"module": "ESNext"
}, },
"exclude": ["node_modules", "build"] "exclude": ["node_modules", "build", "./app-plaid.js"]
} }

View file

@ -17,11 +17,11 @@ function sequential(fn) {
sequenceState.running = fn(...args); sequenceState.running = fn(...args);
sequenceState.running.then( sequenceState.running.then(
val => { (val) => {
pump(); pump();
resolve(val); resolve(val);
}, },
err => { (err) => {
pump(); pump();
reject(err); reject(err);
} }

View file

@ -1,4 +1,4 @@
async function middleware(err, req, res, next) { async function middleware(err, req, res, _next) {
console.log('ERROR', err); console.log('ERROR', err);
res.status(500).send({ status: 'error', reason: 'internal-error' }); res.status(500).send({ status: 'error', reason: 'internal-error' });
} }

View file

@ -1,11 +1,11 @@
function handleError(func) { function handleError(func) {
return (req, res) => { return (req, res) => {
func(req, res).catch(err => { func(req, res).catch((err) => {
console.log('Error', req.originalUrl, err); console.log('Error', req.originalUrl, err);
res.status(500); res.status(500);
res.send({ status: 'error', reason: 'internal-error' }); res.send({ status: 'error', reason: 'internal-error' });
}); });
}; };
}; }
module.exports = { handleError } module.exports = { handleError };

12
util/paths.js Normal file
View file

@ -0,0 +1,12 @@
let { join } = require('path');
let config = require('../load-config');
function getPathForUserFile(fileId) {
return join(config.userFiles, `file-${fileId}.blob`);
}
function getPathForGroupFile(groupId) {
return join(config.userFiles, `group-${groupId}.sqlite`);
}
module.exports = { getPathForUserFile, getPathForGroupFile };

1844
yarn.lock

File diff suppressed because it is too large Load diff