From d7e71158b931842afa51698e6c5a1662c79c4510 Mon Sep 17 00:00:00 2001 From: James Long Date: Wed, 25 May 2022 22:42:10 -0400 Subject: [PATCH] Add ability to import Actual files; enable export on desktop --- .../desktop-client/src/components/Settings.js | 12 +- .../src/components/manager/Modals.js | 5 + .../src/platform/client/fetch/index.web.js | 33 +++--- .../server/connection/index.electron.js | 73 ++++-------- .../src/platform/server/fs/index.web.js | 12 +- .../loot-core/src/server/cloud-storage.js | 110 +++++++++--------- packages/loot-core/src/server/main.js | 35 ++++++ .../src/components/manager/Import.js | 17 +-- .../src/components/manager/ImportActual.js | 104 +++++++++++++++++ 9 files changed, 264 insertions(+), 137 deletions(-) create mode 100644 packages/loot-design/src/components/manager/ImportActual.js diff --git a/packages/desktop-client/src/components/Settings.js b/packages/desktop-client/src/components/Settings.js index 9a90bbc..c3630b4 100644 --- a/packages/desktop-client/src/components/Settings.js +++ b/packages/desktop-client/src/components/Settings.js @@ -334,7 +334,7 @@ function FileSettings({ async function onExport() { let data = await send('export-budget'); - window.Actual.saveFile(data, 'budget.zip', 'Export budget'); + window.Actual.saveFile(data, `${prefs.id}.zip`, 'Export budget'); } let dateFormat = prefs.dateFormat || 'MM/dd/yyyy'; @@ -431,12 +431,10 @@ function FileSettings({ - {Platform.isBrowser && ( - - - <Button onClick={onExport}>Export data</Button> - </View> - )} + <View style={{ marginTop: 30, alignItems: 'flex-start' }}> + <Title name="Export" /> + <Button onClick={onExport}>Export data</Button> + </View> <Advanced prefs={prefs} diff --git a/packages/desktop-client/src/components/manager/Modals.js b/packages/desktop-client/src/components/manager/Modals.js index 0fbc7f2..2d6b120 100644 --- a/packages/desktop-client/src/components/manager/Modals.js +++ b/packages/desktop-client/src/components/manager/Modals.js @@ -10,6 +10,7 @@ import LoadBackup from 'loot-design/src/components/modals/LoadBackup'; import Import from 'loot-design/src/components/manager/Import'; import ImportYNAB4 from 'loot-design/src/components/manager/ImportYNAB4'; import ImportYNAB5 from 'loot-design/src/components/manager/ImportYNAB5'; +import ImportActual from 'loot-design/src/components/manager/ImportActual'; import DeleteFile from 'loot-design/src/components/manager/DeleteFile'; import CreateEncryptionKey from '../modals/CreateEncryptionKey'; import FixEncryptionKey from '../modals/FixEncryptionKey'; @@ -72,6 +73,10 @@ function Modals({ return ( <ImportYNAB5 key={name} modalProps={modalProps} actions={actions} /> ); + case 'import-actual': + return ( + <ImportActual key={name} modalProps={modalProps} actions={actions} /> + ); case 'load-backup': { return ( <Component diff --git a/packages/loot-core/src/platform/client/fetch/index.web.js b/packages/loot-core/src/platform/client/fetch/index.web.js index a1a25ca..d50b8f0 100644 --- a/packages/loot-core/src/platform/client/fetch/index.web.js +++ b/packages/loot-core/src/platform/client/fetch/index.web.js @@ -8,7 +8,7 @@ let socketClient = null; function connectSocket(name, onOpen) { global.Actual.ipcConnect(name, function(client) { client.on('message', data => { - const msg = JSON.parse(data); + const msg = data; if (msg.type === 'error') { // An error happened while handling a message so cleanup the @@ -19,7 +19,15 @@ function connectSocket(name, onOpen) { const { id } = msg; replyHandlers.delete(id); } else if (msg.type === 'reply') { - const { id, result, mutated, undoTag } = msg; + let { id, result, mutated, undoTag } = msg; + + // Check if the result is a serialized buffer, and if so + // convert it to a Uint8Array. This is only needed when working + // with node; the web version connection layer automatically + // supports buffers + if (result && result.type === 'Buffer' && Array.isArray(result.data)) { + result = new Uint8Array(result.data); + } const handler = replyHandlers.get(id); if (handler) { @@ -53,9 +61,7 @@ function connectSocket(name, onOpen) { // Send any messages that were queued while closed if (messageQueue.length > 0) { - messageQueue.forEach(msg => - client.emit('message', JSON.stringify(msg)) - ); + messageQueue.forEach(msg => client.emit('message', msg)); messageQueue = []; } @@ -78,16 +84,13 @@ module.exports.send = function send(name, args, { catchErrors = false } = {}) { replyHandlers.set(id, { resolve, reject }); if (socketClient) { - socketClient.emit( - 'message', - JSON.stringify({ - id, - name, - args, - undoTag: undo.snapshot(), - catchErrors: !!catchErrors - }) - ); + socketClient.emit('message', { + id, + name, + args, + undoTag: undo.snapshot(), + catchErrors: !!catchErrors + }); } else { messageQueue.push({ id, diff --git a/packages/loot-core/src/platform/server/connection/index.electron.js b/packages/loot-core/src/platform/server/connection/index.electron.js index f463aec..3bd2b15 100644 --- a/packages/loot-core/src/platform/server/connection/index.electron.js +++ b/packages/loot-core/src/platform/server/connection/index.electron.js @@ -16,7 +16,7 @@ function init(socketName, handlers) { ipc.serve(() => { ipc.server.on('message', (data, socket) => { - let msg = JSON.parse(data); + let msg = data; let { id, name, args, undoTag, catchErrors } = msg; if (handlers[name]) { @@ -26,20 +26,16 @@ function init(socketName, handlers) { result = { data: result, error: null }; } - ipc.server.emit( - socket, - 'message', - JSON.stringify({ - type: 'reply', - id, - result, - mutated: - isMutating(handlers[name]) && - name !== 'undo' && - name !== 'redo', - undoTag - }) - ); + ipc.server.emit(socket, 'message', { + type: 'reply', + id, + result, + mutated: + isMutating(handlers[name]) && + name !== 'undo' && + name !== 'redo', + undoTag + }); }, nativeError => { let error = coerceError(nativeError); @@ -47,27 +43,15 @@ function init(socketName, handlers) { if (name.startsWith('api/')) { // The API is newer and does automatically forward // errors - ipc.server.emit( - socket, - 'message', - JSON.stringify({ type: 'reply', id, error }) - ); + ipc.server.emit(socket, 'message', { type: 'reply', id, error }); } else if (catchErrors) { - ipc.server.emit( - socket, - 'message', - JSON.stringify({ - type: 'reply', - id, - result: { error, data: null } - }) - ); + ipc.server.emit(socket, 'message', { + type: 'reply', + id, + result: { error, data: null } + }); } else { - ipc.server.emit( - socket, - 'message', - JSON.stringify({ type: 'error', id }) - ); + ipc.server.emit(socket, 'message', { type: 'error', id }); } if (error.type === 'InternalError' && name !== 'api/load-budget') { @@ -83,16 +67,12 @@ function init(socketName, handlers) { } else { console.warn('Unknown method: ' + name); captureException(new Error('Unknown server method: ' + name)); - ipc.server.emit( - socket, - 'message', - JSON.stringify({ - type: 'reply', - id, - result: null, - error: { type: 'APIError', message: 'Unknown method: ' + name } - }) - ); + ipc.server.emit(socket, 'message', { + type: 'reply', + id, + result: null, + error: { type: 'APIError', message: 'Unknown method: ' + name } + }); } }); }); @@ -106,10 +86,7 @@ function getNumClients() { function send(name, args) { if (ipc.server) { - ipc.server.broadcast( - 'message', - JSON.stringify({ type: 'push', name, args }) - ); + ipc.server.broadcast('message', { type: 'push', name, args }); } } diff --git a/packages/loot-core/src/platform/server/fs/index.web.js b/packages/loot-core/src/platform/server/fs/index.web.js index 0628952..a819f0e 100644 --- a/packages/loot-core/src/platform/server/fs/index.web.js +++ b/packages/loot-core/src/platform/server/fs/index.web.js @@ -16,12 +16,16 @@ function pathToId(filepath) { } function _exists(filepath) { + try { + FS.readlink(filepath); + return true; + } catch (e) {} + try { FS.stat(filepath); - } catch (e) { - return false; - } - return true; + return true; + } catch (e) {} + return false; } function _mkdirRecursively(dir) { diff --git a/packages/loot-core/src/server/cloud-storage.js b/packages/loot-core/src/server/cloud-storage.js index c86d36f..498571d 100644 --- a/packages/loot-core/src/server/cloud-storage.js +++ b/packages/loot-core/src/server/cloud-storage.js @@ -154,7 +154,60 @@ export async function exportBuffer() { zipped.addFile('metadata.json', metaContent); }); - return zipped.toBuffer(); + return Buffer.from(zipped.toBuffer()); +} + +export async function importBuffer(fileData, buffer) { + let zipped = new AdmZip(buffer); + let entries = zipped.getEntries(); + let dbEntry = entries.find(e => e.entryName.includes('db.sqlite')); + let metaEntry = entries.find(e => e.entryName.includes('metadata.json')); + + if (!dbEntry || !metaEntry) { + throw FileDownloadError('invalid-zip-file'); + } + + let dbContent = zipped.readFile(dbEntry); + let metaContent = zipped.readFile(metaEntry); + + let meta; + try { + meta = JSON.parse(metaContent.toString('utf8')); + } catch (err) { + throw FileDownloadError('invalid-meta-file'); + } + + // Update the metadata. The stored file on the server might be + // out-of-date with a few keys + meta = { + ...meta, + cloudFileId: fileData.fileId, + groupId: fileData.groupId, + lastUploaded: monthUtils.currentDay(), + encryptKeyId: fileData.encryptMeta ? fileData.encryptMeta.keyId : null + }; + + let budgetDir = fs.getBudgetDir(meta.id); + + if (await fs.exists(budgetDir)) { + // Don't remove the directory so that backups are retained + let dbFile = fs.join(budgetDir, 'db.sqlite'); + let metaFile = fs.join(budgetDir, 'metadata.json'); + + if (await fs.exists(dbFile)) { + await fs.removeFile(dbFile); + } + if (await fs.exists(metaFile)) { + await fs.removeFile(metaFile); + } + } else { + await fs.mkdir(budgetDir); + } + + await fs.writeFile(fs.join(budgetDir, 'db.sqlite'), dbContent); + await fs.writeFile(fs.join(budgetDir, 'metadata.json'), JSON.stringify(meta)); + + return { id: meta.id }; } export async function upload() { @@ -355,58 +408,5 @@ export async function download(fileId, replace) { } } - let zipped = new AdmZip(buffer); - let entries = zipped.getEntries(); - let dbEntry = entries.find(e => e.entryName.includes('db.sqlite')); - let metaEntry = entries.find(e => e.entryName.includes('metadata.json')); - - if (!dbEntry || !metaEntry) { - throw FileDownloadError('invalid-zip-file'); - } - - let dbContent = zipped.readFile(dbEntry); - let metaContent = zipped.readFile(metaEntry); - - let meta; - try { - meta = JSON.parse(metaContent.toString('utf8')); - } catch (err) { - throw FileDownloadError('invalid-meta-file'); - } - - // Update the metadata. The stored file on the server might be - // out-of-date with a few keys - meta = { - ...meta, - cloudFileId: fileData.fileId, - groupId: fileData.groupId, - lastUploaded: monthUtils.currentDay(), - encryptKeyId: fileData.encryptMeta ? fileData.encryptMeta.keyId : null - }; - - let budgetDir = fs.getBudgetDir(meta.id); - - if (await fs.exists(budgetDir)) { - if (replace) { - // Don't remove the directory so that backups are retained - let dbFile = fs.join(budgetDir, 'db.sqlite'); - let metaFile = fs.join(budgetDir, 'metadata.json'); - - if (await fs.exists(dbFile)) { - await fs.removeFile(dbFile); - } - if (await fs.exists(metaFile)) { - await fs.removeFile(metaFile); - } - } else { - throw FileDownloadError('file-exists', { id: meta.id }); - } - } else { - await fs.mkdir(budgetDir); - } - - await fs.writeFile(fs.join(budgetDir, 'db.sqlite'), dbContent); - await fs.writeFile(fs.join(budgetDir, 'metadata.json'), JSON.stringify(meta)); - - return { id: meta.id }; + return importBuffer(fileData, buffer, replace); } diff --git a/packages/loot-core/src/server/main.js b/packages/loot-core/src/server/main.js index 7231386..ede9e09 100644 --- a/packages/loot-core/src/server/main.js +++ b/packages/loot-core/src/server/main.js @@ -1834,6 +1834,41 @@ handlers['import-budget'] = async function({ filepath, type }) { } catch (e) { return { error: 'not-ynab5' }; } + break; + case 'actual': + // We should pull out import/export into its own app so this + // can be abstracted out better. Importing Actual files is a + // special case because we can directly write down the files, + // but because it doesn't go through the API layer we need to + // duplicate some of the workflow + await handlers['close-budget'](); + + let { id } = await cloudStorage.importBuffer( + { cloudFileId: null, groupId: null }, + buffer + ); + + // We never want to load cached data from imported files, so + // delete the cache + let sqliteDb = await sqlite.openDatabase( + fs.join(fs.getBudgetDir(id), 'db.sqlite') + ); + sqlite.execQuery( + sqliteDb, + ` + DELETE FROM kvcache; + DELETE FROM kvcache_key; + ` + ); + sqlite.closeDatabase(sqliteDb); + + // Load the budget, force everything to be computed, and try + // to upload it as a cloud file + await handlers['load-budget']({ id }); + await handlers['get-budget-bounds'](); + await sheet.waitOnSpreadsheet(); + await cloudStorage.upload().catch(err => {}); + break; default: } diff --git a/packages/loot-design/src/components/manager/Import.js b/packages/loot-design/src/components/manager/Import.js index b6d2c61..73b0a60 100644 --- a/packages/loot-design/src/components/manager/Import.js +++ b/packages/loot-design/src/components/manager/Import.js @@ -37,6 +37,9 @@ function Import({ modalProps, actions, availableImports }) { case 'ynab5': actions.pushModal('import-ynab5'); break; + case 'actual': + actions.pushModal('import-actual'); + break; default: } } @@ -83,14 +86,12 @@ function Import({ modalProps, actions, availableImports }) { <div>The newer web app</div> </View> </Button> - {false && ( - <Button style={itemStyle}> - <span style={{ fontWeight: 700 }}>Actual</span> - <View style={{ color: colors.n5 }}> - <div>Import a backup or external file</div> - </View> - </Button> - )} + <Button style={itemStyle} onClick={() => onSelectType('actual')}> + <span style={{ fontWeight: 700 }}>Actual</span> + <View style={{ color: colors.n5 }}> + <div>Import a file exported from Actual</div> + </View> + </Button> </View> </View> diff --git a/packages/loot-design/src/components/manager/ImportActual.js b/packages/loot-design/src/components/manager/ImportActual.js new file mode 100644 index 0000000..a44926e --- /dev/null +++ b/packages/loot-design/src/components/manager/ImportActual.js @@ -0,0 +1,104 @@ +import React, { useState } from 'react'; +import { useDispatch } from 'react-redux'; +import { importBudget } from 'loot-core/src/client/actions/budgets'; +import { + View, + Block, + Modal, + ButtonWithLoading, + Button, + Link, + P, + ExternalLink +} from '../common'; +import { styles, colors } from '../../style'; + +function getErrorMessage(error) { + switch (error) { + case 'parse-error': + return 'Unable to parse file. Please select a JSON file exported from nYNAB.'; + case 'not-ynab5': + return 'This file is not valid. Please select a JSON file exported from nYNAB.'; + default: + return 'An unknown error occurred while importing. Sorry! We have been notified of this issue.'; + } +} + +function Import({ modalProps, availableImports }) { + const dispatch = useDispatch(); + const [error, setError] = useState(false); + const [importing, setImporting] = useState(false); + + async function onImport() { + const res = await window.Actual.openFileDialog({ + properties: ['openFile'], + filters: [{ name: 'actual', extensions: ['zip'] }] + }); + if (res) { + setImporting(true); + setError(false); + try { + await dispatch(importBudget(res[0], 'actual')); + } catch (err) { + setError(err.message); + } finally { + setImporting(false); + } + } + } + + return ( + <Modal + {...modalProps} + showHeader={false} + showOverlay={false} + noAnimation={true} + style={{ width: 400 }} + > + {() => ( + <View style={[styles.smallText, { lineHeight: 1.5, marginTop: 20 }]}> + {error && ( + <Block style={{ color: colors.r4, marginBottom: 15 }}> + {getErrorMessage(error)} + </Block> + )} + + <View style={{ '& > div': { lineHeight: '1.7em' } }}> + <P> + You can import data from another Actual account or instance. First + export your data from a different account, and it will give you a + compressed file. This file is simple zip file that contains the + "db.sqlite" and "metadata.json" files. + </P> + + <P>Select one of these compressed files and import it here.</P> + + <View style={{ alignSelf: 'center' }}> + <ButtonWithLoading loading={importing} primary onClick={onImport}> + Select file... + </ButtonWithLoading> + </View> + </View> + + <View + style={{ + flexDirection: 'row', + marginTop: 20, + alignItems: 'center' + }} + > + <View style={{ flex: 1 }} /> + <Button + style={{ marginRight: 10 }} + onClick={() => modalProps.onBack()} + > + Back + </Button> + </View> + </View> + )} + </Modal> + ); +} + +export default Import;