mirrors/Plume
1
0
Fork 0
mirror of https://git.joinplu.me/Plume/Plume.git synced 2024-11-22 11:31:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

import migrations and don't require diesel_cli for admins (#555)

Browse source 
* import migrations via macro

* panic on database not to the latest migration

* add subcommand to plm

* create migration that run tantivy index creation

* remove diesel_cli from places it was

* use our migration system for tests

* create table __diesel_schema_migrations if needed
This commit is contained in:
fdb-hiroshima 2019-04-29 16:30:20 +02:00 committed by GitHub
parent ec57f1e687
commit 49bb8cb0bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 475 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.circleci/images/plume-buildenv/Dockerfile
View file

@ -13,7 +13,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0
#compile some deps #compile some deps
RUN cargo install cargo-web &&\ RUN cargo install cargo-web &&\
cargo install diesel_cli --no-default-features --features postgres,sqlite --version '=1.3.0' &&\
rm -fr ~/.cargo/registry rm -fr ~/.cargo/registry
#install coverage tools #install coverage tools

47
Cargo.lock generated
View file

@ -607,7 +607,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"devise_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "devise_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -617,7 +617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -641,7 +641,7 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -801,7 +801,7 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -1055,7 +1055,7 @@ dependencies = [
"mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"markup5ever 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", "markup5ever 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -1716,7 +1716,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1860,6 +1860,15 @@ dependencies = [
"stdweb 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", "stdweb 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "plume-macro"
version = "0.1.0"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "plume-models" name = "plume-models"
version = "0.3.0" version = "0.3.0"
@ -1874,10 +1883,12 @@ dependencies = [
"guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"migrations_internals 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl 0.10.19 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.19 (registry+https://github.com/rust-lang/crates.io-index)",
"plume-api 0.3.0", "plume-api 0.3.0",
"plume-common 0.3.0", "plume-common 0.3.0",
"plume-macro 0.1.0",
"reqwest 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)", "rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)",
@ -1988,7 +1999,7 @@ dependencies = [
[[package]] [[package]]
name = "quote" name = "quote"
version = "0.6.11" version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2202,7 +2213,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2273,7 +2284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"devise 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "devise 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_http 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_http 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2495,7 +2506,7 @@ version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2590,7 +2601,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2603,7 +2614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"base-x 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "base-x 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2643,7 +2654,7 @@ dependencies = [
"phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)",
"phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2693,7 +2704,7 @@ version = "0.15.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -2711,7 +2722,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -3111,7 +3122,7 @@ dependencies = [
"if_chain 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "if_chain 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"validator 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "validator 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3461,7 +3472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408" "checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408"
"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8"
"checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db"
"checksum r2d2 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d746fc8a0dab19ccea7ff73ad535854e90ddb3b4b8cdce953dd5cd0b2e7bd22" "checksum r2d2 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d746fc8a0dab19ccea7ff73ad535854e90ddb3b4b8cdce953dd5cd0b2e7bd22"
"checksum rand 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)" = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" "checksum rand 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)" = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c"
"checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"

2
Cargo.toml
View file

@ -77,4 +77,4 @@ debug-mailer = []
test = [] test = []
[workspace] [workspace]
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front"] members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]

2
Dockerfile
View file

@ -18,7 +18,6 @@ RUN chmod a+x ./wasm-deps.sh && sleep 1 && ./wasm-deps.sh
WORKDIR /app WORKDIR /app
COPY Cargo.toml Cargo.lock rust-toolchain ./ COPY Cargo.toml Cargo.lock rust-toolchain ./
RUN cargo install diesel_cli --no-default-features --features postgres --version '=1.3.0'
RUN cargo install cargo-web RUN cargo install cargo-web
COPY . . COPY . .
@ -40,7 +39,6 @@ WORKDIR /app
COPY --from=builder /app /app COPY --from=builder /app /app
COPY --from=builder /usr/local/cargo/bin/plm /bin/ COPY --from=builder /usr/local/cargo/bin/plm /bin/
COPY --from=builder /usr/local/cargo/bin/plume /bin/ COPY --from=builder /usr/local/cargo/bin/plume /bin/
COPY --from=builder /usr/local/cargo/bin/diesel /bin/
CMD ["plume"] CMD ["plume"]

6
migrations/postgres/2019-04-28-201506_create_tantivy_index/down.sql Normal file
View file

@ -0,0 +1,6 @@
-- This file should undo anything in `up.sql`
--#!|_conn, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! std::fs::remove_dir_all(pb).map_err(Error::from)
--#!}

10
migrations/postgres/2019-04-28-201506_create_tantivy_index/up.sql Normal file
View file

@ -0,0 +1,10 @@
-- Your SQL goes here
--#!|conn: &Connection, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! let searcher = super::search::Searcher::create(&pb)?;
--#! searcher.fill(conn)?;
--#! searcher.commit();
--#! Ok(())
--#!}

6
migrations/sqlite/2019-04-28-201506_create_tantivy_index/down.sql Normal file
View file

@ -0,0 +1,6 @@
-- This file should undo anything in `up.sql`
--#!|_conn, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! std::fs::remove_dir_all(pb).map_err(Error::from)
--#!}

10
migrations/sqlite/2019-04-28-201506_create_tantivy_index/up.sql Normal file
View file

@ -0,0 +1,10 @@
-- Your SQL goes here
--#!|conn: &Connection, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! let searcher = super::search::Searcher::create(&pb)?;
--#! searcher.fill(conn)?;
--#! searcher.commit();
--#! Ok(())
--#!}

13
plume-cli/src/main.rs
View file

@ -10,6 +10,7 @@ use plume_models::{Connection as Conn, CONFIG};
use std::io::{self, prelude::*}; use std::io::{self, prelude::*};
mod instance; mod instance;
mod migration;
mod search; mod search;
mod users; mod users;
@ -19,8 +20,9 @@ fn main() {
.version(env!("CARGO_PKG_VERSION")) .version(env!("CARGO_PKG_VERSION"))
.about("Collection of tools to manage your Plume instance.") .about("Collection of tools to manage your Plume instance.")
.subcommand(instance::command()) .subcommand(instance::command())
.subcommand(users::command()) .subcommand(migration::command())
.subcommand(search::command()); .subcommand(search::command())
.subcommand(users::command());
let matches = app.clone().get_matches(); let matches = app.clone().get_matches();
dotenv::dotenv().ok(); dotenv::dotenv().ok();
@ -30,12 +32,15 @@ fn main() {
("instance", Some(args)) => { ("instance", Some(args)) => {
instance::run(args, &conn.expect("Couldn't connect to the database.")) instance::run(args, &conn.expect("Couldn't connect to the database."))
} }
("users", Some(args)) => { ("migration", Some(args)) => {
users::run(args, &conn.expect("Couldn't connect to the database.")) migration::run(args, &conn.expect("Couldn't connect to the database."))
} }
("search", Some(args)) => { ("search", Some(args)) => {
search::run(args, &conn.expect("Couldn't connect to the database.")) search::run(args, &conn.expect("Couldn't connect to the database."))
} }
("users", Some(args)) => {
users::run(args, &conn.expect("Couldn't connect to the database."))
}
_ => app.print_help().expect("Couldn't print help"), _ => app.print_help().expect("Couldn't print help"),
}; };
} }

59
plume-cli/src/migration.rs Normal file
View file

@ -0,0 +1,59 @@
use clap::{App, Arg, ArgMatches, SubCommand};
use plume_models::{migrations::IMPORTED_MIGRATIONS, Connection};
use std::path::Path;
pub fn command<'a, 'b>() -> App<'a, 'b> {
SubCommand::with_name("migration")
.about("Manage migrations")
.subcommand(
SubCommand::with_name("run")
.arg(
Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(false)
.help("Path to Plume's working directory"),
)
.about("Run migrations"),
)
.subcommand(
SubCommand::with_name("redo")
.arg(
Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(false)
.help("Path to Plume's working directory"),
)
.about("Rerun latest migration"),
)
}
pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let conn = conn;
match args.subcommand() {
("run", Some(x)) => run_(x, conn),
("redo", Some(x)) => redo(x, conn),
("", None) => command().print_help().unwrap(),
_ => println!("Unknown subcommand"),
}
}
fn run_<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let path = args.value_of("path").unwrap_or(".");
IMPORTED_MIGRATIONS
.run_pending_migrations(conn, Path::new(path))
.expect("Failed to run migrations")
}
fn redo<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let path = args.value_of("path").unwrap_or(".");
IMPORTED_MIGRATIONS
.rerun_last_migration(conn, Path::new(path))
.expect("Failed to rerun migrations")
}

16
plume-cli/src/search.rs
View file

@ -1,7 +1,6 @@
use clap::{App, Arg, ArgMatches, SubCommand}; use clap::{App, Arg, ArgMatches, SubCommand};
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use plume_models::{posts::Post, schema::posts, search::Searcher, Connection, CONFIG}; use plume_models::{search::Searcher, Connection, CONFIG};
use std::fs::{read_dir, remove_file}; use std::fs::{read_dir, remove_file};
use std::io::ErrorKind; use std::io::ErrorKind;
use std::path::Path; use std::path::Path;
@ -98,18 +97,7 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
let path = Path::new(path).join("search_index"); let path = Path::new(path).join("search_index");
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap()); let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
let posts = posts::table searcher.fill(conn).expect("Couldn't import post");
.filter(posts::published.eq(true))
.load::<Post>(conn)
.expect("Post::get_recents: loading error");
let len = posts.len();
for (i, post) in posts.iter().enumerate() {
println!("Importing {}/{} : {}", i + 1, len, post.title);
searcher
.update_document(conn, &post)
.expect("Couldn't import post");
}
println!("Commiting result"); println!("Commiting result");
searcher.commit(); searcher.commit();
} }

21
plume-macro/Cargo.toml Normal file
View file

@ -0,0 +1,21 @@
[package]
name = "plume-macro"
version = "0.1.0"
authors = ["Trinity Pointard <trinity.pointard@insa-rennes.fr>"]
edition = "2018"
description = "Plume procedural macros"
license = "AGPLv3"
[lib]
proc-macro = true
[dependencies]
proc-macro2 = "0.4"
quote = "0.6.12"
syn = "0.11.4"
[features]
default = []
postgres = []
sqlite = []

139
plume-macro/src/lib.rs Normal file
View file

@ -0,0 +1,139 @@
#![recursion_limit = "128"]
extern crate proc_macro;
#[macro_use]
extern crate quote;
extern crate syn;
use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use std::fs::{read_dir, File};
use std::io::Read;
use std::str::FromStr;
#[proc_macro]
pub fn import_migrations(input: TokenStream) -> TokenStream {
assert!(input.is_empty());
let migration_dir = if cfg!(feature = "postgres") {
"migrations/postgres"
} else if cfg!(feature = "sqlite") {
"migrations/sqlite"
} else {
"migrations"
};
let mut files = read_dir(migration_dir)
.unwrap()
.map(|dir| dir.unwrap())
.filter(|dir| dir.file_type().unwrap().is_dir())
.map(|dir| dir.path())
.collect::<Vec<_>>();
files.sort_unstable();
let migrations = files
.into_iter()
.map(|path| {
let mut up = path.clone();
let mut down = path.clone();
up.push("up.sql");
down.push("down.sql");
let mut up_sql = String::new();
let mut down_sql = String::new();
File::open(up).unwrap().read_to_string(&mut up_sql).unwrap();
File::open(down)
.unwrap()
.read_to_string(&mut down_sql)
.unwrap();
let name = path
.file_name()
.unwrap()
.to_str()
.unwrap()
.chars()
.filter(char::is_ascii_digit)
.take(14)
.collect::<String>();
(name, up_sql, down_sql)
})
.collect::<Vec<_>>();
let migrations_name = migrations.iter().map(|m| &m.0).collect::<Vec<_>>();
let migrations_up = migrations
.iter()
.map(|m| m.1.as_str())
.map(file_to_migration)
.collect::<Vec<_>>();
let migrations_down = migrations
.iter()
.map(|m| m.2.as_str())
.map(file_to_migration)
.collect::<Vec<_>>();
/*
enum Action {
Sql(&'static str),
Function(&'static Fn(&Connection, &Path) -> Result<()>)
}*/
quote!(
ImportedMigrations(
&[#(ComplexMigration{name: #migrations_name, up: #migrations_up, down: #migrations_down}),*]
)
).into()
}
fn file_to_migration(file: &str) -> TokenStream2 {
let mut sql = true;
let mut acc = String::new();
let mut actions = vec![];
for line in file.lines() {
if sql {
if line.starts_with("--#!") {
if !acc.trim().is_empty() {
actions.push(quote!(Action::Sql(#acc)));
}
sql = false;
acc = line[4..].to_string();
acc.push('\n');
} else if line.starts_with("--") {
continue;
} else {
acc.push_str(line);
acc.push('\n');
}
} else {
if line.starts_with("--#!") {
acc.push_str(&line[4..]);
acc.push('\n');
} else if line.starts_with("--") {
continue;
} else {
let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into());
actions.push(quote!(Action::Function(&#func)));
sql = true;
acc = line.to_string();
acc.push('\n');
}
}
}
if !acc.trim().is_empty() {
if sql {
actions.push(quote!(Action::Sql(#acc)));
} else {
let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into());
actions.push(quote!(Action::Function(&#func)));
}
}
quote!(
&[#(#actions),*]
)
}
/// Build a trampoline to allow reference to closure from const context
fn trampoline(closure: TokenStream2) -> TokenStream2 {
quote! {
{
fn trampoline<'a, 'b>(conn: &'a Connection, path: &'b Path) -> Result<()> {
(#closure)(conn, path)
}
trampoline
}
}
}

8
plume-models/Cargo.toml
View file

@ -12,6 +12,7 @@ guid-create = "0.1"
heck = "0.3.0" heck = "0.3.0"
itertools = "0.8.0" itertools = "0.8.0"
lazy_static = "*" lazy_static = "*"
migrations_internals= "1.4.0"
openssl = "0.10.15" openssl = "0.10.15"
rocket = "0.4.0" rocket = "0.4.0"
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" } rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
@ -39,9 +40,12 @@ path = "../plume-api"
[dependencies.plume-common] [dependencies.plume-common]
path = "../plume-common" path = "../plume-common"
[dependencies.plume-macro]
path = "../plume-macro"
[dev-dependencies] [dev-dependencies]
diesel_migrations = "1.3.0" diesel_migrations = "1.3.0"
[features] [features]
postgres = ["diesel/postgres"] postgres = ["diesel/postgres", "plume-macro/postgres"]
sqlite = ["diesel/sqlite"] sqlite = ["diesel/sqlite", "plume-macro/sqlite"]

23
plume-models/src/lib.rs
View file

@ -1,6 +1,7 @@
#![feature(try_trait)] #![feature(try_trait)]
#![feature(never_type)] #![feature(never_type)]
#![feature(custom_attribute)] #![feature(custom_attribute)]
#![feature(proc_macro_hygiene)]
extern crate activitypub; extern crate activitypub;
extern crate ammonia; extern crate ammonia;
@ -14,9 +15,12 @@ extern crate heck;
extern crate itertools; extern crate itertools;
#[macro_use] #[macro_use]
extern crate lazy_static; extern crate lazy_static;
extern crate migrations_internals;
extern crate openssl; extern crate openssl;
extern crate plume_api; extern crate plume_api;
extern crate plume_common; extern crate plume_common;
#[macro_use]
extern crate plume_macro;
extern crate reqwest; extern crate reqwest;
extern crate rocket; extern crate rocket;
extern crate rocket_i18n; extern crate rocket_i18n;
@ -32,10 +36,6 @@ extern crate url;
extern crate webfinger; extern crate webfinger;
extern crate whatlang; extern crate whatlang;
#[cfg(test)]
#[macro_use]
extern crate diesel_migrations;
use plume_common::activity_pub::inbox::InboxError; use plume_common::activity_pub::inbox::InboxError;
#[cfg(not(any(feature = "sqlite", feature = "postgres")))] #[cfg(not(any(feature = "sqlite", feature = "postgres")))]
@ -302,18 +302,15 @@ mod tests {
use diesel::r2d2::ConnectionManager; use diesel::r2d2::ConnectionManager;
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
use diesel::{dsl::sql_query, RunQueryDsl}; use diesel::{dsl::sql_query, RunQueryDsl};
use migrations::IMPORTED_MIGRATIONS;
use plume_common::utils::random_hex;
use scheduled_thread_pool::ScheduledThreadPool; use scheduled_thread_pool::ScheduledThreadPool;
use search; use search;
use std::env::temp_dir;
use std::sync::Arc; use std::sync::Arc;
use Connection as Conn; use Connection as Conn;
use CONFIG; use CONFIG;
#[cfg(feature = "sqlite")]
embed_migrations!("../migrations/sqlite");
#[cfg(feature = "postgres")]
embed_migrations!("../migrations/postgres");
#[macro_export] #[macro_export]
macro_rules! part_eq { macro_rules! part_eq {
( $x:expr, $y:expr, [$( $var:ident ),*] ) => { ( $x:expr, $y:expr, [$( $var:ident ),*] ) => {
@ -335,7 +332,10 @@ mod tests {
.connection_customizer(Box::new(db_conn::PragmaForeignKey)) .connection_customizer(Box::new(db_conn::PragmaForeignKey))
.build(ConnectionManager::<Conn>::new(CONFIG.database_url.as_str())) .build(ConnectionManager::<Conn>::new(CONFIG.database_url.as_str()))
.unwrap(); .unwrap();
embedded_migrations::run(&*pool.get().unwrap()).expect("Migrations error"); let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
IMPORTED_MIGRATIONS
.run_pending_migrations(&pool.get().unwrap(), &dir)
.expect("Migrations error");
pool pool
}; };
} }
@ -365,6 +365,7 @@ pub mod instance;
pub mod likes; pub mod likes;
pub mod medias; pub mod medias;
pub mod mentions; pub mod mentions;
pub mod migrations;
pub mod notifications; pub mod notifications;
pub mod plume_rocket; pub mod plume_rocket;
pub mod post_authors; pub mod post_authors;

122
plume-models/src/migrations.rs Normal file
View file

@ -0,0 +1,122 @@
use Connection;
use Error;
use Result;
use diesel::connection::{Connection as Conn, SimpleConnection};
use migrations_internals::{setup_database, MigrationConnection};
use std::path::Path;
#[allow(dead_code)] //variants might not be constructed if not required by current migrations
enum Action {
Sql(&'static str),
Function(&'static Fn(&Connection, &Path) -> Result<()>),
}
impl Action {
fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
match self {
Action::Sql(sql) => conn.batch_execute(sql).map_err(Error::from),
Action::Function(f) => f(conn, path),
}
}
}
struct ComplexMigration {
name: &'static str,
up: &'static [Action],
down: &'static [Action],
}
impl ComplexMigration {
fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
println!("Running migration {}", self.name);
for step in self.up {
step.run(conn, path)?
}
Ok(())
}
fn revert(&self, conn: &Connection, path: &Path) -> Result<()> {
println!("Reverting migration {}", self.name);
for step in self.down {
step.run(conn, path)?
}
Ok(())
}
}
pub struct ImportedMigrations(&'static [ComplexMigration]);
impl ImportedMigrations {
pub fn run_pending_migrations(&self, conn: &Connection, path: &Path) -> Result<()> {
use diesel::dsl::sql;
use diesel::sql_types::Bool;
use diesel::{select, RunQueryDsl};
#[cfg(feature = "postgres")]
let schema_exists: bool = select(sql::<Bool>(
"EXISTS \
(SELECT 1 \
FROM information_schema.tables \
WHERE table_name = '__diesel_schema_migrations')",
))
.get_result(conn)?;
#[cfg(feature = "sqlite")]
let schema_exists: bool = select(sql::<Bool>(
"EXISTS \
(SELECT 1 \
FROM sqlite_master \
WHERE type = 'table' \
AND name = '__diesel_schema_migrations')",
))
.get_result(conn)?;
if !schema_exists {
setup_database(conn)?;
}
let latest_migration = conn.latest_run_migration_version()?;
let latest_id = if let Some(migration) = latest_migration {
self.0
.binary_search_by_key(&migration.as_str(), |mig| mig.name)
.map(|id| id + 1)
.map_err(|_| Error::NotFound)?
} else {
0
};
let to_run = &self.0[latest_id..];
for migration in to_run {
conn.transaction(|| {
migration.run(conn, path)?;
conn.insert_new_migration(migration.name)
.map_err(Error::from)
})?;
}
Ok(())
}
pub fn is_pending(&self, conn: &Connection) -> Result<bool> {
let latest_migration = conn.latest_run_migration_version()?;
if let Some(migration) = latest_migration {
Ok(self.0.last().expect("no migrations found").name != migration)
} else {
Ok(true)
}
}
pub fn rerun_last_migration(&self, conn: &Connection, path: &Path) -> Result<()> {
let latest_migration = conn.latest_run_migration_version()?;
let id = latest_migration
.and_then(|m| self.0.binary_search_by_key(&m.as_str(), |m| m.name).ok())?;
let migration = &self.0[id];
conn.transaction(|| {
migration.revert(conn, path)?;
migration.run(conn, path)
})
}
}
pub const IMPORTED_MIGRATIONS: ImportedMigrations = {
import_migrations! {}
};

12
plume-models/src/search/searcher.rs
View file

@ -1,9 +1,11 @@
use instance::Instance; use instance::Instance;
use posts::Post; use posts::Post;
use schema::posts;
use tags::Tag; use tags::Tag;
use Connection; use Connection;
use chrono::Datelike; use chrono::Datelike;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use itertools::Itertools; use itertools::Itertools;
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex}; use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use tantivy::{ use tantivy::{
@ -222,6 +224,16 @@ impl Searcher {
.collect() .collect()
} }
pub fn fill(&self, conn: &Connection) -> Result<()> {
for post in posts::table
.filter(posts::published.eq(true))
.load::<Post>(conn)?
{
self.update_document(conn, &post)?
}
Ok(())
}
pub fn commit(&self) { pub fn commit(&self) {
let mut writer = self.writer.lock().unwrap(); let mut writer = self.writer.lock().unwrap();
writer.as_mut().unwrap().commit().unwrap(); writer.as_mut().unwrap().commit().unwrap();

17
plume-models/tests/lib.rs
View file

@ -1,22 +1,21 @@
extern crate diesel; extern crate diesel;
#[macro_use] extern crate plume_common;
extern crate diesel_migrations;
extern crate plume_models; extern crate plume_models;
use diesel::Connection; use diesel::Connection;
use plume_common::utils::random_hex;
use plume_models::migrations::IMPORTED_MIGRATIONS;
use plume_models::{Connection as Conn, CONFIG}; use plume_models::{Connection as Conn, CONFIG};
#[cfg(feature = "sqlite")] use std::env::temp_dir;
embed_migrations!("../migrations/sqlite");
#[cfg(feature = "postgres")]
embed_migrations!("../migrations/postgres");
fn db() -> Conn { fn db() -> Conn {
let conn = let conn =
Conn::establish(CONFIG.database_url.as_str()).expect("Couldn't connect to the database"); Conn::establish(CONFIG.database_url.as_str()).expect("Couldn't connect to the database");
embedded_migrations::run(&conn).expect("Couldn't run migrations"); let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
IMPORTED_MIGRATIONS
.run_pending_migrations(&conn, &dir)
.expect("Couldn't run migrations");
conn conn
} }

1
script/generate_artifact.sh
View file

@ -1,6 +1,5 @@
#!/bin/bash #!/bin/bash
mkdir bin mkdir bin
cp target/release/{plume,plm} bin cp target/release/{plume,plm} bin
cp "$(which diesel)" bin
strip -s bin/* strip -s bin/*
tar -cvzf plume.tar.gz bin/ static/ migrations/$FEATURES tar -cvzf plume.tar.gz bin/ static/ migrations/$FEATURES

5
script/run_browser_test.sh
View file

@ -7,11 +7,10 @@ mkdir -p "target/cov/plume"
mkdir -p "target/cov/plm" mkdir -p "target/cov/plm"
plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm' plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm'
diesel migration run $plm migration run
diesel migration redo $plm migration redo
$plm instance new -d plume-test.local -n plume-test $plm instance new -d plume-test.local -n plume-test
$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' $plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
$plm search init
kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume & kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume &
caddy -conf /Caddyfile & caddy -conf /Caddyfile &

4
script/run_unit_test.sh
View file

@ -4,6 +4,10 @@ for file in target/debug/*-*[^\.d]; do
if [[ -x "$file" ]] if [[ -x "$file" ]]
then then
filename=$(basename $file) filename=$(basename $file)
if [[ $filename =~ ^plume_macro ]]; then
rm $file
continue
fi
mkdir -p "target/cov/$filename" mkdir -p "target/cov/$filename"
kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file" kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file"
rm $file rm $file

17
src/main.rs
View file

@ -41,6 +41,7 @@ extern crate webfinger;
use diesel::r2d2::ConnectionManager; use diesel::r2d2::ConnectionManager;
use plume_models::{ use plume_models::{
db_conn::{DbPool, PragmaForeignKey}, db_conn::{DbPool, PragmaForeignKey},
migrations::IMPORTED_MIGRATIONS,
search::{Searcher as UnmanagedSearcher, SearcherError}, search::{Searcher as UnmanagedSearcher, SearcherError},
Connection, Error, CONFIG, Connection, Error, CONFIG,
}; };
@ -83,6 +84,22 @@ fn init_pool() -> Option<DbPool> {
fn main() { fn main() {
let dbpool = init_pool().expect("main: database pool initialization error"); let dbpool = init_pool().expect("main: database pool initialization error");
if IMPORTED_MIGRATIONS
.is_pending(&dbpool.get().unwrap())
.unwrap_or(true)
{
panic!(
r#"
It appear your database migration does not run the migration required
by this version of Plume. To fix this, you can run migrations via
this command:
plm migration run
Then try to restart Plume.
"#
)
}
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
// we want a fast exit here, so // we want a fast exit here, so
#[allow(clippy::match_wild_err_arm)] #[allow(clippy::match_wild_err_arm)]