rewrite circleci config (#558)

* rewrite circleci config with better syntax

and try using grcov for coverage

* attempt to use less memory and to rerun if memory lack

* actually remove use of kcov

* use the right var

* try looping

* run test serialized

* use feature where we should and make macro path independant

* cargo fmt and remove some '"'

* add zip to dependancies

* reduce number of used thread after first failure

* push translations to crowdin on commit to master

* don't cache result of coverage

.circleci/config.yml

@@ -1,314 +1,253 @@
 version: 2.1
 
-aliases:
+executors:
-  - &plume-docker
+  default:
-    image: plumeorg/plume-buildenv:v0.0.5
+    parameters:
-  - &defaults
+      postgres:
+        type: boolean
+        default: false
+      selenium:
+        type: boolean
+        default: false
     docker:
-      - *plume-docker
+    - image: plumeorg/plume-buildenv:v0.0.7
-    working_directory: ~/projects/Plume
+    - image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<</parameters.postgres>><<^parameters.postgres>>alpine:latest<</parameters.postgres>>
-  - &postgresql
-    docker:
-      - *plume-docker
-      - image: circleci/postgres:9.6-alpine
       environment:
         POSTGRES_USER: postgres
         POSTGRES_DB: plume
+    - image: <<#parameters.selenium>>elgalu/selenium:latest<</parameters.selenium>><<^parameters.selenium>>alpine:latest<</parameters.selenium>>
     working_directory: ~/projects/Plume
-  - &selenium
-    docker:
-      - *plume-docker
-      - image: elgalu/selenium:latest
-    working_directory: ~/projects/Plume
-  - &postgresql_selenium
-    docker:
-      - *plume-docker
-      - image: circleci/postgres:9.6-alpine
     environment:
-          POSTGRES_USER: postgres
-          POSTGRES_DB: plume
-      - image: elgalu/selenium:latest
-    working_directory: ~/projects/Plume
-
-  - &attach_workspace
-    attach_workspace:
-      at: ~/projects/Plume/
-  - &persist_to_workspace
-    persist_to_workspace:
-      root: ~/projects/Plume/
-      paths:
-        - ./
-
-  - &env_postgresql
-    environment:
-      MIGRATION_DIRECTORY: migrations/postgres
-      FEATURES: postgres
-      DATABASE_URL: postgres://postgres@localhost/plume
-      RUST_TEST_THREADS: 1
-  - &env_sqlite
-    environment:
-      MIGRATION_DIRECTORY: migrations/sqlite
-      FEATURES: sqlite
-      DATABASE_URL: plume.sqlite3
       RUST_TEST_THREADS: 1
+      FEATURES: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
+      DATABASE_URL: <<#parameters.postgres>>postgres://postgres@localhost/plume<</parameters.postgres>><<^parameters.postgres>>plume.sqlite<</parameters.postgres>>
 
-  - &restore_cache
+
-    restore_cache:
+commands:
+  restore_env:
+    description: checkout and pull cache
+    parameters:
+      cache:
+        type: enum
+        default: none
+        enum: ["none", "clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
+    steps:
+    - checkout
+    - run: git config --global --remove-section url."ssh://git@github.com"
+    - restore_cache:
         keys:
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}
+        - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
-        - v2-plume-notest
+        - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-master
-  - &save_cache
+
-    save_cache:
+  cache:
-      key: v2-plume-notest-{{ checksum "Cargo.lock" }}
+    description: push cache
+    parameters:
+      cache:
+        type: enum
+        enum: ["clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
+    steps:
+    - save_cache:
+        key: v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
         paths:
         - ~/.cargo/
         - ./target
 
-  - &restore_cache_web
+  clippy:
-    restore_cache:
+    description: run cargo clippy
-      keys:
+    parameters:
-        - v2-plume-web-{{ checksum "Cargo.lock" }}
+      package:
-        - v2-plume-web
+        type: string
-  - &save_cache_web
+        default: plume
-    save_cache:
+      no_feature:
-      key: v2-plume-web-{{ checksum "Cargo.lock" }}
+        type: boolean
-      paths:
+        default: false
-        - ~/.cargo/
-        - ./target
-
-  - &restore_cache_cli
-    restore_cache:
-      keys:
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}
-        - v2-plume-notest
-  - &save_cache_cli
-    save_cache:
-      key: v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
-      paths:
-        - ~/.cargo/
-        - target
-
-  - &restore_cache_release
-    restore_cache:
-      keys:
-        - v2-plume-release-{{ checksum "Cargo.lock" }}
-        - v2-plume-release
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}
-        - v2-plume-notest
-  - &save_cache_release
-    save_cache:
-      key: v2-plume-release-{{ checksum "Cargo.lock" }}
-      paths:
-        - ~/.cargo
-        - target/release
-        - target/wasm32-unknown-unknown/release
-
-  - &restore_cache_plume_dead_code
-    restore_cache:
-      keys:
-        - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
-        - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}
-        - v2-plume-test-{{ checksum "/FEATURES" }}
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
-        - v2-plume-notest-{{ checksum "Cargo.lock" }}
-        - v2-plume-notest
-  - &save_cache_plume_dead_code
-    save_cache:
-      key: v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
-      paths:
-        - ~/.cargo/
-        - target
-
-  - &test_cli
     steps:
-      - *attach_workspace
+    - run: cargo clippy <<^parameters.no_feature>>--no-default-features --features="${FEATURES}"<</parameters.no_feature>> --release -p <<parameters.package>> -- -D warnings
-      - run:
-          name: Set cache key
-          command: echo "$FEATURES" > /FEATURES
-      - *restore_cache_cli
-      - run:
-           name: clippy
-           command: cargo clippy --no-default-features --features="${FEATURES}" --release -p plume-cli -- -D warnings
-      - *save_cache_cli
-  - &test_unit
-    steps:
-      - *attach_workspace
-      - run:
-          name: Set cache key
-          command: echo "$FEATURES" > /FEATURES
-      - *restore_cache_plume_dead_code
-      - run:
-          name: clippy
-          command: cargo clippy --no-default-features --features="${FEATURES}" --release -- -D warnings
-      - run:
-          name: compile test
-          command: cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run || cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run
-      - run:
-          name: run test
-          command: ./script/run_unit_test.sh
-      - run:
-          name: upload coverage
-          command: ./script/upload_coverage.sh unit
-      - *save_cache_plume_dead_code
 
-  - &test_browser
+  run_with_coverage:
+    description: run command with environment for coverage
+    parameters:
+      cmd:
+        type: string
     steps:
-      - *attach_workspace
+    - run: |
-      - run:
+       export RUSTFLAGS="-Zprofile -Zfewer-names -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads -Clink-arg=-Xlinker -Clink-arg=--no-keep-memory -Clink-arg=-Xlinker -Clink-arg=--reduce-memory-overheads"
-          name: Set cache key
+       export CARGO_INCREMENTAL=0
-          command: echo "$FEATURES" > /FEATURES
+       << parameters.cmd >>
-      - *restore_cache_plume_dead_code
+
-      - run:
+  upload_coverage:
-          name: install server
+    description: merge coverage files and upload to codecov.io
-          command: cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force || cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force
+    parameters:
-      - run:
+      type:
-          name: install plm
+        type: string
-          command: cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force || cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force
+    steps:
+    - run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'`
+    - run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info
+    - run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <<parameters.type>>
+    - run: find . -name 'plume*.gc*' -delete -o -name 'plm*.gc*' -delete 
+    - run: rm ccov.zip lcov.info
+
+  build:
+    description: build a package
+    parameters:
+      package:
+        type: string
+        default: plume
+      release:
+        type: boolean
+        default: false
+    steps:
+    - run: |
+            cmd="cargo build <<#parameters.release>>--release<</parameters.release>> --no-default-features --features="${FEATURES}" -p <<parameters.package>> -j"
+            for i in 36 4 2 1 1; do
+                $cmd $i && exit 0
+            done
+            exit 1
+
+jobs:
+  cargo fmt:
+    executor:
+      name: default
+    steps:
+    - restore_env
+    - run: cargo fmt --all -- --check
+
+  clippy:
+    parameters:
+      postgres:
+        type: boolean
+    executor:
+      name: default
+      postgres: << parameters.postgres >>
+    steps:
+    - restore_env:
+        cache: clippy
+    - clippy
+    - clippy:
+        package: plume-cli
+    - clippy:
+        package: plume-front
+        no_feature: true
+    - cache:
+        cache: clippy
+
+  unit:
+    parameters:
+      postgres:
+        type: boolean
+    executor:
+      name: default
+      postgres: << parameters.postgres >>
+    steps:
+    - restore_env:
+        cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
+    - run_with_coverage:
+        cmd: |
+          cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j"
+          for i in 36 4 2 1 1; do
+              $cmd $i && break
+          done
+          cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -j1 -- --test-threads=1
+    - upload_coverage:
+        type: unit
+    - cache:
+        cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
+
+  integration:
+    parameters:
+      postgres:
+        type: boolean
+    executor:
+      name: default
+      postgres: << parameters.postgres >>
+      selenium: true
+    steps:
+    - restore_env:
+        cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
+    - run: cargo web deploy -p plume-front
+    - run_with_coverage:
+        cmd: |
+            cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . -j"
+            for i in 36 4 2 1 1; do
+                $cmd $i && exit 0
+            done
+            exit 1
+    - run_with_coverage:
+        cmd: |
+            cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli -j"
+            for i in 36 4 2 1 1; do
+                $cmd $i && exit 0
+            done
+            exit 1
     - run:
         name: run test
         command: ./script/run_browser_test.sh
         environment:
           BROWSER: firefox
-      - run:
+    - upload_coverage:
-          name: upload coverage
+        type: integration
-          command: ./script/upload_coverage.sh integration
+    - cache:
-      - *save_cache_plume_dead_code
+        cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
 
-  - &release
+  release:
+    parameters:
+      postgres:
+        type: boolean
+    executor:
+      name: default
+      postgres: << parameters.postgres >>
     steps:
-      - *attach_workspace
+    - restore_env:
-      - *restore_cache_release
+        cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
-      - run:
+    - run: cargo web deploy -p plume-front --release
-          name: build frontend
+    - build:
-          command: cargo web deploy -p plume-front --release
+        package: plume
-      - run:
+        release: true
-          name: build server
+    - build:
-          command: cargo build --release --no-default-features --features="${FEATURES}" || cargo build --release --no-default-features --features="${FEATURES}"
+        package: plume-cli
-      - run:
+        release: true
-          name: build plm
+    - cache:
-          command: cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli || cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli
+        cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
-      - *save_cache_release
+    - run: ./script/generate_artifact.sh
-      - run: script/generate_artifact.sh
+    - unless:
-      - run: script/upload_test_environment.sh
+        condition: << parameters.postgres >>
+        steps:
+        - run: ./script/upload_test_environment.sh
     - store_artifacts:
         path: plume.tar.gz
         destination: plume.tar.gz
 
-
+  push translations:
-jobs:
+    executor:
-  download_deps:
+      name: default
-    <<: *defaults
     steps:
-      - checkout
+    - restore_env:
-      - *attach_workspace
+        cache: none
-      - *restore_cache
+    - run: crowdin upload
-      - run: git config --global --remove-section url."ssh://git@github.com"
-      - run: cargo fetch
-      - *save_cache
-      - *persist_to_workspace
-
-  cargo_fmt:
-    <<: *defaults
-    steps:
-      - *attach_workspace
-      - run: cargo fmt --all -- --check
-
-  build_web:
-    <<: *defaults
-    steps:
-      - *attach_workspace
-      - *restore_cache_web
-      - run: cargo clippy -p plume-front -- -D warnings
-      - run: cargo web deploy -p plume-front
-      - *save_cache_web
-      - *persist_to_workspace
-
-  test_cli_postgresql:
-    <<: *postgresql
-    <<: *env_postgresql
-    <<: *test_cli
-
-  test_cli_sqlite:
-    <<: *defaults
-    <<: *env_sqlite
-    <<: *test_cli
-
-  test_unit_postgresql:
-    <<: *postgresql
-    <<: *env_postgresql
-    <<: *test_unit
-
-  test_unit_sqlite:
-    <<: *defaults
-    <<: *env_sqlite
-    <<: *test_unit
-
-  test_browser_postgresql:
-    <<: *postgresql_selenium
-    <<: *env_postgresql
-    <<: *test_browser
-
-  test_browser_sqlite:
-    <<: *selenium
-    <<: *env_sqlite
-    <<: *test_browser
-  all_ok:
-    docker:
-      - image: alpine:3.7
-    steps:
-      - run: /bin/true
-  release_postgresql:
-    <<: *defaults
-    <<: *env_postgresql
-    <<: *release
-  release_sqlite:
-    <<: *defaults
-    <<: *env_sqlite
-    <<: *release
 
 workflows:
   version: 2
-  build_and_test:
+  build and test:
     jobs:
-      - download_deps
+    - cargo fmt
-      - cargo_fmt:
+    - clippy:
-          requires:
+        postgres: false
-            - download_deps
+    - clippy:
-      - build_web:
+        postgres: true
-          requires:
+    - unit:
-            - download_deps
+        postgres: false
-      - test_cli_postgresql:
+    - unit:
-          requires:
+        postgres: true
-            - download_deps
+    - integration:
-      - test_cli_sqlite:
+        postgres: false
-          requires:
+    - integration:
-            - download_deps
+        postgres: true
-      - test_unit_postgresql:
+    - release:
-          requires:
+        postgres: false
-            - download_deps
+    - release:
-      - test_unit_sqlite:
+        postgres: true
-          requires:
+    - push translations:
-            - download_deps
+        filters:
-      - test_browser_postgresql:
+          branches:
-          requires:
+            only:
-            - build_web
+              - /^master/
-      - test_browser_sqlite:
-          requires:
-            - build_web
-      - all_ok:
-          requires:
-            - cargo_fmt
-            - test_cli_postgresql
-            - test_cli_sqlite
-            - test_unit_postgresql
-            - test_unit_sqlite
-            - test_browser_postgresql
-            - test_browser_sqlite
-      - release_postgresql:
-          requires:
-            - all_ok
-      - release_sqlite:
-          requires:
-            - all_ok

.circleci/images/plume-buildenv/Dockerfile

@@ -3,8 +3,9 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 
 #install native/circleci/build dependancies
 RUN apt update &&\
-    apt install -y git ssh tar gzip ca-certificates &&\
+    apt install -y --no-install-recommends git ssh tar gzip ca-certificates default-jre&&\
-    apt install -y binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip
+    apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip unzip &&\
+    rm -rf /var/lib/apt/lists/*
 
 #install and configure rust
 RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\
@@ -13,13 +14,10 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0
 
 #compile some deps
 RUN cargo install cargo-web &&\
+    cargo install grcov &&\
+    strip /root/.cargo/bin/* &&\
     rm -fr ~/.cargo/registry
 
-#install coverage tools
-RUN curl -L https://github.com/SimonKagstrom/kcov/archive/master.tar.gz | tar xz &&\
-    mkdir -p kcov-master/build && cd kcov-master/build && cmake .. && make &&\
-    make install && cd ../.. && rm -rf kcov-master
-
 #set some compilation parametters
 COPY cargo_config /root/.cargo/config
 
@@ -29,3 +27,11 @@ RUN pip3 install selenium
 #install and configure caddy
 RUN curl https://getcaddy.com | bash -s personal
 COPY Caddyfile /Caddyfile
+
+#install crowdin
+RUN mkdir /crowdin && cd /crowdin &&\
+    curl -O https://downloads.crowdin.com/cli/v2/crowdin-cli.zip &&\
+    unzip crowdin-cli.zip && rm crowdin-cli.zip &&\
+    cd * && mv crowdin-cli.jar /usr/local/bin && cd && rm -rf /crowdin &&\
+    /bin/echo -e '#!/bin/sh\njava -jar /usr/local/bin/crowdin-cli.jar $@' > /usr/local/bin/crowdin &&\
+    chmod +x /usr/local/bin/crowdin

.circleci/images/plume-buildenv/cargo_config

@@ -1,3 +1,3 @@
 [target.x86_64-unknown-linux-gnu]
 # link dead code for coverage, attempt to reduce linking memory usage to not get killed
-rustflags = ["-Clink-dead-code", "-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"]
+rustflags = ["-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"]

crowdin.yml

@@ -1,3 +1,5 @@
+"project_identifier": "plume"
+"api_key_env": CROWDIN_API_KEY
 files:
   - source: /po/plume/plume.pot
     translation: /po/plume/%two_letters_code%.po

plume-macro/src/lib.rs

@@ -8,6 +8,7 @@ use proc_macro::TokenStream;
 use proc_macro2::TokenStream as TokenStream2;
 use std::fs::{read_dir, File};
 use std::io::Read;
+use std::path::Path;
 use std::str::FromStr;
 
 #[proc_macro]
@@ -20,7 +21,12 @@ pub fn import_migrations(input: TokenStream) -> TokenStream {
     } else {
         "migrations"
     };
-    let mut files = read_dir(migration_dir)
+    let path = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .ancestors()
+        .find(|path| path.join(migration_dir).is_dir() || path.join(".git").exists())
+        .expect("migrations dir not found")
+        .join(migration_dir);
+    let mut files = read_dir(path)
         .unwrap()
         .map(|dir| dir.unwrap())
         .filter(|dir| dir.file_type().unwrap().is_dir())

script/run_browser_test.sh

@@ -3,16 +3,12 @@ set -eo pipefail
 
 export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
 
-mkdir -p "target/cov/plume"
+plm migration run
-mkdir -p "target/cov/plm"
+plm migration redo
-plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm'
+plm instance new -d plume-test.local -n plume-test
+plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
 
-$plm migration run
+plume &
-$plm migration redo
-$plm instance new -d plume-test.local -n plume-test
-$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
-
-kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume &
 caddy -conf /Caddyfile &
 
 until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null
@@ -22,4 +18,4 @@ python3 -m unittest *.py
 
 kill -SIGINT  %1
 kill -SIGKILL %2
-wait
+sleep 15

script/run_unit_test.sh

@@ -1,15 +0,0 @@
-#!/bin/bash 
-set -eo pipefail
-for file in target/debug/*-*[^\.d]; do
-	if [[ -x "$file" ]]
-	then
-		filename=$(basename $file)
-		if [[ $filename =~ ^plume_macro ]]; then
-			rm $file
-			continue
-		fi
-		mkdir -p "target/cov/$filename"
-		kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file"
-		rm $file
-	fi
-done

script/upload_coverage.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-bash <(curl -s https://codecov.io/bash) -F $1